1 /*
   2  * Copyright (c) 2008-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30 
  31 #include <sys/types.h>
  32 #include <sys/sysmacros.h>
  33 #include <sys/ddi.h>
  34 #include <sys/sunddi.h>
  35 #include <sys/atomic.h>
  36 #include <sys/stream.h>
  37 #include <sys/strsun.h>
  38 #include <sys/strsubr.h>
  39 #include <sys/strft.h>
  40 #include <sys/ksynch.h>
  41 #include <sys/ethernet.h>
  42 #include <sys/crc32.h>
  43 #include <sys/pattr.h>
  44 #include <sys/cpu.h>
  45 
  46 #include <sys/ethernet.h>
  47 #include <inet/ip.h>
  48 
  49 #include <netinet/in.h>
  50 #include <netinet/ip.h>
  51 #include <netinet/tcp.h>
  52 
  53 #include "sfxge.h"
  54 
  55 #include "efx.h"
  56 
  57 /* RXQ flush response timeout (in microseconds) */
  58 #define SFXGE_RX_QFLUSH_USEC    (2000000)
  59 
  60 /* RXQ flush tries in the case of failure */
  61 #define SFXGE_RX_QFLUSH_TRIES   (5)
  62 
  63 /* RXQ default packet buffer preallocation (number of packet buffers) */
  64 #define SFXGE_RX_QPREALLOC      (0)
  65 
  66 /* Receive packet DMA attributes */
  67 static ddi_device_acc_attr_t sfxge_rx_packet_devacc = {
  68 
  69         DDI_DEVICE_ATTR_V0,     /* devacc_attr_version */
  70         DDI_NEVERSWAP_ACC,      /* devacc_attr_endian_flags */
  71         DDI_STRICTORDER_ACC     /* devacc_attr_dataorder */
  72 };
  73 
  74 static ddi_dma_attr_t sfxge_rx_packet_dma_attr = {
  75         DMA_ATTR_V0,            /* dma_attr_version     */
  76         0,                      /* dma_attr_addr_lo     */
  77         0xffffffffffffffffull,  /* dma_attr_addr_hi     */
  78         0xffffffffffffffffull,  /* dma_attr_count_max   */
  79         SFXGE_CPU_CACHE_SIZE,   /* dma_attr_align       */
  80         0xffffffff,             /* dma_attr_burstsizes  */
  81         1,                      /* dma_attr_minxfer     */
  82         0xffffffffffffffffull,  /* dma_attr_maxxfer     */
  83         0xffffffffffffffffull,  /* dma_attr_seg         */
  84         1,                      /* dma_attr_sgllen      */
  85         1,                      /* dma_attr_granular    */
  86         0                       /* dma_attr_flags       */
  87 };
  88 
  89 /* Receive queue DMA attributes */
  90 static ddi_device_acc_attr_t sfxge_rxq_devacc = {
  91 
  92         DDI_DEVICE_ATTR_V0,     /* devacc_attr_version */
  93         DDI_NEVERSWAP_ACC,      /* devacc_attr_endian_flags */
  94         DDI_STRICTORDER_ACC     /* devacc_attr_dataorder */
  95 };
  96 
  97 static ddi_dma_attr_t sfxge_rxq_dma_attr = {
  98         DMA_ATTR_V0,            /* dma_attr_version     */
  99         0,                      /* dma_attr_addr_lo     */
 100         0xffffffffffffffffull,  /* dma_attr_addr_hi     */
 101         0xffffffffffffffffull,  /* dma_attr_count_max   */
 102         EFX_BUF_SIZE,           /* dma_attr_align       */
 103         0xffffffff,             /* dma_attr_burstsizes  */
 104         1,                      /* dma_attr_minxfer     */
 105         0xffffffffffffffffull,  /* dma_attr_maxxfer     */
 106         0xffffffffffffffffull,  /* dma_attr_seg         */
 107         1,                      /* dma_attr_sgllen      */
 108         1,                      /* dma_attr_granular    */
 109         0                       /* dma_attr_flags       */
 110 };
 111 
 112 /* Forward declaration */
 113 static void sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc);
 114 
 115 static int
 116 sfxge_rx_packet_ctor(void *buf, void *arg, int kmflags)
 117 {
 118         sfxge_rx_packet_t *srpp = buf;
 119         sfxge_t *sp = arg;
 120         dev_info_t *dip = sp->s_dip;
 121         int err;
 122 
 123         ASSERT3U(sizeof (srpp->__srp_u1.__srp_s1), <=,
 124             sizeof (srpp->__srp_u1.__srp_pad));
 125         ASSERT3U(sizeof (srpp->__srp_u2.__srp_s2), <=,
 126             sizeof (srpp->__srp_u2.__srp_pad));
 127 
 128         bzero(buf, sizeof (sfxge_rx_packet_t));
 129 
 130         /* Allocate a DMA handle */
 131         err = ddi_dma_alloc_handle(dip, &sfxge_rx_packet_dma_attr,
 132             (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
 133             NULL, &(srpp->srp_dma_handle));
 134         if (err != DDI_SUCCESS)
 135                 goto fail1;
 136 
 137         return (0);
 138 
 139 fail1:
 140         DTRACE_PROBE1(fail1, int, err);
 141 
 142         SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
 143 
 144         return (-1);
 145 }
 146 
 147 static void
 148 sfxge_rx_packet_dtor(void *buf, void *arg)
 149 {
 150         sfxge_rx_packet_t *srpp = buf;
 151 
 152         _NOTE(ARGUNUSED(arg))
 153 
 154         /* Free the DMA handle */
 155         ddi_dma_free_handle(&(srpp->srp_dma_handle));
 156         srpp->srp_dma_handle = NULL;
 157 
 158         SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
 159 }
 160 
 161 static int
 162 sfxge_rx_qctor(void *buf, void *arg, int kmflags)
 163 {
 164         sfxge_rxq_t *srp = buf;
 165         efsys_mem_t *esmp = &(srp->sr_mem);
 166         sfxge_t *sp = arg;
 167         sfxge_dma_buffer_attr_t dma_attr;
 168         sfxge_rx_fpp_t *srfppp;
 169         int nprealloc;
 170         unsigned int id;
 171         int rc;
 172 
 173         /* Compile-time structure layout checks */
 174         EFX_STATIC_ASSERT(sizeof (srp->__sr_u1.__sr_s1) <=
 175             sizeof (srp->__sr_u1.__sr_pad));
 176         EFX_STATIC_ASSERT(sizeof (srp->__sr_u2.__sr_s2) <=
 177             sizeof (srp->__sr_u2.__sr_pad));
 178         EFX_STATIC_ASSERT(sizeof (srp->__sr_u3.__sr_s3) <=
 179             sizeof (srp->__sr_u3.__sr_pad));
 180 
 181         bzero(buf, sizeof (sfxge_rxq_t));
 182 
 183         srp->sr_sp = sp;
 184 
 185         dma_attr.sdba_dip        = sp->s_dip;
 186         dma_attr.sdba_dattrp     = &sfxge_rxq_dma_attr;
 187         dma_attr.sdba_callback   = DDI_DMA_SLEEP;
 188         dma_attr.sdba_length     = EFX_RXQ_SIZE(sp->s_rxq_size);
 189         dma_attr.sdba_memflags   = DDI_DMA_CONSISTENT;
 190         dma_attr.sdba_devaccp    = &sfxge_rxq_devacc;
 191         dma_attr.sdba_bindflags  = DDI_DMA_READ | DDI_DMA_CONSISTENT;
 192         dma_attr.sdba_maxcookies = 1;
 193         dma_attr.sdba_zeroinit   = B_FALSE;
 194 
 195         if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
 196                 goto fail1;
 197 
 198         /* Allocate some buffer table entries */
 199         if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_RXQ_NBUFS(sp->s_rxq_size),
 200             &(srp->sr_id))) != 0)
 201                 goto fail2;
 202 
 203         /* Allocate the context array */
 204         if ((srp->sr_srpp = kmem_zalloc(sizeof (sfxge_rx_packet_t *) *
 205             sp->s_rxq_size, kmflags)) == NULL) {
 206                 rc = ENOMEM;
 207                 goto fail3;
 208         }
 209 
 210         /* Allocate the flow table */
 211         if ((srp->sr_flow = kmem_zalloc(sizeof (sfxge_rx_flow_t) *
 212             SFXGE_MAX_FLOW, kmflags)) == NULL) {
 213                 rc = ENOMEM;
 214                 goto fail4;
 215         }
 216 
 217         srp->sr_srfpp = &(srp->sr_srfp);
 218         srp->sr_rto = drv_usectohz(200000);
 219 
 220         srp->sr_mpp = &(srp->sr_mp);
 221 
 222         /* Initialize the free packet pool */
 223         srfppp = &(srp->sr_fpp);
 224         if ((srfppp->srfpp_putp = kmem_zalloc(SFXGE_CPU_CACHE_SIZE *
 225             SFXGE_RX_FPP_NSLOTS, kmflags)) == NULL) {
 226                 rc = ENOMEM;
 227                 goto fail5;
 228         }
 229         for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
 230                 sfxge_rx_fpp_putlist_t *putp;
 231                 size_t off;
 232 
 233                 off = id * SFXGE_CPU_CACHE_SIZE;
 234                 putp = (void *)(srfppp->srfpp_putp + off);
 235 
 236                 putp->srfpl_putp = NULL;
 237                 putp->srfpl_putpp = &(putp->srfpl_putp);
 238                 mutex_init(&(putp->srfpl_lock), NULL, MUTEX_DRIVER,
 239                     DDI_INTR_PRI(sp->s_intr.si_intr_pri));
 240         }
 241 
 242         cv_init(&(srp->sr_flush_kv), NULL, CV_DRIVER, NULL);
 243 
 244         /* Preallocate some packets on the free packet pool */
 245         nprealloc = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
 246             DDI_PROP_DONTPASS, "rx_prealloc_pkt_buffers", SFXGE_RX_QPREALLOC);
 247         sfxge_rx_qpreallocate(srp, nprealloc);
 248 
 249 
 250         return (0);
 251 
 252 fail5:
 253         DTRACE_PROBE(fail5);
 254 
 255         srp->sr_mpp = NULL;
 256 
 257         srp->sr_rto = 0;
 258         srp->sr_srfpp = NULL;
 259 
 260         /* Free the flow table */
 261         kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
 262             SFXGE_MAX_FLOW);
 263         srp->sr_flow = NULL;
 264 
 265 fail4:
 266         DTRACE_PROBE(fail4);
 267 
 268         /* Free the context array */
 269         kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
 270             sp->s_rxq_size);
 271         srp->sr_srpp = NULL;
 272 
 273 fail3:
 274         DTRACE_PROBE(fail3);
 275 
 276         /* Free the buffer table entries */
 277         sfxge_sram_buf_tbl_free(sp, srp->sr_id,
 278             EFX_RXQ_NBUFS(sp->s_rxq_size));
 279         srp->sr_id = 0;
 280 
 281 fail2:
 282         DTRACE_PROBE(fail2);
 283         /* Remove dma setup */
 284         sfxge_dma_buffer_destroy(esmp);
 285 
 286 fail1:
 287         DTRACE_PROBE1(fail1, int, rc);
 288 
 289         srp->sr_sp = NULL;
 290 
 291         SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
 292 
 293         return (-1);
 294 }
 295 
 296 static void
 297 sfxge_rx_qdtor(void *buf, void *arg)
 298 {
 299         sfxge_rxq_t *srp = buf;
 300         efsys_mem_t *esmp = &(srp->sr_mem);
 301         sfxge_t *sp = srp->sr_sp;
 302         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
 303         unsigned int id;
 304 
 305         _NOTE(ARGUNUSED(arg))
 306 
 307         cv_destroy(&(srp->sr_flush_kv));
 308 
 309         /* Tear down the free packet pool */
 310         for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
 311                 sfxge_rx_fpp_putlist_t *putp;
 312                 size_t off;
 313 
 314                 off = id * SFXGE_CPU_CACHE_SIZE;
 315                 putp = (void *)(srfppp->srfpp_putp + off);
 316 
 317                 putp->srfpl_putpp = NULL;
 318                 mutex_destroy(&(putp->srfpl_lock));
 319 
 320                 SFXGE_OBJ_CHECK(putp, sfxge_rx_fpp_putlist_t);
 321         }
 322         kmem_free(srfppp->srfpp_putp, SFXGE_CPU_CACHE_SIZE *
 323             SFXGE_RX_FPP_NSLOTS);
 324         srfppp->srfpp_putp = NULL;
 325 
 326         srp->sr_mpp = NULL;
 327 
 328         srp->sr_rto = 0;
 329         srp->sr_srfpp = NULL;
 330 
 331         /* Free the flow table */
 332         kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
 333             SFXGE_MAX_FLOW);
 334         srp->sr_flow = NULL;
 335 
 336         /* Free the context array */
 337         kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
 338             sp->s_rxq_size);
 339         srp->sr_srpp = NULL;
 340 
 341         /* Free the buffer table entries */
 342         sfxge_sram_buf_tbl_free(sp, srp->sr_id,
 343             EFX_RXQ_NBUFS(sp->s_rxq_size));
 344         srp->sr_id = 0;
 345 
 346         /* Tear down dma setup */
 347         sfxge_dma_buffer_destroy(esmp);
 348 
 349         SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
 350 }
 351 
 352 /* Note: This function takes ownership of *srpp. */
 353 static inline void
 354 sfxge_rx_qfpp_put(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
 355 {
 356         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
 357         mblk_t *mp = srpp->srp_mp;
 358         unsigned int id;
 359         size_t off;
 360         sfxge_rx_fpp_putlist_t *putp;
 361 
 362         ASSERT3P(mp->b_next, ==, NULL);
 363         ASSERT3P(mp->b_prev, ==, NULL);
 364 
 365         id = CPU->cpu_seqid & SFXGE_RX_FPP_MASK;
 366         off = id * SFXGE_CPU_CACHE_SIZE;
 367 
 368         ASSERT3P(srpp->srp_putp, ==, srfppp->srfpp_putp);
 369         putp = (void *)(srpp->srp_putp + off);
 370 
 371         mutex_enter(&(putp->srfpl_lock));
 372         putp->srfpl_count++;
 373         *putp->srfpl_putpp = mp;
 374         putp->srfpl_putpp = &(mp->b_next);
 375         mutex_exit(&(putp->srfpl_lock));
 376 }
 377 
 378 static unsigned int
 379 sfxge_rx_qfpp_swizzle(sfxge_rxq_t *srp)
 380 {
 381         sfxge_t *sp = srp->sr_sp;
 382         unsigned int index = srp->sr_index;
 383         sfxge_evq_t *sep = sp->s_sep[index];
 384         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
 385         unsigned int start;
 386         unsigned int id;
 387         mblk_t *p;
 388         mblk_t **pp;
 389         unsigned int count;
 390         unsigned int loaned;
 391 
 392         ASSERT(mutex_owned(&(sep->se_lock)));
 393 
 394         /* We want to access the put list for the current CPU last */
 395         id = start = (CPU->cpu_seqid + 1) & SFXGE_RX_FPP_MASK;
 396 
 397         do {
 398                 sfxge_rx_fpp_putlist_t *putp;
 399                 size_t off;
 400 
 401                 off = id * SFXGE_CPU_CACHE_SIZE;
 402                 id  = (id + 1) & SFXGE_RX_FPP_MASK;
 403 
 404                 putp = (void *)(srfppp->srfpp_putp + off);
 405 
 406                 /* Acquire the put list */
 407                 mutex_enter(&(putp->srfpl_lock));
 408 
 409                 p = putp->srfpl_putp;
 410                 pp = putp->srfpl_putpp;
 411                 count = putp->srfpl_count;
 412 
 413                 putp->srfpl_putp = NULL;
 414                 putp->srfpl_putpp = &(putp->srfpl_putp);
 415                 putp->srfpl_count = 0;
 416 
 417                 mutex_exit(&(putp->srfpl_lock));
 418 
 419                 if (p == NULL)
 420                         continue;
 421 
 422                 /* Add the list to the head of the get list */
 423                 *pp = srfppp->srfpp_get;
 424                 srfppp->srfpp_get = p;
 425 
 426                 /* Adjust the counters */
 427                 ASSERT3U(srfppp->srfpp_loaned, >=, count);
 428                 srfppp->srfpp_loaned -= count;
 429                 srfppp->srfpp_count += count;
 430 
 431 #if 0
 432                 /* NOTE: this probe is disabled because it is expensive!! */
 433                 DTRACE_PROBE2(count,
 434                     unsigned int, (id - 1) & SFXGE_RX_FPP_MASK,
 435                     unsigned int, count);
 436 #endif
 437 
 438         } while (id != start);
 439 
 440         /* Return the number of packets yet to appear in the put list */
 441         loaned = srfppp->srfpp_loaned;
 442 
 443 
 444         return (loaned);
 445 }
 446 
 447 
 448 #define DB_FRTNP(mp)    ((mp)->b_datap->db_frtnp)
 449 
 450 static void
 451 sfxge_rx_qfpp_empty(sfxge_rxq_t *srp)
 452 {
 453         sfxge_t *sp = srp->sr_sp;
 454         unsigned int index = srp->sr_index;
 455         sfxge_evq_t *sep = sp->s_sep[index];
 456         sfxge_rx_fpp_t *srfppp;
 457         mblk_t *mp;
 458 
 459         mutex_enter(&(sep->se_lock));
 460         srfppp = &(srp->sr_fpp);
 461 
 462         /* Swizzle put list to get list */
 463         (void) sfxge_rx_qfpp_swizzle(srp);
 464         ASSERT3U(srfppp->srfpp_loaned, ==, 0);
 465 
 466         mp = srfppp->srfpp_get;
 467         srfppp->srfpp_get = NULL;
 468 
 469         /* Free the remainder */
 470         while (mp != NULL) {
 471                 mblk_t *next;
 472                 frtn_t *freep;
 473                 sfxge_rx_packet_t *srpp;
 474 
 475                 next = mp->b_next;
 476                 mp->b_next = NULL;
 477 
 478                 ASSERT3U(srfppp->srfpp_count, >, 0);
 479                 srfppp->srfpp_count--;
 480 
 481                 freep = DB_FRTNP(mp);
 482                 /*
 483                  * ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
 484                  *   is implied by srpp test below
 485                  */
 486                 /*LINTED*/
 487                 srpp = (sfxge_rx_packet_t *)(freep->free_arg);
 488                 ASSERT3P(srpp->srp_mp, ==, mp);
 489                 ASSERT3P(mp->b_cont, ==, NULL);
 490                 srpp->srp_recycle = B_FALSE;
 491 
 492                 freeb(mp);
 493 
 494                 mp = next;
 495         }
 496         ASSERT3U(srfppp->srfpp_count, ==, 0);
 497 
 498         srfppp->srfpp_min = 0;
 499 
 500         mutex_exit(&(sep->se_lock));
 501 }
 502 
 503 /*
 504  * This is an estimate of all memory consumed per RX packet
 505  * it can be inaccurate but but sp->s_rx_pkt_mem_alloc mustn't drift
 506  */
 507 static uint64_t
 508 sfxge_rx_pkt_mem_approx(const sfxge_rx_packet_t *srpp)
 509 {
 510         return (srpp->srp_mblksize + sizeof (mblk_t) + sizeof (dblk_t) +
 511             sizeof (sfxge_rx_packet_t));
 512 }
 513 
 514 static void
 515 sfxge_rx_qpacket_destroy(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
 516 {
 517         sfxge_t *sp = srp->sr_sp;
 518         int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
 519 
 520         ASSERT(!(srpp->srp_recycle));
 521         ASSERT3P(srpp->srp_mp, ==, NULL);
 522 
 523         srpp->srp_off = 0;
 524         srpp->srp_thp = NULL;
 525         srpp->srp_iphp = NULL;
 526         srpp->srp_etherhp = NULL;
 527         srpp->srp_size = 0;
 528         srpp->srp_flags = 0;
 529 
 530         bzero(&(srpp->srp_free), sizeof (frtn_t));
 531 
 532         srpp->srp_mblksize = 0;
 533         srpp->srp_base = NULL;
 534 
 535         /* Unbind the DMA memory from the DMA handle */
 536         srpp->srp_addr = 0;
 537         (void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
 538 
 539         /* Free the DMA memory */
 540         srpp->srp_base = NULL;
 541         ddi_dma_mem_free(&(srpp->srp_acc_handle));
 542         srpp->srp_acc_handle = NULL;
 543 
 544         srpp->srp_putp = NULL;
 545         srpp->srp_srp = NULL;
 546 
 547         kmem_cache_free(sp->s_rpc, srpp);
 548         if (sp->s_rx_pkt_mem_max)
 549                 atomic_add_64(&sp->s_rx_pkt_mem_alloc, -delta);
 550 }
 551 
 552 static void
 553 sfxge_rx_qpacket_free(void *arg)
 554 {
 555         sfxge_rx_packet_t *srpp = arg;
 556         sfxge_rxq_t *srp = srpp->srp_srp;
 557 
 558         /*
 559          * WARNING "man -s 9f esballoc"  states:
 560          * => runs sync from the thread calling freeb()
 561          * => must not sleep, or access data structures that could be freed
 562          */
 563 
 564         /* Check whether we want to recycle the receive packets */
 565         if (srpp->srp_recycle) {
 566                 frtn_t *freep;
 567                 mblk_t *mp;
 568                 size_t size;
 569 
 570                 freep = &(srpp->srp_free);
 571                 ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
 572                 ASSERT3P(freep->free_arg, ==, (caddr_t)srpp);
 573 
 574                 /*
 575                  * Allocate a matching mblk_t before the current one is
 576                  * freed.
 577                  */
 578                 size = srpp->srp_mblksize;
 579 
 580                 if ((mp = desballoc(srpp->srp_base, size, BPRI_HI,
 581                     freep)) != NULL) {
 582                         srpp->srp_mp = mp;
 583 
 584                         /* NORMAL recycled case */
 585                         sfxge_rx_qfpp_put(srp, srpp);
 586                         return;
 587                 }
 588         }
 589 
 590         srpp->srp_mp = NULL;
 591 
 592         sfxge_rx_qpacket_destroy(srp, srpp);
 593 }
 594 
 595 static sfxge_rx_packet_t *
 596 sfxge_rx_qpacket_create(sfxge_rxq_t *srp)
 597 {
 598         sfxge_t *sp = srp->sr_sp;
 599         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
 600         sfxge_rx_packet_t *srpp;
 601         size_t size;
 602         caddr_t base;
 603         size_t unit;
 604         ddi_dma_cookie_t dmac;
 605         unsigned int ncookies;
 606         frtn_t *freep;
 607         mblk_t *mp;
 608         int err;
 609         int rc;
 610 
 611         size = sp->s_rx_buffer_size;
 612 
 613         if (sp->s_rx_pkt_mem_max &&
 614             (sp->s_rx_pkt_mem_alloc + size >= sp->s_rx_pkt_mem_max)) {
 615                 DTRACE_PROBE(rx_pkt_mem_max);
 616                 srp->sr_kstat.srk_rx_pkt_mem_limit++;
 617                 return (NULL);
 618         }
 619 
 620         /* Allocate a new packet */
 621         if ((srpp = kmem_cache_alloc(sp->s_rpc, KM_NOSLEEP)) == NULL) {
 622                 srp->sr_kstat.srk_kcache_alloc_nomem++;
 623                 rc = ENOMEM;
 624                 goto fail1;
 625         }
 626 
 627         srpp->srp_srp = srp;
 628         srpp->srp_putp = srfppp->srfpp_putp;
 629 
 630         /* Allocate some DMA memory */
 631         err = ddi_dma_mem_alloc(srpp->srp_dma_handle, size,
 632             &sfxge_rx_packet_devacc, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
 633             NULL, &base, &unit, &(srpp->srp_acc_handle));
 634         switch (err) {
 635         case DDI_SUCCESS:
 636                 break;
 637 
 638         case DDI_FAILURE:
 639                 srp->sr_kstat.srk_dma_alloc_nomem++;
 640                 rc = ENOMEM;
 641                 goto fail2;
 642 
 643         default:
 644                 srp->sr_kstat.srk_dma_alloc_fail++;
 645                 rc = EFAULT;
 646                 goto fail2;
 647         }
 648 
 649         /* Adjust the buffer to align the start of the DMA area correctly */
 650         base += sp->s_rx_buffer_align;
 651         size -= sp->s_rx_buffer_align;
 652 
 653         /* Bind the DMA memory to the DMA handle */
 654         err = ddi_dma_addr_bind_handle(srpp->srp_dma_handle, NULL,
 655             base, size, DDI_DMA_READ | DDI_DMA_STREAMING,
 656             DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies);
 657         switch (err) {
 658         case DDI_DMA_MAPPED:
 659                 break;
 660 
 661         case DDI_DMA_INUSE:
 662                 srp->sr_kstat.srk_dma_bind_fail++;
 663                 rc = EEXIST;
 664                 goto fail3;
 665 
 666         case DDI_DMA_NORESOURCES:
 667                 srp->sr_kstat.srk_dma_bind_nomem++;
 668                 rc = ENOMEM;
 669                 goto fail3;
 670 
 671         case DDI_DMA_NOMAPPING:
 672                 srp->sr_kstat.srk_dma_bind_fail++;
 673                 rc = ENOTSUP;
 674                 goto fail3;
 675 
 676         case DDI_DMA_TOOBIG:
 677                 srp->sr_kstat.srk_dma_bind_fail++;
 678                 rc = EFBIG;
 679                 goto fail3;
 680 
 681         default:
 682                 srp->sr_kstat.srk_dma_bind_fail++;
 683                 rc = EFAULT;
 684                 goto fail3;
 685         }
 686         ASSERT3U(ncookies, ==, 1);
 687 
 688         srpp->srp_addr = dmac.dmac_laddress;
 689 
 690         srpp->srp_base = (unsigned char *)base;
 691         srpp->srp_mblksize = size;
 692 
 693         /*
 694          * Allocate a STREAMS block: We use size 1 so that the allocator will
 695          * use the first (and smallest) dblk cache.
 696          */
 697         freep = &(srpp->srp_free);
 698         freep->free_func = sfxge_rx_qpacket_free;
 699         freep->free_arg  = (caddr_t)srpp;
 700 
 701         if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) {
 702                 srp->sr_kstat.srk_desballoc_fail++;
 703                 rc = ENOMEM;
 704                 goto fail4;
 705         }
 706 
 707         srpp->srp_mp = mp;
 708         srpp->srp_recycle = B_TRUE;
 709 
 710         if (sp->s_rx_pkt_mem_max) {
 711                 int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
 712                 atomic_add_64(&sp->s_rx_pkt_mem_alloc, delta);
 713         }
 714 
 715         return (srpp);
 716 
 717 fail4:
 718         DTRACE_PROBE(fail4);
 719 
 720         bzero(&(srpp->srp_free), sizeof (frtn_t));
 721 
 722         srpp->srp_mblksize = 0;
 723         srpp->srp_base = NULL;
 724 
 725         /* Unbind the DMA memory from the DMA handle */
 726         srpp->srp_addr = 0;
 727         (void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
 728 
 729 fail3:
 730         DTRACE_PROBE(fail3);
 731 
 732         /* Free the DMA memory */
 733         ddi_dma_mem_free(&(srpp->srp_acc_handle));
 734         srpp->srp_acc_handle = NULL;
 735 
 736 fail2:
 737         DTRACE_PROBE(fail2);
 738 
 739         srpp->srp_putp = NULL;
 740         srpp->srp_srp = NULL;
 741 
 742         kmem_cache_free(sp->s_rpc, srpp);
 743 
 744 fail1:
 745         DTRACE_PROBE1(fail1, int, rc);
 746 
 747         return (NULL);
 748 }
 749 
 750 #define SFXGE_REFILL_BATCH  64
 751 
 752 /* Try to refill the RX descriptor ring from the associated free pkt pool */
 753 static void
 754 sfxge_rx_qrefill(sfxge_rxq_t *srp, unsigned int target)
 755 {
 756         sfxge_t *sp = srp->sr_sp;
 757         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
 758         unsigned int index = srp->sr_index;
 759         sfxge_evq_t *sep = sp->s_sep[index];
 760         efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
 761         mblk_t *mp;
 762         int ntodo;
 763         unsigned int count;
 764         unsigned int batch;
 765         unsigned int rxfill;
 766         unsigned int mblksize;
 767 
 768         prefetch_read_many(sp->s_enp);
 769         prefetch_read_many(srp->sr_erp);
 770 
 771         ASSERT(mutex_owned(&(sep->se_lock)));
 772 
 773         if (srp->sr_state != SFXGE_RXQ_STARTED)
 774                 return;
 775 
 776         rxfill = srp->sr_added - srp->sr_completed;
 777         ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
 778         ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
 779         ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
 780 
 781         if (ntodo == 0)
 782                 goto out;
 783 
 784         (void) sfxge_rx_qfpp_swizzle(srp);
 785 
 786         mp = srfppp->srfpp_get;
 787         count = srfppp->srfpp_count;
 788         mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
 789 
 790         batch = 0;
 791         while (ntodo-- > 0) {
 792                 mblk_t *next;
 793                 frtn_t *freep;
 794                 sfxge_rx_packet_t *srpp;
 795                 unsigned int id;
 796 
 797                 if (mp == NULL)
 798                         break;
 799 
 800                 next = mp->b_next;
 801                 mp->b_next = NULL;
 802 
 803                 if (next != NULL)
 804                         prefetch_read_many(next);
 805 
 806                 freep = DB_FRTNP(mp);
 807                 /*LINTED*/
 808                 srpp = (sfxge_rx_packet_t *)(freep->free_arg);
 809                 ASSERT3P(srpp->srp_mp, ==, mp);
 810 
 811                 /* The MTU may have changed since the packet was allocated */
 812                 if (MBLKSIZE(mp) != mblksize) {
 813                         srpp->srp_recycle = B_FALSE;
 814 
 815                         freeb(mp);
 816 
 817                         --count;
 818                         mp = next;
 819                         continue;
 820                 }
 821 
 822                 srpp->srp_off = 0;
 823                 srpp->srp_thp = NULL;
 824                 srpp->srp_iphp = NULL;
 825                 srpp->srp_etherhp = NULL;
 826                 srpp->srp_size = 0;
 827                 srpp->srp_flags = EFX_DISCARD;
 828 
 829                 id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
 830                 ASSERT(srp->sr_srpp[id] == NULL);
 831                 srp->sr_srpp[id] = srpp;
 832 
 833                 addr[batch++] = srpp->srp_addr;
 834                 if (batch == SFXGE_REFILL_BATCH) {
 835                         efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
 836                             srp->sr_completed, srp->sr_added);
 837                         srp->sr_added += batch;
 838                         batch = 0;
 839                 }
 840 
 841                 --count;
 842                 mp = next;
 843         }
 844 
 845         srfppp->srfpp_get = mp;
 846         srfppp->srfpp_count = count;
 847 
 848         if (batch != 0) {
 849                 efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
 850                     srp->sr_completed, srp->sr_added);
 851                 srp->sr_added += batch;
 852         }
 853 
 854         efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
 855 
 856 out:
 857         if (srfppp->srfpp_count < srfppp->srfpp_min)
 858                 srfppp->srfpp_min = srfppp->srfpp_count;
 859 }
 860 
 861 /* Preallocate packets and put them in the free packet pool */
 862 static void
 863 sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc)
 864 {
 865         sfxge_rx_fpp_t *srfppp = &((srp)->sr_fpp);
 866         srfppp->srfpp_lowat = nprealloc;
 867         while (nprealloc-- > 0) {
 868                 sfxge_rx_packet_t *srpp;
 869 
 870                 if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
 871                         break;
 872                 sfxge_rx_qfpp_put(srp, srpp);
 873         }
 874 }
 875 
 876 /* Try to refill the RX descriptor ring by allocating new packets */
 877 static void
 878 sfxge_rx_qfill(sfxge_rxq_t *srp, unsigned int target)
 879 {
 880         sfxge_t *sp = srp->sr_sp;
 881         unsigned int index = srp->sr_index;
 882         sfxge_evq_t *sep = sp->s_sep[index];
 883         unsigned int batch;
 884         unsigned int rxfill;
 885         unsigned int mblksize;
 886         int ntodo;
 887         efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
 888         mblk_t *mp = NULL;
 889 
 890         prefetch_read_many(sp->s_enp);
 891         prefetch_read_many(srp->sr_erp);
 892 
 893         ASSERT(mutex_owned(&(sep->se_lock)));
 894 
 895         if (srp->sr_state != SFXGE_RXQ_STARTED)
 896                 return;
 897 
 898         rxfill = srp->sr_added - srp->sr_completed;
 899         ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
 900         ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
 901         ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
 902 
 903         if (ntodo == 0)
 904                 return;
 905 
 906         mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
 907 
 908         batch = 0;
 909         while (ntodo-- > 0) {
 910                 sfxge_rx_packet_t *srpp;
 911                 unsigned int id;
 912 
 913                 if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
 914                         break;
 915 
 916                 mp = srpp->srp_mp;
 917 
 918                 ASSERT3U(MBLKSIZE(mp), ==, mblksize);
 919 
 920                 ASSERT3U(srpp->srp_off, ==, 0);
 921                 ASSERT3P(srpp->srp_thp, ==, NULL);
 922                 ASSERT3P(srpp->srp_iphp, ==, NULL);
 923                 ASSERT3P(srpp->srp_etherhp, ==, NULL);
 924                 ASSERT3U(srpp->srp_size, ==, 0);
 925 
 926                 srpp->srp_flags = EFX_DISCARD;
 927 
 928                 id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
 929                 ASSERT(srp->sr_srpp[id] == NULL);
 930                 srp->sr_srpp[id] = srpp;
 931 
 932                 addr[batch++] = srpp->srp_addr;
 933                 if (batch == SFXGE_REFILL_BATCH) {
 934                         efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
 935                             srp->sr_completed, srp->sr_added);
 936                         srp->sr_added += batch;
 937                         batch = 0;
 938                 }
 939         }
 940 
 941         if (batch != 0) {
 942                 efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
 943                     srp->sr_completed, srp->sr_added);
 944                 srp->sr_added += batch;
 945         }
 946 
 947         efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
 948 }
 949 
 950 void
 951 sfxge_rx_qfpp_trim(sfxge_rxq_t *srp)
 952 {
 953         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
 954         sfxge_t *sp = srp->sr_sp;
 955         unsigned int index = srp->sr_index;
 956         sfxge_evq_t *sep = sp->s_sep[index];
 957         mblk_t *p;
 958         mblk_t **pp;
 959         int count;
 960 
 961         ASSERT(mutex_owned(&(sep->se_lock)));
 962 
 963         if (srp->sr_state != SFXGE_RXQ_STARTED)
 964                 goto done;
 965 
 966         /* Make sure the queue is full */
 967         sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
 968 
 969         /* The refill may have emptied the pool */
 970         if (srfppp->srfpp_min == 0)
 971                 goto done;
 972 
 973         /* Don't trim below the pool's low water mark */
 974         if (srfppp->srfpp_count <= srfppp->srfpp_lowat)
 975                 goto done;
 976 
 977         ASSERT(srfppp->srfpp_min <= srfppp->srfpp_count);
 978 
 979         /* Trim to the largest of srfppp->srfpp_min and srfpp->srfpp_lowat */
 980         if (srfppp->srfpp_lowat > srfppp->srfpp_min)
 981                 count = srfppp->srfpp_count - srfppp->srfpp_lowat;
 982         else
 983                 count = srfppp->srfpp_count - srfppp->srfpp_min;
 984 
 985         /* Walk the get list */
 986         pp = &(srfppp->srfpp_get);
 987         while (--count >= 0) {
 988                 ASSERT(pp);
 989                 p = *pp;
 990                 ASSERT(p != NULL);
 991 
 992                 pp = &(p->b_next);
 993         }
 994         ASSERT(pp);
 995         p = *pp;
 996 
 997         /* Truncate the get list */
 998         *pp = NULL;
 999 
1000         /* Free the remainder */
1001         while (p != NULL) {
1002                 mblk_t *next;
1003                 frtn_t *freep;
1004                 sfxge_rx_packet_t *srpp;
1005 
1006                 next = p->b_next;
1007                 p->b_next = NULL;
1008 
1009                 ASSERT3U(srfppp->srfpp_min, >, 0);
1010                 srfppp->srfpp_min--;
1011                 srfppp->srfpp_count--;
1012 
1013                 freep = DB_FRTNP(p);
1014                 /*LINTED*/
1015                 srpp = (sfxge_rx_packet_t *)(freep->free_arg);
1016                 ASSERT3P(srpp->srp_mp, ==, p);
1017 
1018                 srpp->srp_recycle = B_FALSE;
1019 
1020                 freeb(p);
1021 
1022                 p = next;
1023         }
1024 
1025 done:
1026         srfppp->srfpp_min = srfppp->srfpp_count;
1027 }
1028 
1029 static void
1030 sfxge_rx_qpoll(void *arg)
1031 {
1032         sfxge_rxq_t *srp = arg;
1033         sfxge_t *sp = srp->sr_sp;
1034         unsigned int index = srp->sr_index;
1035         sfxge_evq_t *sep = sp->s_sep[index];
1036         uint16_t magic;
1037 
1038         /*
1039          * man timeout(9f) states that this code should adhere to the
1040          * same requirements as a softirq handler - DO NOT BLOCK
1041          */
1042 
1043         /*
1044          * Post an event to the event queue to cause the free packet pool to be
1045          * trimmed if it is oversize.
1046          */
1047         magic = SFXGE_MAGIC_RX_QFPP_TRIM | index;
1048 
1049 #if defined(DEBUG)
1050         /* This is guaranteed due to the start/stop order of rx and ev */
1051         ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
1052         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
1053 #else
1054         /*
1055          * Bug22691 WORKAROUND:
1056          * This handler has been observed in the field to be invoked for a
1057          * queue in the INITIALIZED state, which should never happen.
1058          * Until the mechanism for this is properly understood, add defensive
1059          * checks.
1060          */
1061         if ((sep->se_state != SFXGE_EVQ_STARTED) ||
1062             (srp->sr_state != SFXGE_RXQ_STARTED) ||
1063             (!sep->se_eep)) {
1064                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
1065                     "RXQ[%d] bad state in sfxge_rx_qpoll %d %d %p",
1066                     index, sep->se_state, srp->sr_state, sep->se_eep);
1067                 return;
1068         }
1069 #endif
1070         efx_ev_qpost(sep->se_eep, magic);
1071 
1072         srp->sr_tid = timeout(sfxge_rx_qpoll, srp,
1073             drv_usectohz(sp->s_rxq_poll_usec));
1074 }
1075 
1076 static void
1077 sfxge_rx_qpoll_start(sfxge_rxq_t *srp)
1078 {
1079         sfxge_t *sp = srp->sr_sp;
1080         unsigned int index = srp->sr_index;
1081         sfxge_evq_t *sep = sp->s_sep[index];
1082 
1083         ASSERT(mutex_owned(&(sep->se_lock)));
1084         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
1085 
1086         /* Schedule a poll */
1087         ASSERT3P(srp->sr_tid, ==, 0);
1088         srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 0);
1089 }
1090 
1091 static void
1092 sfxge_rx_qpoll_stop(sfxge_rxq_t *srp)
1093 {
1094         sfxge_t *sp = srp->sr_sp;
1095         unsigned int index = srp->sr_index;
1096         sfxge_evq_t *sep = sp->s_sep[index];
1097         timeout_id_t tid;
1098 
1099         ASSERT(mutex_owned(&(sep->se_lock)));
1100         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
1101 
1102         /*
1103          * Cancel the qpoll timer. Care is needed as this function
1104          * can race with sfxge_rx_qpoll() for timeout id updates.
1105          *
1106          * Do not hold locks used by any timeout(9f) handlers across
1107          * calls to untimeout(9f) as this will deadlock.
1108          */
1109         tid = 0;
1110         while ((srp->sr_tid != 0) && (srp->sr_tid != tid)) {
1111                 tid = srp->sr_tid;
1112                 (void) untimeout(tid);
1113         }
1114         srp->sr_tid = 0;
1115 }
1116 
1117 static int
1118 sfxge_rx_kstat_update(kstat_t *ksp, int rw)
1119 {
1120         sfxge_rxq_t *srp = ksp->ks_private;
1121         sfxge_t *sp = srp->sr_sp;
1122         unsigned int index = srp->sr_index;
1123         sfxge_evq_t *sep = sp->s_sep[index];
1124         kstat_named_t *knp;
1125         int rc;
1126 
1127         if (rw != KSTAT_READ) {
1128                 rc = EACCES;
1129                 goto fail1;
1130         }
1131 
1132         ASSERT(mutex_owned(&(sep->se_lock)));
1133         if (srp->sr_state != SFXGE_RXQ_STARTED)
1134                 goto done;
1135 
1136         knp = ksp->ks_data;
1137         /* NB pointer post-increment below */
1138         knp++->value.ui32 = srp->sr_kstat.srk_rx_pkt_mem_limit;
1139         knp++->value.ui32 = srp->sr_kstat.srk_kcache_alloc_nomem;
1140         knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_nomem;
1141         knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_fail;
1142         knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_nomem;
1143         knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_fail;
1144         knp++->value.ui32 = srp->sr_kstat.srk_desballoc_fail;
1145         knp++->value.ui32 = srp->sr_kstat.srk_rxq_empty_discard;
1146 
1147 done:
1148         return (0);
1149 
1150 fail1:
1151         DTRACE_PROBE1(fail1, int, rc);
1152 
1153         return (rc);
1154 }
1155 
1156 static int
1157 sfxge_rx_kstat_init(sfxge_rxq_t *srp)
1158 {
1159         sfxge_t *sp = srp->sr_sp;
1160         unsigned int index = srp->sr_index;
1161         sfxge_evq_t *sep = sp->s_sep[index];
1162         dev_info_t *dip = sp->s_dip;
1163         char name[MAXNAMELEN];
1164         kstat_t *ksp;
1165         kstat_named_t *knp;
1166         int rc;
1167 
1168         /* Create the set */
1169         (void) snprintf(name, MAXNAMELEN - 1, "%s_rxq%04d",
1170             ddi_driver_name(dip), index);
1171 
1172         if ((ksp = kstat_create((char *)ddi_driver_name(dip),
1173             ddi_get_instance(dip), name, "rxq", KSTAT_TYPE_NAMED,
1174             SFXGE_RX_NSTATS, 0)) == NULL) {
1175                 rc = ENOMEM;
1176                 goto fail1;
1177         }
1178 
1179         srp->sr_ksp = ksp;
1180 
1181         ksp->ks_update = sfxge_rx_kstat_update;
1182         ksp->ks_private = srp;
1183         ksp->ks_lock = &(sep->se_lock);
1184 
1185         /* Initialise the named stats */
1186         knp = ksp->ks_data;
1187         kstat_named_init(knp, "rx_pkt_mem_limit", KSTAT_DATA_UINT32);
1188         knp++;
1189         kstat_named_init(knp, "kcache_alloc_nomem", KSTAT_DATA_UINT32);
1190         knp++;
1191         kstat_named_init(knp, "dma_alloc_nomem", KSTAT_DATA_UINT32);
1192         knp++;
1193         kstat_named_init(knp, "dma_alloc_fail", KSTAT_DATA_UINT32);
1194         knp++;
1195         kstat_named_init(knp, "dma_bind_nomem", KSTAT_DATA_UINT32);
1196         knp++;
1197         kstat_named_init(knp, "dma_bind_fail", KSTAT_DATA_UINT32);
1198         knp++;
1199         kstat_named_init(knp, "desballoc_fail", KSTAT_DATA_UINT32);
1200         knp++;
1201         kstat_named_init(knp, "rxq_empty_discard", KSTAT_DATA_UINT32);
1202 
1203         kstat_install(ksp);
1204         return (0);
1205 
1206 fail1:
1207         DTRACE_PROBE1(fail1, int, rc);
1208 
1209         return (rc);
1210 }
1211 
1212 static int
1213 sfxge_rx_qinit(sfxge_t *sp, unsigned int index)
1214 {
1215         sfxge_rxq_t *srp;
1216         int rc;
1217 
1218         ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
1219 
1220         if ((srp = kmem_cache_alloc(sp->s_rqc, KM_SLEEP)) == NULL) {
1221                 rc = ENOMEM;
1222                 goto fail1;
1223         }
1224         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_UNINITIALIZED);
1225 
1226         srp->sr_index = index;
1227         sp->s_srp[index] = srp;
1228 
1229         if ((rc = sfxge_rx_kstat_init(srp)) != 0)
1230                 goto fail2;
1231 
1232         srp->sr_state = SFXGE_RXQ_INITIALIZED;
1233 
1234         return (0);
1235 
1236 fail2:
1237         DTRACE_PROBE(fail2);
1238         kmem_cache_free(sp->s_rqc, srp);
1239 
1240 fail1:
1241         DTRACE_PROBE1(fail1, int, rc);
1242 
1243         return (rc);
1244 }
1245 
1246 static int
1247 sfxge_rx_qstart(sfxge_t *sp, unsigned int index)
1248 {
1249         sfxge_evq_t *sep = sp->s_sep[index];
1250         sfxge_rxq_t *srp;
1251         efsys_mem_t *esmp;
1252         efx_nic_t *enp;
1253         unsigned int level;
1254         int rc;
1255 
1256         mutex_enter(&(sep->se_lock));
1257         srp = sp->s_srp[index];
1258         enp = sp->s_enp;
1259         esmp = &(srp->sr_mem);
1260 
1261         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
1262         ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
1263 
1264         /* Zero the memory */
1265         bzero(esmp->esm_base, EFX_RXQ_SIZE(sp->s_rxq_size));
1266 
1267         /* Program the buffer table */
1268         if ((rc = sfxge_sram_buf_tbl_set(sp, srp->sr_id, esmp,
1269             EFX_RXQ_NBUFS(sp->s_rxq_size))) != 0)
1270                 goto fail1;
1271 
1272         /* Create the receive queue */
1273         if ((rc = efx_rx_qcreate(enp, index, index, EFX_RXQ_TYPE_DEFAULT,
1274             esmp, sp->s_rxq_size, srp->sr_id, sep->se_eep, &(srp->sr_erp)))
1275             != 0)
1276                 goto fail2;
1277 
1278         /* Enable the receive queue */
1279         efx_rx_qenable(srp->sr_erp);
1280 
1281         /* Set the water marks */
1282         srp->sr_hiwat = EFX_RXQ_LIMIT(sp->s_rxq_size) * 9 / 10;
1283         srp->sr_lowat = srp->sr_hiwat / 2;
1284 
1285         srp->sr_state = SFXGE_RXQ_STARTED;
1286         srp->sr_flush = SFXGE_FLUSH_INACTIVE;
1287 
1288         sfxge_rx_qpoll_start(srp);
1289 
1290         /* Try to fill the queue from the pool */
1291         sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
1292 
1293         /*
1294          * If there were insufficient buffers in the pool to reach the at
1295          * least a batch then allocate some.
1296          */
1297         level = srp->sr_added - srp->sr_completed;
1298         if (level < SFXGE_RX_BATCH)
1299                 sfxge_rx_qfill(srp, SFXGE_RX_BATCH);
1300 
1301         mutex_exit(&(sep->se_lock));
1302 
1303         return (0);
1304 
1305 fail2:
1306         DTRACE_PROBE(fail2);
1307 
1308         /* Clear entries from the buffer table */
1309         sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
1310             EFX_RXQ_NBUFS(sp->s_rxq_size));
1311 
1312 fail1:
1313         DTRACE_PROBE1(fail1, int, rc);
1314 
1315         mutex_exit(&(sep->se_lock));
1316 
1317         return (rc);
1318 }
1319 
1320 static void
1321 sfxge_rx_qflow_complete(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp)
1322 {
1323         mblk_t *mp;
1324         struct ether_header *etherhp;
1325         struct ip *iphp;
1326         struct tcphdr *thp;
1327 
1328         if (srfp->srf_mp == NULL)
1329                 return;
1330 
1331         mp = srfp->srf_mp;
1332         etherhp = srfp->srf_etherhp;
1333         iphp = srfp->srf_iphp;
1334         thp = srfp->srf_last_thp;
1335 
1336         ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
1337             sizeof (struct ether_vlan_header) :
1338             sizeof (struct ether_header)) +
1339             srfp->srf_len, ==, msgdsize(mp));
1340 
1341         ASSERT3U(srfp->srf_len & 0xffff, ==, srfp->srf_len);
1342         iphp->ip_len = htons(srfp->srf_len);
1343 
1344         srfp->srf_first_thp->th_ack = thp->th_ack;
1345         srfp->srf_first_thp->th_win = thp->th_win;
1346         srfp->srf_first_thp->th_flags = thp->th_flags;
1347 
1348         DTRACE_PROBE2(flow_complete, uint32_t, srfp->srf_tag,
1349             size_t, srfp->srf_len);
1350 
1351         srfp->srf_mp = NULL;
1352         srfp->srf_len = 0;
1353 
1354         ASSERT(mp->b_next == NULL);
1355         *(srp->sr_mpp) = mp;
1356         srp->sr_mpp = &(mp->b_next);
1357 }
1358 
1359 static boolean_t
1360 sfxge_rx_qflow_add(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp,
1361     sfxge_rx_packet_t *srpp, clock_t now)
1362 {
1363         sfxge_t *sp = srp->sr_sp;
1364         struct ether_header *etherhp = srpp->srp_etherhp;
1365         struct ip *iphp = srpp->srp_iphp;
1366         struct tcphdr *thp = srpp->srp_thp;
1367         size_t off = srpp->srp_off;
1368         size_t size = (size_t)(srpp->srp_size);
1369         mblk_t *mp = srpp->srp_mp;
1370         uint32_t seq;
1371         unsigned int shift;
1372 
1373         ASSERT3U(MBLKL(mp), ==, off + size);
1374         ASSERT3U(DB_CKSUMFLAGS(mp), ==,
1375             HCK_FULLCKSUM | HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM);
1376 
1377         seq = htonl(thp->th_seq);
1378 
1379         /*
1380          * If the time between this segment and the last is greater than RTO
1381          * then consider this a new flow.
1382          */
1383         if (now - srfp->srf_lbolt > srp->sr_rto) {
1384                 srfp->srf_count = 1;
1385                 srfp->srf_seq = seq + size;
1386 
1387                 goto fail1;
1388         }
1389 
1390         if (seq != srfp->srf_seq) {
1391                 if (srfp->srf_count > SFXGE_SLOW_START)
1392                         srfp->srf_count = SFXGE_SLOW_START;
1393 
1394                 srfp->srf_count >>= 1;
1395 
1396                 srfp->srf_count++;
1397                 srfp->srf_seq = seq + size;
1398 
1399                 goto fail2;
1400         }
1401 
1402         /* Update the in-order segment count and sequence number */
1403         srfp->srf_count++;
1404         srfp->srf_seq = seq + size;
1405 
1406         /* Don't merge across pure ACK, URG, SYN or RST segments */
1407         if (size == 0 || thp->th_flags & (TH_URG | TH_SYN | TH_RST) ||
1408             thp->th_urp != 0)
1409                 goto fail3;
1410 
1411         /*
1412          * If the in-order segment count has not yet reached the slow-start
1413          * threshold then we cannot coalesce.
1414          */
1415         if (srfp->srf_count < SFXGE_SLOW_START)
1416                 goto fail4;
1417 
1418         /* Scale up the packet size from 4k (the maximum being 64k) */
1419         ASSERT3U(srfp->srf_count, >=, SFXGE_SLOW_START);
1420         shift = MIN(srfp->srf_count - SFXGE_SLOW_START + 12, 16);
1421         if (srfp->srf_len + size >= (1 << shift))
1422                 sfxge_rx_qflow_complete(srp, srfp);
1423 
1424         ASSERT(mp->b_cont == NULL);
1425 
1426         if (srfp->srf_mp == NULL) {
1427                 /* First packet in this flow */
1428                 srfp->srf_etherhp = etherhp;
1429                 srfp->srf_iphp = iphp;
1430                 srfp->srf_first_thp = srfp->srf_last_thp = thp;
1431 
1432                 ASSERT3P(mp->b_cont, ==, NULL);
1433                 srfp->srf_mp = mp;
1434                 srfp->srf_mpp = &(mp->b_cont);
1435 
1436                 srfp->srf_len = ntohs(iphp->ip_len);
1437 
1438                 /*
1439                  * If the flow is not already in the list of occupied flows then
1440                  * add it.
1441                  */
1442                 if (srfp->srf_next == NULL &&
1443                     srp->sr_srfpp != &(srfp->srf_next)) {
1444                         *(srp->sr_srfpp) = srfp;
1445                         srp->sr_srfpp = &(srfp->srf_next);
1446                 }
1447         } else {
1448                 /* Later packet in this flow - skip TCP header */
1449                 srfp->srf_last_thp = thp;
1450 
1451                 mp->b_rptr += off;
1452                 ASSERT3U(MBLKL(mp), ==, size);
1453 
1454                 ASSERT3P(mp->b_cont, ==, NULL);
1455                 *(srfp->srf_mpp) = mp;
1456                 srfp->srf_mpp = &(mp->b_cont);
1457 
1458                 srfp->srf_len += size;
1459 
1460                 ASSERT(srfp->srf_next != NULL ||
1461                     srp->sr_srfpp == &(srfp->srf_next));
1462         }
1463 
1464         DTRACE_PROBE2(flow_add, uint32_t, srfp->srf_tag, size_t, size);
1465 
1466         /*
1467          * Try to align coalesced segments on push boundaries, unless they
1468          * are too frequent.
1469          */
1470         if (sp->s_rx_coalesce_mode == SFXGE_RX_COALESCE_ALLOW_PUSH &&
1471             thp->th_flags & TH_PUSH)
1472                 sfxge_rx_qflow_complete(srp, srfp);
1473 
1474         srfp->srf_lbolt = now;
1475         return (B_TRUE);
1476 
1477 fail4:
1478 fail3:
1479 fail2:
1480 fail1:
1481         sfxge_rx_qflow_complete(srp, srfp);
1482 
1483         srfp->srf_lbolt = now;
1484         return (B_FALSE);
1485 }
1486 
1487 void
1488 sfxge_rx_qpacket_coalesce(sfxge_rxq_t *srp)
1489 {
1490         sfxge_t *sp = srp->sr_sp;
1491         clock_t now;
1492         mblk_t *mp;
1493         sfxge_rx_flow_t *srfp;
1494 
1495         ASSERT(sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF);
1496 
1497         now = ddi_get_lbolt();
1498 
1499         mp = srp->sr_mp;
1500 
1501         srp->sr_mp = NULL;
1502         srp->sr_mpp = &(srp->sr_mp);
1503 
1504         /* Start with the last flow to be appended to */
1505         srfp = *(srp->sr_srfpp);
1506 
1507         while (mp != NULL) {
1508                 frtn_t *freep;
1509                 sfxge_rx_packet_t *srpp;
1510                 struct ether_header *etherhp;
1511                 struct ip *iphp;
1512                 struct tcphdr *thp;
1513                 size_t off;
1514                 size_t size;
1515                 uint16_t ether_tci;
1516                 uint32_t hash;
1517                 uint32_t tag;
1518                 mblk_t *next;
1519                 sfxge_packet_type_t pkt_type;
1520                 uint16_t sport, dport;
1521 
1522                 next = mp->b_next;
1523                 mp->b_next = NULL;
1524 
1525                 if (next != NULL)
1526                         prefetch_read_many(next);
1527 
1528                 freep = DB_FRTNP(mp);
1529                 /*LINTED*/
1530                 srpp = (sfxge_rx_packet_t *)(freep->free_arg);
1531                 ASSERT3P(srpp->srp_mp, ==, mp);
1532 
1533                 /* If the packet is not TCP then we cannot coalesce it */
1534                 if (~(srpp->srp_flags) & EFX_PKT_TCP)
1535                         goto reject;
1536 
1537                 /*
1538                  * If the packet is not fully checksummed then we cannot
1539                  * coalesce it.
1540                  */
1541                 if (~(srpp->srp_flags) & (EFX_CKSUM_TCPUDP | EFX_CKSUM_IPV4))
1542                         goto reject;
1543 
1544                 /* Parse the TCP header */
1545                 pkt_type = sfxge_pkthdr_parse(mp, &etherhp, &iphp, &thp, &off,
1546                     &size, &sport, &dport);
1547                 ASSERT(pkt_type == SFXGE_PACKET_TYPE_IPV4_TCP);
1548                 ASSERT(etherhp != NULL);
1549                 ASSERT(iphp != NULL);
1550                 ASSERT(thp != NULL);
1551                 ASSERT(off != 0);
1552 
1553                 if ((iphp->ip_off & ~htons(IP_DF)) != 0)
1554                         goto reject;
1555 
1556                 if (etherhp->ether_type == htons(ETHERTYPE_VLAN)) {
1557                         struct ether_vlan_header *ethervhp;
1558 
1559                         ethervhp = (struct ether_vlan_header *)etherhp;
1560                         ether_tci = ethervhp->ether_tci;
1561                 } else {
1562                         ether_tci = 0;
1563                 }
1564 
1565                 /*
1566                  * Make sure any minimum length padding is stripped
1567                  * before we try to add the packet to a flow.
1568                  */
1569                 ASSERT3U(sp->s_rx_prefix_size + MBLKL(mp), ==,
1570                     (size_t)(srpp->srp_size));
1571                 ASSERT3U(sp->s_rx_prefix_size + off + size, <=,
1572                     (size_t)(srpp->srp_size));
1573 
1574                 if (sp->s_rx_prefix_size + off + size <
1575                     (size_t)(srpp->srp_size))
1576                         mp->b_wptr = mp->b_rptr + off + size;
1577 
1578                 /*
1579                  * If there is no current flow, or the segment does not match
1580                  * the current flow then we must attempt to look up the
1581                  * correct flow in the table.
1582                  */
1583                 if (srfp == NULL)
1584                         goto lookup;
1585 
1586                 if (srfp->srf_saddr != iphp->ip_src.s_addr ||
1587                     srfp->srf_daddr != iphp->ip_dst.s_addr)
1588                         goto lookup;
1589 
1590                 if (srfp->srf_sport != thp->th_sport ||
1591                     srfp->srf_dport != thp->th_dport)
1592                         goto lookup;
1593 
1594                 if (srfp->srf_tci != ether_tci)
1595                         goto lookup;
1596 
1597 add:
1598                 ASSERT(srfp != NULL);
1599 
1600                 srpp->srp_etherhp = etherhp;
1601                 srpp->srp_iphp = iphp;
1602                 srpp->srp_thp = thp;
1603                 srpp->srp_off = off;
1604 
1605                 ASSERT3U(size, <, (1 << 16));
1606                 srpp->srp_size = (uint16_t)size;
1607 
1608                 /* Try to append the packet to the flow */
1609                 if (!sfxge_rx_qflow_add(srp, srfp, srpp, now))
1610                         goto reject;
1611 
1612                 mp = next;
1613                 continue;
1614 
1615 lookup:
1616                 /*
1617                  * If there is a prefix area then read the hash from that,
1618                  * otherwise calculate it.
1619                  */
1620                 if (sp->s_rx_prefix_size != 0) {
1621                         hash = efx_psuedo_hdr_hash_get(sp->s_enp,
1622                             EFX_RX_HASHALG_TOEPLITZ,
1623                             DB_BASE(mp));
1624                 } else {
1625                         SFXGE_TCP_HASH(sp,
1626                             &iphp->ip_src.s_addr,
1627                             thp->th_sport,
1628                             &iphp->ip_dst.s_addr,
1629                             thp->th_dport,
1630                             hash);
1631                 }
1632 
1633                 srfp = &(srp->sr_flow[(hash >> 6) % SFXGE_MAX_FLOW]);
1634                 tag = hash + 1; /* Make sure it's not zero */
1635 
1636                 /*
1637                  * If the flow we have found does not match the hash then
1638                  * it may be an unused flow, or it may be stale.
1639                  */
1640                 if (tag != srfp->srf_tag) {
1641                         if (srfp->srf_count != 0) {
1642                                 if (now - srfp->srf_lbolt <= srp->sr_rto)
1643                                         goto reject;
1644                         }
1645 
1646                         if (srfp->srf_mp != NULL)
1647                                 goto reject;
1648 
1649                         /* Start a new flow */
1650                         ASSERT(srfp->srf_next == NULL);
1651 
1652                         srfp->srf_tag = tag;
1653 
1654                         srfp->srf_saddr = iphp->ip_src.s_addr;
1655                         srfp->srf_daddr = iphp->ip_dst.s_addr;
1656                         srfp->srf_sport = thp->th_sport;
1657                         srfp->srf_dport = thp->th_dport;
1658                         srfp->srf_tci = ether_tci;
1659 
1660                         srfp->srf_count = 0;
1661                         srfp->srf_seq = ntohl(thp->th_seq);
1662 
1663                         srfp->srf_lbolt = now;
1664                         goto add;
1665                 }
1666 
1667                 /*
1668                  * If the flow we have found does match the hash then it could
1669                  * still be an alias.
1670                  */
1671                 if (srfp->srf_saddr != iphp->ip_src.s_addr ||
1672                     srfp->srf_daddr != iphp->ip_dst.s_addr)
1673                         goto reject;
1674 
1675                 if (srfp->srf_sport != thp->th_sport ||
1676                     srfp->srf_dport != thp->th_dport)
1677                         goto reject;
1678 
1679                 if (srfp->srf_tci != ether_tci)
1680                         goto reject;
1681 
1682                 goto add;
1683 
1684 reject:
1685                 *(srp->sr_mpp) = mp;
1686                 srp->sr_mpp = &(mp->b_next);
1687 
1688                 mp = next;
1689         }
1690 }
1691 
1692 void
1693 sfxge_rx_qcomplete(sfxge_rxq_t *srp, boolean_t eop)
1694 {
1695         sfxge_t *sp = srp->sr_sp;
1696         unsigned int index = srp->sr_index;
1697         sfxge_evq_t *sep = sp->s_sep[index];
1698         unsigned int completed;
1699         sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
1700         unsigned int level;
1701 
1702         ASSERT(mutex_owned(&(sep->se_lock)));
1703 
1704         ASSERT(srp->sr_mp == NULL);
1705         ASSERT(srp->sr_mpp == &(srp->sr_mp));
1706 
1707         completed = srp->sr_completed;
1708         while (completed != srp->sr_pending) {
1709                 unsigned int id;
1710                 sfxge_rx_packet_t *srpp;
1711                 mblk_t *mp;
1712                 size_t size;
1713                 uint16_t flags;
1714                 int rc;
1715 
1716                 id = completed++ & (sp->s_rxq_size - 1);
1717 
1718                 if (srp->sr_pending - completed >= 4) {
1719                         unsigned int prefetch;
1720 
1721                         prefetch = (id + 4) & (sp->s_rxq_size - 1);
1722 
1723                         srpp = srp->sr_srpp[prefetch];
1724                         ASSERT(srpp != NULL);
1725 
1726                         mp = srpp->srp_mp;
1727                         prefetch_read_many(mp->b_datap);
1728                 } else if (completed == srp->sr_pending) {
1729                         prefetch_read_many(srp->sr_mp);
1730                 }
1731 
1732                 srpp = srp->sr_srpp[id];
1733                 ASSERT(srpp != NULL);
1734 
1735                 srp->sr_srpp[id] = NULL;
1736 
1737                 mp = srpp->srp_mp;
1738                 ASSERT(mp->b_cont == NULL);
1739 
1740                 /* when called from sfxge_rx_qstop() */
1741                 if (srp->sr_state != SFXGE_RXQ_STARTED)
1742                         goto discard;
1743 
1744                 if (srpp->srp_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
1745                         goto discard;
1746 
1747                 /* Make the data visible to the kernel */
1748                 rc = ddi_dma_sync(srpp->srp_dma_handle, 0,
1749                     sp->s_rx_buffer_size, DDI_DMA_SYNC_FORKERNEL);
1750                 ASSERT3P(rc, ==, DDI_SUCCESS);
1751 
1752                 /* Read the length from the psuedo header if required */
1753                 if (srpp->srp_flags & EFX_PKT_PREFIX_LEN) {
1754                         rc = efx_psuedo_hdr_pkt_length_get(sp->s_enp,
1755                             mp->b_rptr,
1756                             &srpp->srp_size);
1757                         ASSERT3P(rc, ==, 0);
1758                         srpp->srp_size += sp->s_rx_prefix_size;
1759                 }
1760 
1761                 /* Set up the packet length */
1762                 ASSERT3P(mp->b_rptr, ==, DB_BASE(mp));
1763                 mp->b_rptr += sp->s_rx_prefix_size;
1764 
1765                 prefetch_read_many(mp->b_rptr);
1766 
1767                 ASSERT3P(mp->b_wptr, ==, DB_BASE(mp));
1768                 mp->b_wptr += (size_t)(srpp->srp_size);
1769                 ASSERT3P(mp->b_wptr, <=, DB_LIM(mp));
1770 
1771                 /* Calculate the maximum packet size */
1772                 size = sp->s_mtu;
1773                 size += (srpp->srp_flags & EFX_PKT_VLAN_TAGGED) ?
1774                     sizeof (struct ether_vlan_header) :
1775                     sizeof (struct ether_header);
1776 
1777                 if (MBLKL(mp) > size)
1778                         goto discard;
1779 
1780                 /* Check for loopback packets */
1781                 if (!(srpp->srp_flags & EFX_PKT_IPV4) &&
1782                     !(srpp->srp_flags & EFX_PKT_IPV6)) {
1783                         struct ether_header *etherhp;
1784 
1785                         /*LINTED*/
1786                         etherhp = (struct ether_header *)(mp->b_rptr);
1787 
1788                         if (etherhp->ether_type ==
1789                             htons(SFXGE_ETHERTYPE_LOOPBACK)) {
1790                                 DTRACE_PROBE(loopback);
1791 
1792                                 srp->sr_loopback++;
1793                                 goto discard;
1794                         }
1795                 }
1796 
1797                 /* Set up the checksum information */
1798                 flags = 0;
1799 
1800                 if (srpp->srp_flags & EFX_CKSUM_IPV4) {
1801                         ASSERT(srpp->srp_flags & EFX_PKT_IPV4);
1802                         flags |= HCK_IPV4_HDRCKSUM;
1803                 }
1804 
1805                 if (srpp->srp_flags & EFX_CKSUM_TCPUDP) {
1806                         ASSERT(srpp->srp_flags & EFX_PKT_TCP ||
1807                             srpp->srp_flags & EFX_PKT_UDP);
1808                         flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
1809                 }
1810 
1811                 DB_CKSUMSTART(mp) = 0;
1812                 DB_CKSUMSTUFF(mp) = 0;
1813                 DB_CKSUMEND(mp) = 0;
1814                 DB_CKSUMFLAGS(mp) = flags;
1815                 DB_CKSUM16(mp) = 0;
1816 
1817                 /* Add the packet to the tail of the chain */
1818                 srfppp->srfpp_loaned++;
1819 
1820                 ASSERT(mp->b_next == NULL);
1821                 *(srp->sr_mpp) = mp;
1822                 srp->sr_mpp = &(mp->b_next);
1823 
1824                 continue;
1825 
1826 discard:
1827                 /* Return the packet to the pool */
1828                 srfppp->srfpp_loaned++;
1829                 freeb(mp); /* Equivalent to freemsg() as b_cont==0 */
1830         }
1831         srp->sr_completed = completed;
1832 
1833         /* Attempt to coalesce any TCP packets */
1834         if (sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF)
1835                 sfxge_rx_qpacket_coalesce(srp);
1836 
1837         /*
1838          * If there are any pending flows and this is the end of the
1839          * poll then they must be completed.
1840          */
1841         if (srp->sr_srfp != NULL && eop) {
1842                 sfxge_rx_flow_t *srfp;
1843 
1844                 srfp = srp->sr_srfp;
1845 
1846                 srp->sr_srfp = NULL;
1847                 srp->sr_srfpp = &(srp->sr_srfp);
1848 
1849                 do {
1850                         sfxge_rx_flow_t *next;
1851 
1852                         next = srfp->srf_next;
1853                         srfp->srf_next = NULL;
1854 
1855                         sfxge_rx_qflow_complete(srp, srfp);
1856 
1857                         srfp = next;
1858                 } while (srfp != NULL);
1859         }
1860 
1861         level = srp->sr_pushed - srp->sr_completed;
1862 
1863         /* If there are any packets then pass them up the stack */
1864         if (srp->sr_mp != NULL) {
1865                 mblk_t *mp;
1866 
1867                 mp = srp->sr_mp;
1868 
1869                 srp->sr_mp = NULL;
1870                 srp->sr_mpp = &(srp->sr_mp);
1871 
1872                 if (level == 0) {
1873                         /* Try to refill ASAP */
1874                         sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
1875                         level = srp->sr_pushed - srp->sr_completed;
1876                 }
1877 
1878                 /*
1879                  * If the RXQ is still empty, discard and recycle the
1880                  * current entry to ensure that the ring always
1881                  * contains at least one descriptor. This ensures that
1882                  * the next hardware RX will trigger an event
1883                  * (possibly delayed by interrupt moderation) and
1884                  * trigger another refill/fill attempt.
1885                  *
1886                  * Note this drops a complete LRO fragment from the
1887                  * start of the batch.
1888                  *
1889                  * Note also that copymsgchain() does not help with
1890                  * resource starvation here, unless we are short of DMA
1891                  * mappings.
1892                  */
1893                 if (level == 0) {
1894                         mblk_t *nmp;
1895 
1896                         srp->sr_kstat.srk_rxq_empty_discard++;
1897                         DTRACE_PROBE1(rxq_empty_discard, int, index);
1898                         nmp = mp->b_next;
1899                         if (nmp)
1900                                 sfxge_gld_rx_post(sp, index, nmp);
1901                         /* as level==0 will swizzle,rxpost below */
1902                         freemsg(mp);
1903                 } else {
1904                         sfxge_gld_rx_post(sp, index, mp);
1905                 }
1906         }
1907 
1908         /* Top up the queue if necessary */
1909         if (level < srp->sr_hiwat) {
1910                 sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
1911 
1912                 level = srp->sr_added - srp->sr_completed;
1913                 if (level < srp->sr_lowat)
1914                         sfxge_rx_qfill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
1915         }
1916 }
1917 
1918 void
1919 sfxge_rx_qflush_done(sfxge_rxq_t *srp)
1920 {
1921         sfxge_t *sp = srp->sr_sp;
1922         unsigned int index = srp->sr_index;
1923         sfxge_evq_t *sep = sp->s_sep[index];
1924         boolean_t flush_pending;
1925 
1926         ASSERT(mutex_owned(&(sep->se_lock)));
1927 
1928         /*
1929          * Flush successful: wakeup sfxge_rx_qstop() if flush is pending.
1930          *
1931          * A delayed flush event received after RxQ stop has timed out
1932          * will be ignored, as then the flush state will not be PENDING
1933          * (see SFCbug22989).
1934          */
1935         flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
1936         srp->sr_flush = SFXGE_FLUSH_DONE;
1937         if (flush_pending)
1938                 cv_broadcast(&(srp->sr_flush_kv));
1939 }
1940 
1941 void
1942 sfxge_rx_qflush_failed(sfxge_rxq_t *srp)
1943 {
1944         sfxge_t *sp = srp->sr_sp;
1945         unsigned int index = srp->sr_index;
1946         sfxge_evq_t *sep = sp->s_sep[index];
1947         boolean_t flush_pending;
1948 
1949         ASSERT(mutex_owned(&(sep->se_lock)));
1950 
1951         /*
1952          * Flush failed: wakeup sfxge_rx_qstop() if flush is pending.
1953          *
1954          * A delayed flush event received after RxQ stop has timed out
1955          * will be ignored, as then the flush state will not be PENDING
1956          * (see SFCbug22989).
1957          */
1958         flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
1959         srp->sr_flush = SFXGE_FLUSH_FAILED;
1960         if (flush_pending)
1961                 cv_broadcast(&(srp->sr_flush_kv));
1962 }
1963 
1964 static void
1965 sfxge_rx_qstop(sfxge_t *sp, unsigned int index)
1966 {
1967         dev_info_t *dip = sp->s_dip;
1968         sfxge_evq_t *sep = sp->s_sep[index];
1969         sfxge_rxq_t *srp;
1970         clock_t timeout;
1971         unsigned int flush_tries = SFXGE_RX_QFLUSH_TRIES;
1972         int rc;
1973 
1974         ASSERT(mutex_owned(&(sp->s_state_lock)));
1975 
1976         mutex_enter(&(sep->se_lock));
1977 
1978         srp = sp->s_srp[index];
1979         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
1980 
1981         sfxge_rx_qpoll_stop(srp);
1982 
1983         /* Further packets are discarded by sfxge_rx_qcomplete() */
1984         srp->sr_state = SFXGE_RXQ_INITIALIZED;
1985 
1986         if (sp->s_hw_err != SFXGE_HW_OK) {
1987                 /*
1988                  * Flag indicates possible hardware failure.
1989                  * Attempt flush but do not wait for it to complete.
1990                  */
1991                 srp->sr_flush = SFXGE_FLUSH_DONE;
1992                 (void) efx_rx_qflush(srp->sr_erp);
1993         }
1994 
1995         /* Wait upto 2sec for queue flushing to complete */
1996         timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_RX_QFLUSH_USEC);
1997 
1998         while (srp->sr_flush != SFXGE_FLUSH_DONE && flush_tries-- > 0) {
1999                 if ((rc = efx_rx_qflush(srp->sr_erp)) != 0) {
2000                         if (rc == EALREADY)
2001                                 srp->sr_flush = SFXGE_FLUSH_DONE;
2002                         else
2003                                 srp->sr_flush = SFXGE_FLUSH_FAILED;
2004                         break;
2005                 }
2006                 srp->sr_flush = SFXGE_FLUSH_PENDING;
2007                 if (cv_timedwait(&(srp->sr_flush_kv), &(sep->se_lock),
2008                     timeout) < 0) {
2009                         /* Timeout waiting for successful or failed flush */
2010                         dev_err(dip, CE_NOTE,
2011                             SFXGE_CMN_ERR "rxq[%d] flush timeout", index);
2012                         break;
2013                 }
2014         }
2015 
2016         if (srp->sr_flush == SFXGE_FLUSH_FAILED)
2017                 dev_err(dip, CE_NOTE,
2018                     SFXGE_CMN_ERR "rxq[%d] flush failed", index);
2019 
2020         DTRACE_PROBE1(flush, sfxge_flush_state_t, srp->sr_flush);
2021         srp->sr_flush = SFXGE_FLUSH_DONE;
2022 
2023         /* Destroy the receive queue */
2024         efx_rx_qdestroy(srp->sr_erp);
2025         srp->sr_erp = NULL;
2026 
2027         /* Clear entries from the buffer table */
2028         sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
2029             EFX_RXQ_NBUFS(sp->s_rxq_size));
2030 
2031         /*
2032          * Free any unused RX packets which had descriptors on the RXQ
2033          * Packets will be discard as state != STARTED
2034          */
2035         srp->sr_pending = srp->sr_added;
2036         sfxge_rx_qcomplete(srp, B_TRUE);
2037 
2038         ASSERT3U(srp->sr_completed, ==, srp->sr_pending);
2039 
2040         srp->sr_added = 0;
2041         srp->sr_pushed = 0;
2042         srp->sr_pending = 0;
2043         srp->sr_completed = 0;
2044         srp->sr_loopback = 0;
2045 
2046         srp->sr_lowat = 0;
2047         srp->sr_hiwat = 0;
2048 
2049         mutex_exit(&(sep->se_lock));
2050 }
2051 
2052 static void
2053 sfxge_rx_kstat_fini(sfxge_rxq_t *srp)
2054 {
2055         kstat_delete(srp->sr_ksp);
2056         srp->sr_ksp = NULL;
2057 }
2058 
2059 static void
2060 sfxge_rx_qfini(sfxge_t *sp, unsigned int index)
2061 {
2062         sfxge_rxq_t *srp = sp->s_srp[index];
2063 
2064         ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
2065 
2066         sp->s_srp[index] = NULL;
2067         srp->sr_state = SFXGE_RXQ_UNINITIALIZED;
2068 
2069         sfxge_rx_kstat_fini(srp);
2070 
2071         /* Empty the pool */
2072         sfxge_rx_qfpp_empty(srp);
2073 
2074         srp->sr_index = 0;
2075 
2076         kmem_cache_free(sp->s_rqc, srp);
2077 }
2078 
2079 static int
2080 sfxge_rx_scale_kstat_update(kstat_t *ksp, int rw)
2081 {
2082         sfxge_t *sp = ksp->ks_private;
2083         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2084         sfxge_intr_t *sip = &(sp->s_intr);
2085         kstat_named_t *knp;
2086         unsigned int index;
2087         unsigned int entry;
2088         unsigned int *freq;
2089         int rc;
2090 
2091         ASSERT(mutex_owned(&(srsp->srs_lock)));
2092 
2093         if (rw != KSTAT_READ) {
2094                 rc = EACCES;
2095                 goto fail1;
2096         }
2097 
2098         if ((freq = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
2099             KM_NOSLEEP)) == NULL) {
2100                 rc = ENOMEM;
2101                 goto fail2;
2102         }
2103 
2104         for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
2105                 index = srsp->srs_tbl[entry];
2106 
2107                 freq[index]++;
2108         }
2109 
2110         knp = ksp->ks_data;
2111         for (index = 0; index < sip->si_nalloc; index++) {
2112                 knp->value.ui64 = freq[index];
2113                 knp++;
2114         }
2115 
2116         knp->value.ui64 = srsp->srs_count;
2117 
2118         kmem_free(freq, sizeof (unsigned int) * sip->si_nalloc);
2119 
2120         return (0);
2121 
2122 fail2:
2123         DTRACE_PROBE(fail2);
2124 fail1:
2125         DTRACE_PROBE1(fail1, int, rc);
2126         return (rc);
2127 }
2128 
2129 static int
2130 sfxge_rx_scale_kstat_init(sfxge_t *sp)
2131 {
2132         dev_info_t *dip = sp->s_dip;
2133         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2134         sfxge_intr_t *sip = &(sp->s_intr);
2135         char name[MAXNAMELEN];
2136         kstat_t *ksp;
2137         kstat_named_t *knp;
2138         unsigned int index;
2139         int rc;
2140 
2141         /* Create the set */
2142         (void) snprintf(name, MAXNAMELEN - 1, "%s_rss", ddi_driver_name(dip));
2143 
2144         if ((ksp = kstat_create((char *)ddi_driver_name(dip),
2145             ddi_get_instance(dip), name, "rss", KSTAT_TYPE_NAMED,
2146             sip->si_nalloc + 1, 0)) == NULL) {
2147                 rc = ENOMEM;
2148                 goto fail1;
2149         }
2150 
2151         srsp->srs_ksp = ksp;
2152 
2153         ksp->ks_update = sfxge_rx_scale_kstat_update;
2154         ksp->ks_private = sp;
2155         ksp->ks_lock = &(srsp->srs_lock);
2156 
2157         /* Initialise the named stats */
2158         knp = ksp->ks_data;
2159         for (index = 0; index < sip->si_nalloc; index++) {
2160                 char name[MAXNAMELEN];
2161 
2162                 (void) snprintf(name, MAXNAMELEN - 1, "evq%04d_count", index);
2163                 kstat_named_init(knp, name, KSTAT_DATA_UINT64);
2164                 knp++;
2165         }
2166 
2167         kstat_named_init(knp, "scale", KSTAT_DATA_UINT64);
2168 
2169         kstat_install(ksp);
2170         return (0);
2171 
2172 fail1:
2173         DTRACE_PROBE1(fail1, int, rc);
2174 
2175         return (rc);
2176 }
2177 
2178 static void
2179 sfxge_rx_scale_kstat_fini(sfxge_t *sp)
2180 {
2181         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2182 
2183         /* Destroy the set */
2184         kstat_delete(srsp->srs_ksp);
2185         srsp->srs_ksp = NULL;
2186 }
2187 
2188 
2189 unsigned int
2190 sfxge_rx_scale_prop_get(sfxge_t *sp)
2191 {
2192         int rx_scale;
2193 
2194         rx_scale = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
2195             DDI_PROP_DONTPASS, "rx_scale_count", SFXGE_RX_SCALE_MAX);
2196         /* 0 and all -ve numbers sets to number of logical CPUs */
2197         if (rx_scale <= 0)
2198                 rx_scale = ncpus;
2199 
2200         return (rx_scale);
2201 }
2202 
2203 
2204 static int
2205 sfxge_rx_scale_init(sfxge_t *sp)
2206 {
2207         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2208         sfxge_intr_t *sip = &(sp->s_intr);
2209         int rc;
2210 
2211         ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_UNINITIALIZED);
2212 
2213         /* Create tables for CPU, core, cache and chip counts */
2214         srsp->srs_cpu = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
2215 
2216         mutex_init(&(srsp->srs_lock), NULL, MUTEX_DRIVER, NULL);
2217 
2218         /* We need at least one event queue */
2219         srsp->srs_count = sfxge_rx_scale_prop_get(sp);
2220         if (srsp->srs_count > sip->si_nalloc)
2221                 srsp->srs_count = sip->si_nalloc;
2222         if (srsp->srs_count < 1)
2223                 srsp->srs_count = 1;
2224 
2225         /* Set up the kstats */
2226         if ((rc = sfxge_rx_scale_kstat_init(sp)) != 0)
2227                 goto fail1;
2228 
2229         srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
2230 
2231         return (0);
2232 
2233 fail1:
2234         DTRACE_PROBE1(fail1, int, rc);
2235         mutex_destroy(&(srsp->srs_lock));
2236 
2237         return (rc);
2238 }
2239 
2240 void
2241 sfxge_rx_scale_update(void *arg)
2242 {
2243         sfxge_t *sp = arg;
2244         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2245         sfxge_intr_t *sip;
2246         processorid_t id;
2247         unsigned int count;
2248         unsigned int *tbl;
2249         unsigned int *rating;
2250         unsigned int entry;
2251         int rc;
2252 
2253         mutex_enter(&(srsp->srs_lock));
2254 
2255         if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
2256                 rc = EFAULT;
2257                 goto fail1;
2258         }
2259 
2260         if ((tbl =  kmem_zalloc(sizeof (unsigned int) * SFXGE_RX_SCALE_MAX,
2261             KM_NOSLEEP)) == NULL) {
2262                 rc = ENOMEM;
2263                 goto fail2;
2264         }
2265 
2266         sip = &(sp->s_intr);
2267         if ((rating = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
2268             KM_NOSLEEP)) == NULL) {
2269                 rc = ENOMEM;
2270                 goto fail3;
2271         }
2272 
2273         mutex_enter(&cpu_lock);
2274 
2275         /*
2276          * Substract any current CPU, core, cache and chip usage from the
2277          * global contention tables.
2278          */
2279         for (id = 0; id < NCPU; id++) {
2280                 ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
2281                 sfxge_cpu[id] -= srsp->srs_cpu[id];
2282                 srsp->srs_cpu[id] = 0;
2283         }
2284 
2285         ASSERT(srsp->srs_count != 0);
2286 
2287         /* Choose as many event queues as we need */
2288         for (count = 0; count < srsp->srs_count; count++) {
2289                 unsigned int index;
2290                 sfxge_evq_t *sep;
2291                 unsigned int choice;
2292                 unsigned int choice_rating;
2293 
2294                 bzero(rating, sizeof (unsigned int) * sip->si_nalloc);
2295 
2296                 /*
2297                  * Rate each event queue on its global level of CPU
2298                  * contention.
2299                  */
2300                 for (index = 0; index < sip->si_nalloc; index++) {
2301                         sep = sp->s_sep[index];
2302 
2303                         id = sep->se_cpu_id;
2304                         rating[index] += sfxge_cpu[id];
2305                 }
2306 
2307                 /* Choose the queue with the lowest CPU contention */
2308                 choice = 0;
2309                 choice_rating = rating[0];
2310 
2311                 for (index = 1; index < sip->si_nalloc; index++) {
2312                         if (rating[index] < choice_rating) {
2313                                 choice = index;
2314                                 choice_rating = rating[index];
2315                         }
2316                 }
2317 
2318                 /* Add our choice to the condensed RSS table */
2319                 tbl[count] = choice;
2320 
2321                 /* Add information to the global contention tables */
2322                 sep = sp->s_sep[choice];
2323 
2324                 id = sep->se_cpu_id;
2325                 srsp->srs_cpu[id]++;
2326                 sfxge_cpu[id]++;
2327         }
2328 
2329         mutex_exit(&cpu_lock);
2330 
2331         /* Build the expanded RSS table */
2332         count = 0;
2333         for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
2334                 unsigned int index;
2335 
2336                 index = tbl[count];
2337                 count = (count + 1) % srsp->srs_count;
2338 
2339                 srsp->srs_tbl[entry] = index;
2340         }
2341 
2342         /* Program the expanded RSS table into the hardware */
2343         (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
2344             SFXGE_RX_SCALE_MAX);
2345 
2346         mutex_exit(&(srsp->srs_lock));
2347         kmem_free(rating, sizeof (unsigned int) * sip->si_nalloc);
2348         kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
2349         return;
2350 
2351 fail3:
2352         DTRACE_PROBE(fail3);
2353         kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
2354 fail2:
2355         DTRACE_PROBE(fail2);
2356 fail1:
2357         DTRACE_PROBE1(fail1, int, rc);
2358 
2359         mutex_exit(&(srsp->srs_lock));
2360 }
2361 
2362 static int
2363 sfxge_rx_scale_start(sfxge_t *sp)
2364 {
2365         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2366         int rc;
2367 
2368         mutex_enter(&(srsp->srs_lock));
2369 
2370         ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
2371 
2372         /* Clear down the RSS table */
2373         bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
2374 
2375         (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
2376             SFXGE_RX_SCALE_MAX);
2377 
2378         if ((rc = sfxge_toeplitz_hash_init(sp)) != 0)
2379                 goto fail1;
2380 
2381         srsp->srs_state = SFXGE_RX_SCALE_STARTED;
2382 
2383         mutex_exit(&(srsp->srs_lock));
2384 
2385         /* sfxge_t->s_state_lock held */
2386         (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
2387             DDI_SLEEP);
2388 
2389         return (0);
2390 
2391 fail1:
2392         DTRACE_PROBE1(fail1, int, rc);
2393 
2394         mutex_exit(&(srsp->srs_lock));
2395 
2396         return (rc);
2397 }
2398 
2399 int
2400 sfxge_rx_scale_count_get(sfxge_t *sp, unsigned int *countp)
2401 {
2402         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2403         int rc;
2404 
2405         mutex_enter(&(srsp->srs_lock));
2406 
2407         if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
2408             srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
2409                 rc = ENOTSUP;
2410                 goto fail1;
2411         }
2412 
2413         *countp = srsp->srs_count;
2414 
2415         mutex_exit(&(srsp->srs_lock));
2416 
2417         return (0);
2418 
2419 fail1:
2420         DTRACE_PROBE1(fail1, int, rc);
2421 
2422         mutex_exit(&(srsp->srs_lock));
2423 
2424         return (rc);
2425 }
2426 
2427 int
2428 sfxge_rx_scale_count_set(sfxge_t *sp, unsigned int count)
2429 {
2430         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2431         sfxge_intr_t *sip = &(sp->s_intr);
2432         int dispatch = 1;
2433         int rc;
2434 
2435         if (count < 1 || count > sip->si_nalloc) {
2436                 rc = EINVAL;
2437                 goto fail1;
2438         }
2439 
2440         mutex_enter(&(srsp->srs_lock));
2441 
2442         if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
2443             srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
2444                 rc = ENOTSUP;
2445                 goto fail2;
2446         }
2447 
2448         srsp->srs_count = count;
2449 
2450         if (srsp->srs_state != SFXGE_RX_SCALE_STARTED)
2451                 dispatch = 0;
2452 
2453         mutex_exit(&(srsp->srs_lock));
2454 
2455         if (dispatch)
2456                 /* no locks held */
2457                 (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
2458                     DDI_SLEEP);
2459 
2460         return (0);
2461 
2462 fail2:
2463         DTRACE_PROBE(fail2);
2464 
2465         mutex_exit(&(srsp->srs_lock));
2466 
2467 fail1:
2468         DTRACE_PROBE1(fail1, int, rc);
2469 
2470         return (rc);
2471 }
2472 
2473 static void
2474 sfxge_rx_scale_stop(sfxge_t *sp)
2475 {
2476         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2477         processorid_t id;
2478 
2479         mutex_enter(&(srsp->srs_lock));
2480 
2481         ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_STARTED);
2482 
2483         srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
2484 
2485         mutex_enter(&cpu_lock);
2486 
2487         /*
2488          * Substract any current CPU, core, cache and chip usage from the
2489          * global contention tables.
2490          */
2491         for (id = 0; id < NCPU; id++) {
2492                 ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
2493                 sfxge_cpu[id] -= srsp->srs_cpu[id];
2494                 srsp->srs_cpu[id] = 0;
2495         }
2496 
2497         mutex_exit(&cpu_lock);
2498 
2499         /* Clear down the RSS table */
2500         bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
2501 
2502         (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
2503             SFXGE_RX_SCALE_MAX);
2504 
2505         mutex_exit(&(srsp->srs_lock));
2506 }
2507 
2508 static void
2509 sfxge_rx_scale_fini(sfxge_t *sp)
2510 {
2511         sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
2512 
2513         ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
2514 
2515         srsp->srs_state = SFXGE_RX_SCALE_UNINITIALIZED;
2516 
2517         /* Tear down the kstats */
2518         sfxge_rx_scale_kstat_fini(sp);
2519 
2520         srsp->srs_count = 0;
2521 
2522         mutex_destroy(&(srsp->srs_lock));
2523 
2524         /* Destroy tables */
2525         kmem_free(srsp->srs_cpu, sizeof (unsigned int) * NCPU);
2526         srsp->srs_cpu = NULL;
2527 
2528         sfxge_toeplitz_hash_fini(sp);
2529 }
2530 
2531 int
2532 sfxge_rx_init(sfxge_t *sp)
2533 {
2534         sfxge_intr_t *sip = &(sp->s_intr);
2535         char name[MAXNAMELEN];
2536         int index;
2537         int rc;
2538 
2539         if (sip->si_state == SFXGE_INTR_UNINITIALIZED) {
2540                 rc = EINVAL;
2541                 goto fail1;
2542         }
2543 
2544         if ((rc = sfxge_rx_scale_init(sp)) != 0)
2545                 goto fail2;
2546 
2547         (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rx_packet_cache",
2548             ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
2549 
2550         sp->s_rpc = kmem_cache_create(name, sizeof (sfxge_rx_packet_t),
2551             SFXGE_CPU_CACHE_SIZE, sfxge_rx_packet_ctor, sfxge_rx_packet_dtor,
2552             NULL, sp, NULL, 0);
2553         ASSERT(sp->s_rpc != NULL);
2554 
2555         (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rxq_cache",
2556             ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
2557 
2558         sp->s_rqc = kmem_cache_create(name, sizeof (sfxge_rxq_t),
2559             SFXGE_CPU_CACHE_SIZE, sfxge_rx_qctor, sfxge_rx_qdtor, NULL, sp,
2560             NULL, 0);
2561         ASSERT(sp->s_rqc != NULL);
2562 
2563         sp->s_rx_pkt_mem_max = ddi_prop_get_int64(DDI_DEV_T_ANY, sp->s_dip,
2564             DDI_PROP_DONTPASS, "rx_pkt_mem_max", 0); /* disabled */
2565 
2566         /* Initialize the receive queue(s) */
2567         for (index = 0; index < sip->si_nalloc; index++) {
2568                 if ((rc = sfxge_rx_qinit(sp, index)) != 0)
2569                         goto fail3;
2570         }
2571 
2572         sp->s_rx_coalesce_mode = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
2573             DDI_PROP_DONTPASS, "rx_coalesce_mode", SFXGE_RX_COALESCE_OFF);
2574 
2575         return (0);
2576 
2577 fail3:
2578         DTRACE_PROBE(fail3);
2579 
2580         /* Tear down the receive queue(s) */
2581         while (--index >= 0)
2582                 sfxge_rx_qfini(sp, index);
2583 
2584         kmem_cache_destroy(sp->s_rqc);
2585         sp->s_rqc = NULL;
2586 
2587         kmem_cache_destroy(sp->s_rpc);
2588         sp->s_rpc = NULL;
2589 
2590         sfxge_rx_scale_fini(sp);
2591 
2592 fail2:
2593         DTRACE_PROBE(fail2);
2594 fail1:
2595         DTRACE_PROBE1(fail1, int, rc);
2596 
2597         return (rc);
2598 }
2599 
2600 int
2601 sfxge_rx_start(sfxge_t *sp)
2602 {
2603         sfxge_mac_t *smp = &(sp->s_mac);
2604         sfxge_intr_t *sip;
2605         const efx_nic_cfg_t *encp;
2606         size_t hdrlen, align;
2607         int index;
2608         int rc;
2609 
2610         mutex_enter(&(smp->sm_lock));
2611 
2612         /* Calculate the receive packet buffer size and alignment */
2613         sp->s_rx_buffer_size = EFX_MAC_PDU(sp->s_mtu);
2614 
2615         encp = efx_nic_cfg_get(sp->s_enp);
2616 
2617         /* Packet buffer allocations are cache line aligned */
2618         EFSYS_ASSERT3U(encp->enc_rx_buf_align_start, <=, SFXGE_CPU_CACHE_SIZE);
2619 
2620         if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
2621                 sp->s_rx_prefix_size = encp->enc_rx_prefix_size;
2622 
2623                 hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);
2624 
2625                 /* Ensure IP headers are 32bit aligned */
2626                 sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
2627                 sp->s_rx_buffer_size += sp->s_rx_buffer_align;
2628 
2629         } else if (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT) {
2630                 sp->s_rx_prefix_size = encp->enc_rx_prefix_size;
2631 
2632                 /*
2633                  * Place the start of the buffer a prefix length minus 2
2634                  * before the start of a cache line. This ensures that the
2635                  * last two bytes of the prefix (which is where the LFSR hash
2636                  * is located) are in the same cache line as the headers, and
2637                  * the IP header is 32-bit aligned.
2638                  */
2639                 sp->s_rx_buffer_align =
2640                     SFXGE_CPU_CACHE_SIZE - (encp->enc_rx_prefix_size - 2);
2641                 sp->s_rx_buffer_size += sp->s_rx_buffer_align;
2642         } else {
2643                 sp->s_rx_prefix_size = 0;
2644 
2645                 /*
2646                  * Place the start of the buffer 2 bytes after a cache line
2647                  * boundary so that the headers fit into the cache line and
2648                  * the IP header is 32-bit aligned.
2649                  */
2650                 hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);
2651 
2652                 sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
2653                 sp->s_rx_buffer_size += sp->s_rx_buffer_align;
2654         }
2655 
2656         /* Align end of packet buffer for RX DMA end padding */
2657         align = MAX(1, encp->enc_rx_buf_align_end);
2658         EFSYS_ASSERT(ISP2(align));
2659         sp->s_rx_buffer_size = P2ROUNDUP(sp->s_rx_buffer_size, align);
2660 
2661         /* Initialize the receive module */
2662         if ((rc = efx_rx_init(sp->s_enp)) != 0)
2663                 goto fail1;
2664 
2665         mutex_exit(&(smp->sm_lock));
2666 
2667         if ((rc = sfxge_rx_scale_start(sp)) != 0)
2668                 goto fail2;
2669 
2670         /* Start the receive queue(s) */
2671         sip = &(sp->s_intr);
2672         for (index = 0; index < sip->si_nalloc; index++) {
2673                 if ((rc = sfxge_rx_qstart(sp, index)) != 0)
2674                         goto fail3;
2675         }
2676 
2677         ASSERT3U(sp->s_srp[0]->sr_state, ==, SFXGE_RXQ_STARTED);
2678         /* It is sufficient to have Rx scale initialized */
2679         ASSERT3U(sp->s_rx_scale.srs_state, ==, SFXGE_RX_SCALE_STARTED);
2680         rc = efx_mac_filter_default_rxq_set(sp->s_enp, sp->s_srp[0]->sr_erp,
2681             sp->s_rx_scale.srs_count > 1);
2682         if (rc != 0)
2683                 goto fail4;
2684 
2685         return (0);
2686 
2687 fail4:
2688         DTRACE_PROBE(fail4);
2689 
2690 fail3:
2691         DTRACE_PROBE(fail3);
2692 
2693         /* Stop the receive queue(s) */
2694         while (--index >= 0)
2695                 sfxge_rx_qstop(sp, index);
2696 
2697         sfxge_rx_scale_stop(sp);
2698 
2699 fail2:
2700         DTRACE_PROBE(fail2);
2701 
2702         mutex_enter(&(smp->sm_lock));
2703 
2704         /* Tear down the receive module */
2705         efx_rx_fini(sp->s_enp);
2706 
2707 fail1:
2708         DTRACE_PROBE1(fail1, int, rc);
2709 
2710         mutex_exit(&(smp->sm_lock));
2711 
2712         return (rc);
2713 }
2714 
2715 void
2716 sfxge_rx_coalesce_mode_get(sfxge_t *sp, sfxge_rx_coalesce_mode_t *modep)
2717 {
2718         *modep = sp->s_rx_coalesce_mode;
2719 }
2720 
2721 int
2722 sfxge_rx_coalesce_mode_set(sfxge_t *sp, sfxge_rx_coalesce_mode_t mode)
2723 {
2724         int rc;
2725 
2726         switch (mode) {
2727         case SFXGE_RX_COALESCE_OFF:
2728         case SFXGE_RX_COALESCE_DISALLOW_PUSH:
2729         case SFXGE_RX_COALESCE_ALLOW_PUSH:
2730                 break;
2731 
2732         default:
2733                 rc = EINVAL;
2734                 goto fail1;
2735         }
2736 
2737         sp->s_rx_coalesce_mode = mode;
2738 
2739         return (0);
2740 
2741 fail1:
2742         DTRACE_PROBE1(fail1, int, rc);
2743 
2744         return (rc);
2745 }
2746 
2747 void
2748 sfxge_rx_stop(sfxge_t *sp)
2749 {
2750         sfxge_mac_t *smp = &(sp->s_mac);
2751         sfxge_intr_t *sip = &(sp->s_intr);
2752         efx_nic_t *enp = sp->s_enp;
2753         int index;
2754 
2755         ASSERT(mutex_owned(&(sp->s_state_lock)));
2756 
2757         efx_mac_filter_default_rxq_clear(enp);
2758 
2759         /* Stop the receive queue(s) */
2760         index = sip->si_nalloc;
2761         while (--index >= 0) {
2762                 /* TBD: Flush RXQs in parallel; HW has limit + may need retry */
2763                 sfxge_rx_qstop(sp, index);
2764         }
2765 
2766         sfxge_rx_scale_stop(sp);
2767 
2768         mutex_enter(&(smp->sm_lock));
2769 
2770         /* Tear down the receive module */
2771         efx_rx_fini(enp);
2772 
2773         sp->s_rx_buffer_align = 0;
2774         sp->s_rx_prefix_size = 0;
2775         sp->s_rx_buffer_size = 0;
2776 
2777         mutex_exit(&(smp->sm_lock));
2778 }
2779 
2780 unsigned int
2781 sfxge_rx_loaned(sfxge_t *sp)
2782 {
2783         sfxge_intr_t *sip = &(sp->s_intr);
2784         int index;
2785         unsigned int loaned;
2786 
2787         ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
2788 
2789         loaned = 0;
2790         for (index = 0; index < sip->si_nalloc; index++) {
2791                 sfxge_rxq_t *srp = sp->s_srp[index];
2792                 sfxge_evq_t *sep = sp->s_sep[srp->sr_index];
2793 
2794                 mutex_enter(&(sep->se_lock));
2795 
2796                 loaned += sfxge_rx_qfpp_swizzle(srp);
2797 
2798                 mutex_exit(&(sep->se_lock));
2799         }
2800 
2801         return (loaned);
2802 }
2803 
2804 void
2805 sfxge_rx_fini(sfxge_t *sp)
2806 {
2807         sfxge_intr_t *sip = &(sp->s_intr);
2808         int index;
2809 
2810         ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
2811 
2812         sp->s_rx_coalesce_mode = SFXGE_RX_COALESCE_OFF;
2813 
2814         /* Tear down the receive queue(s) */
2815         index = sip->si_nalloc;
2816         while (--index >= 0)
2817                 sfxge_rx_qfini(sp, index);
2818 
2819         ASSERT3U(sp->s_rx_pkt_mem_alloc, ==, 0);
2820 
2821         kmem_cache_destroy(sp->s_rqc);
2822         sp->s_rqc = NULL;
2823 
2824         kmem_cache_destroy(sp->s_rpc);
2825         sp->s_rpc = NULL;
2826 
2827         sfxge_rx_scale_fini(sp);
2828 }