--- /dev/null	2013-08-22 18:57:34.000000000 -0400
+++ new/usr/src/uts/common/io/sfxge/sfxge_rx.c	2013-08-22 18:57:34.726123847 -0400
@@ -0,0 +1,2965 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2008-2013 Solarflare Communications Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <sys/sysmacros.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/atomic.h>
+#include <sys/stream.h>
+#include <sys/strsun.h>
+#include <sys/strsubr.h>
+#include <sys/strft.h>
+#include <sys/ksynch.h>
+#include <sys/ethernet.h>
+#include <sys/crc32.h>
+#include <sys/pattr.h>
+#include <sys/cpu.h>
+
+#include <sys/ethernet.h>
+#include <inet/ip.h>
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+
+#include "sfxge.h"
+
+#include "efx.h"
+
+/* RXQ flush response timeout (in microseconds) */
+#define	SFXGE_RX_QFLUSH_USEC	(2000000)
+
+/* RXQ default packet buffer preallocation (number of packet buffers) */
+#define	SFXGE_RX_QPREALLOC	(0)
+
+/* Receive packet DMA attributes */
+static ddi_device_acc_attr_t sfxge_rx_packet_devacc = {
+
+	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
+	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
+	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
+};
+
+static ddi_dma_attr_t sfxge_rx_packet_dma_attr = {
+	DMA_ATTR_V0,		/* dma_attr_version	*/
+	0,			/* dma_attr_addr_lo	*/
+	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
+	0xffffffffffffffffull,	/* dma_attr_count_max	*/
+	SFXGE_CPU_CACHE_SIZE,	/* dma_attr_align	*/
+	0xffffffff,		/* dma_attr_burstsizes	*/
+	1,			/* dma_attr_minxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_seg		*/
+	1,			/* dma_attr_sgllen	*/
+	1,			/* dma_attr_granular	*/
+	0			/* dma_attr_flags	*/
+};
+
+/* Receive queue DMA attributes */
+static ddi_device_acc_attr_t sfxge_rxq_devacc = {
+
+	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
+	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
+	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
+};
+
+static ddi_dma_attr_t sfxge_rxq_dma_attr = {
+	DMA_ATTR_V0,		/* dma_attr_version	*/
+	0,			/* dma_attr_addr_lo	*/
+	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
+	0xffffffffffffffffull,	/* dma_attr_count_max	*/
+	EFX_BUF_SIZE,		/* dma_attr_align	*/
+	0xffffffff,		/* dma_attr_burstsizes	*/
+	1,			/* dma_attr_minxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_seg		*/
+	1,			/* dma_attr_sgllen	*/
+	1,			/* dma_attr_granular	*/
+	0			/* dma_attr_flags	*/
+};
+
+/* Forward declaration */
+static int
+sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc);
+
+static int
+sfxge_rx_packet_ctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_rx_packet_t *srpp = buf;
+	sfxge_t *sp = arg;
+	dev_info_t *dip = sp->s_dip;
+	int err;
+
+	ASSERT3U(sizeof (srpp->__srp_u1.__srp_s1), <=,
+	    sizeof (srpp->__srp_u1.__srp_pad));
+	ASSERT3U(sizeof (srpp->__srp_u2.__srp_s2), <=,
+	    sizeof (srpp->__srp_u2.__srp_pad));
+
+	bzero(buf, sizeof (sfxge_rx_packet_t));
+
+	/* Allocate a DMA handle */
+	err = ddi_dma_alloc_handle(dip, &sfxge_rx_packet_dma_attr,
+	    (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
+	    NULL, &(srpp->srp_dma_handle));
+	if (err != DDI_SUCCESS)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, err);
+
+	SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
+
+	return (-1);
+}
+
+static void
+sfxge_rx_packet_dtor(void *buf, void *arg)
+{
+	sfxge_rx_packet_t *srpp = buf;
+
+	_NOTE(ARGUNUSED(arg))
+
+	/* Free the DMA handle */
+	ddi_dma_free_handle(&(srpp->srp_dma_handle));
+	srpp->srp_dma_handle = NULL;
+
+	SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
+}
+
+static int
+sfxge_rx_qctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_rxq_t *srp = buf;
+	efsys_mem_t *esmp = &(srp->sr_mem);
+	sfxge_t *sp = arg;
+	sfxge_dma_buffer_attr_t dma_attr;
+	sfxge_rx_fpp_t *srfppp;
+	int nprealloc;
+	unsigned int id;
+	int rc;
+
+	/* Compile-time structure layout checks */
+	EFX_STATIC_ASSERT(sizeof (srp->__sr_u1.__sr_s1) <=
+	    sizeof (srp->__sr_u1.__sr_pad));
+	EFX_STATIC_ASSERT(sizeof (srp->__sr_u2.__sr_s2) <=
+	    sizeof (srp->__sr_u2.__sr_pad));
+	EFX_STATIC_ASSERT(sizeof (srp->__sr_u3.__sr_s3) <=
+	    sizeof (srp->__sr_u3.__sr_pad));
+
+	bzero(buf, sizeof (sfxge_rxq_t));
+
+	srp->sr_sp = sp;
+
+	dma_attr.sdba_dip	 = sp->s_dip;
+	dma_attr.sdba_dattrp	 = &sfxge_rxq_dma_attr;
+	dma_attr.sdba_callback	 = DDI_DMA_SLEEP;
+	dma_attr.sdba_length	 = EFX_RXQ_SIZE(sp->s_rxq_size);
+	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
+	dma_attr.sdba_devaccp	 = &sfxge_rxq_devacc;
+	dma_attr.sdba_bindflags	 = DDI_DMA_READ | DDI_DMA_CONSISTENT;
+	dma_attr.sdba_maxcookies = 1;
+	dma_attr.sdba_zeroinit	 = B_FALSE;
+
+	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
+		goto fail1;
+
+	/* Allocate some buffer table entries */
+	if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_RXQ_NBUFS(sp->s_rxq_size),
+	    &(srp->sr_id))) != 0)
+		goto fail2;
+
+	/* Allocate the context array */
+	if ((srp->sr_srpp = kmem_zalloc(sizeof (sfxge_rx_packet_t *) *
+	    sp->s_rxq_size, kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail3;
+	}
+
+	/* Allocate the flow table */
+	if ((srp->sr_flow = kmem_zalloc(sizeof (sfxge_rx_flow_t) *
+	    SFXGE_MAX_FLOW, kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail4;
+	}
+
+	srp->sr_srfpp = &(srp->sr_srfp);
+	srp->sr_rto = drv_usectohz(200000);
+
+	srp->sr_mpp = &(srp->sr_mp);
+
+	/* Initialize the free packet pool */
+	srfppp = &(srp->sr_fpp);
+	if ((srfppp->srfpp_putp = kmem_zalloc(SFXGE_CPU_CACHE_SIZE *
+		SFXGE_RX_FPP_NSLOTS, kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail5;
+	}
+	for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
+		sfxge_rx_fpp_putlist_t *putp;
+		size_t off;
+
+		off = id * SFXGE_CPU_CACHE_SIZE;
+		putp = (void *)(srfppp->srfpp_putp + off);
+
+		putp->srfpl_putp = NULL;
+		putp->srfpl_putpp = &(putp->srfpl_putp);
+		mutex_init(&(putp->srfpl_lock), NULL, MUTEX_DRIVER,
+		    DDI_INTR_PRI(sp->s_intr.si_intr_pri));
+	}
+
+	cv_init(&(srp->sr_flush_kv), NULL, CV_DRIVER, NULL);
+
+	/* Preallocate some packets on the free packet pool */
+	nprealloc = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "rx_prealloc_pkt_buffers", SFXGE_RX_QPREALLOC);
+	sfxge_rx_qpreallocate(srp, nprealloc);
+
+
+	return (0);
+
+fail5:
+	DTRACE_PROBE(fail5);
+
+	srp->sr_mpp = NULL;
+
+	srp->sr_rto = 0;
+	srp->sr_srfpp = NULL;
+
+	/* Free the flow table */
+	kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
+	    SFXGE_MAX_FLOW);
+	srp->sr_flow = NULL;
+
+fail4:
+	DTRACE_PROBE(fail4);
+
+	/* Free the context array */
+	kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
+	    sp->s_rxq_size);
+	srp->sr_srpp = NULL;
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	/* Free the buffer table entries */
+	sfxge_sram_buf_tbl_free(sp, srp->sr_id,
+	    EFX_RXQ_NBUFS(sp->s_rxq_size));
+	srp->sr_id = 0;
+
+fail2:
+	DTRACE_PROBE(fail2);
+	/* Remove dma setup */
+	sfxge_dma_buffer_destroy(esmp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	srp->sr_sp = NULL;
+
+	SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
+
+	return (-1);
+}
+
+static void
+sfxge_rx_qdtor(void *buf, void *arg)
+{
+	sfxge_rxq_t *srp = buf;
+	efsys_mem_t *esmp = &(srp->sr_mem);
+	sfxge_t *sp = srp->sr_sp;
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	unsigned int id;
+
+	_NOTE(ARGUNUSED(arg))
+
+	cv_destroy(&(srp->sr_flush_kv));
+
+	/* Tear down the free packet pool */
+	for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
+		sfxge_rx_fpp_putlist_t *putp;
+		size_t off;
+
+		off = id * SFXGE_CPU_CACHE_SIZE;
+		putp = (void *)(srfppp->srfpp_putp + off);
+
+		putp->srfpl_putpp = NULL;
+		mutex_destroy(&(putp->srfpl_lock));
+
+		SFXGE_OBJ_CHECK(putp, sfxge_rx_fpp_putlist_t);
+	}
+	kmem_free(srfppp->srfpp_putp, SFXGE_CPU_CACHE_SIZE *
+	    SFXGE_RX_FPP_NSLOTS);
+	srfppp->srfpp_putp = NULL;
+
+	srp->sr_mpp = NULL;
+
+	srp->sr_rto = 0;
+	srp->sr_srfpp = NULL;
+
+	/* Free the flow table */
+	kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
+	    SFXGE_MAX_FLOW);
+	srp->sr_flow = NULL;
+
+	/* Free the context array */
+	kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
+	    sp->s_rxq_size);
+	srp->sr_srpp = NULL;
+
+	/* Free the buffer table entries */
+	sfxge_sram_buf_tbl_free(sp, srp->sr_id,
+	    EFX_RXQ_NBUFS(sp->s_rxq_size));
+	srp->sr_id = 0;
+
+	/* Tear down dma setup */
+	sfxge_dma_buffer_destroy(esmp);
+
+	SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
+}
+
+/* Note: This function takes ownership of *srpp. */
+static inline void
+sfxge_rx_qfpp_put(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
+{
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	mblk_t *mp = srpp->srp_mp;
+	unsigned int id;
+	size_t off;
+	sfxge_rx_fpp_putlist_t *putp;
+
+	ASSERT3P(mp->b_next, ==, NULL);
+	ASSERT3P(mp->b_prev, ==, NULL);
+
+	id = CPU->cpu_seqid & SFXGE_RX_FPP_MASK;
+	off = id * SFXGE_CPU_CACHE_SIZE;
+
+	ASSERT3P(srpp->srp_putp, ==, srfppp->srfpp_putp);
+	putp = (void *)(srpp->srp_putp + off);
+
+	mutex_enter(&(putp->srfpl_lock));
+	putp->srfpl_count++;
+	*putp->srfpl_putpp = mp;
+	putp->srfpl_putpp = &(mp->b_next);
+	mutex_exit(&(putp->srfpl_lock));
+}
+
+static unsigned int
+sfxge_rx_qfpp_swizzle(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	unsigned int start;
+	unsigned int id;
+	mblk_t *p;
+	mblk_t **pp;
+	unsigned int count;
+	unsigned int loaned;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* We want to access the put list for the current CPU last */
+	id = start = (CPU->cpu_seqid + 1) & SFXGE_RX_FPP_MASK;
+
+	do {
+		sfxge_rx_fpp_putlist_t *putp;
+		size_t off;
+
+		off = id * SFXGE_CPU_CACHE_SIZE;
+		id  = (id + 1) & SFXGE_RX_FPP_MASK;
+
+		putp = (void *)(srfppp->srfpp_putp + off);
+
+		/* Acquire the put list */
+		mutex_enter(&(putp->srfpl_lock));
+
+		p = putp->srfpl_putp;
+		pp = putp->srfpl_putpp;
+		count = putp->srfpl_count;
+
+		putp->srfpl_putp = NULL;
+		putp->srfpl_putpp = &(putp->srfpl_putp);
+		putp->srfpl_count = 0;
+
+		mutex_exit(&(putp->srfpl_lock));
+
+		if (p == NULL)
+			continue;
+
+		/* Add the list to the head of the get list */
+		*pp = srfppp->srfpp_get;
+		srfppp->srfpp_get = p;
+
+		/* Adjust the counters */
+		ASSERT3U(srfppp->srfpp_loaned, >=, count);
+		srfppp->srfpp_loaned -= count;
+		srfppp->srfpp_count += count;
+
+#if 0
+		/* NOTE: this probe is disabled because it is expensive!! */
+		DTRACE_PROBE2(count,
+		    unsigned int, (id - 1) & SFXGE_RX_FPP_MASK,
+		    unsigned int, count);
+#endif
+
+	} while (id != start);
+
+	/* Return the number of packets yet to appear in the put list */
+	loaned = srfppp->srfpp_loaned;
+
+
+	return (loaned);
+}
+
+
+#define	DB_FRTNP(mp)	((mp)->b_datap->db_frtnp)
+
+static void
+sfxge_rx_qfpp_empty(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	sfxge_rx_fpp_t *srfppp;
+	mblk_t *mp;
+
+	mutex_enter(&(sep->se_lock));
+	srfppp = &(srp->sr_fpp);
+
+	/* Swizzle put list to get list */
+	(void) sfxge_rx_qfpp_swizzle(srp);
+	ASSERT3U(srfppp->srfpp_loaned, ==, 0);
+
+	mp = srfppp->srfpp_get;
+	srfppp->srfpp_get = NULL;
+
+	/* Free the remainder */
+	while (mp != NULL) {
+		mblk_t *next;
+		frtn_t *freep;
+		sfxge_rx_packet_t *srpp;
+
+		next = mp->b_next;
+		mp->b_next = NULL;
+
+		ASSERT3U(srfppp->srfpp_count, >, 0);
+		srfppp->srfpp_count--;
+
+		freep = DB_FRTNP(mp);
+		/*
+		 * ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
+		 *   is implied by srpp test below
+		 */
+		/*LINTED*/
+		srpp = (sfxge_rx_packet_t *)(freep->free_arg);
+		ASSERT3P(srpp->srp_mp, ==, mp);
+		ASSERT3P(mp->b_cont, ==, NULL);
+		srpp->srp_recycle = B_FALSE;
+
+		freeb(mp);
+
+		mp = next;
+	}
+	ASSERT3U(srfppp->srfpp_count, ==, 0);
+
+	srfppp->srfpp_min = 0;
+
+	mutex_exit(&(sep->se_lock));
+}
+
+/*
+ * This is an estimate of all memory consumed per RX packet
+ * it can be inaccurate but but sp->s_rx_pkt_mem_alloc mustn't drift
+ */
+static uint64_t
+sfxge_rx_pkt_mem_approx(const sfxge_rx_packet_t *srpp)
+{
+	return (srpp->srp_mblksize + sizeof (mblk_t) + sizeof (dblk_t) +
+	    sizeof (sfxge_rx_packet_t));
+}
+
+static void
+sfxge_rx_qpacket_destroy(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
+
+	ASSERT(!(srpp->srp_recycle));
+	ASSERT3P(srpp->srp_mp, ==, NULL);
+
+	srpp->srp_off = 0;
+	srpp->srp_thp = NULL;
+	srpp->srp_iphp = NULL;
+	srpp->srp_etherhp = NULL;
+	srpp->srp_size = 0;
+	srpp->srp_flags = 0;
+
+	bzero(&(srpp->srp_free), sizeof (frtn_t));
+
+	srpp->srp_mblksize = 0;
+	srpp->srp_base = NULL;
+
+	/* Unbind the DMA memory from the DMA handle */
+	srpp->srp_addr = 0;
+	(void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
+
+	/* Free the DMA memory */
+	srpp->srp_base = NULL;
+	ddi_dma_mem_free(&(srpp->srp_acc_handle));
+	srpp->srp_acc_handle = NULL;
+
+	srpp->srp_putp = NULL;
+	srpp->srp_srp = NULL;
+
+	kmem_cache_free(sp->s_rpc, srpp);
+	if (sp->s_rx_pkt_mem_max)
+		atomic_add_64(&sp->s_rx_pkt_mem_alloc, -delta);
+}
+
+#ifdef _USE_XESBALLOC
+static void
+sfxge_rx_qpacket_free(void *arg, mblk_t *mp, boolean_t *recyclep)
+{
+	sfxge_rx_packet_t *srpp = arg;
+	sfxge_rxq_t *srp = srpp->srp_srp;
+
+	/*
+	 * WARNING "man -s 9f esballoc" states:
+	 * => runs async in a background context
+	 * => must not sleep, or access data structures that could be freed
+	 */
+	ASSERT3P(DB_BASE(mp), ==, srpp->srp_base);
+	ASSERT3P(MBLKSIZE(mp), ==, srpp->srp_mblksize);
+
+	/* Check whether we want to recycle the receive packets */
+	if (srpp->srp_recycle) {
+		ASSERT3P(DB_FRTNP(mp), ==, &(srpp->srp_free));
+
+		srpp->srp_mp = mp;
+
+		/* NORMAL recycled case */
+		sfxge_rx_qfpp_put(srp, srpp);
+		*recyclep = B_TRUE;
+		return;
+	}
+
+	srpp->srp_mp = NULL;
+
+	sfxge_rx_qpacket_destroy(srp, srpp);
+	*recyclep = B_FALSE;
+}
+#endif	/* _USE_XESBALLOC */
+
+#ifdef _USE_DESBALLOC
+static void
+sfxge_rx_qpacket_free(void *arg)
+{
+	sfxge_rx_packet_t *srpp = arg;
+	sfxge_rxq_t *srp = srpp->srp_srp;
+
+	/*
+	 * WARNING "man -s 9f esballoc"  states:
+	 * => runs sync from the thread calling freeb()
+	 * => must not sleep, or access data structures that could be freed
+	 */
+
+	/* Check whether we want to recycle the receive packets */
+	if (srpp->srp_recycle) {
+		frtn_t *freep;
+		mblk_t *mp;
+		size_t size;
+
+		freep = &(srpp->srp_free);
+		ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
+		ASSERT3P(freep->free_arg, ==, (caddr_t)srpp);
+
+		/*
+		 * Allocate a matching mblk_t before the current one is
+		 * freed.
+		 */
+		size = srpp->srp_mblksize;
+
+		if ((mp = desballoc(srpp->srp_base, size, BPRI_HI,
+		    freep)) != NULL) {
+			srpp->srp_mp = mp;
+
+			/* NORMAL recycled case */
+			sfxge_rx_qfpp_put(srp, srpp);
+			return;
+		}
+	}
+
+	srpp->srp_mp = NULL;
+
+	sfxge_rx_qpacket_destroy(srp, srpp);
+}
+#endif	/* _USE_DESBALLOC */
+
+static sfxge_rx_packet_t *
+sfxge_rx_qpacket_create(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	sfxge_rx_packet_t *srpp;
+	size_t size;
+	caddr_t base;
+	size_t unit;
+	ddi_dma_cookie_t dmac;
+	unsigned int ncookies;
+	frtn_t *freep;
+	mblk_t *mp;
+	int err;
+	int rc;
+
+	size = sp->s_rx_buffer_size;
+
+	if (sp->s_rx_pkt_mem_max &&
+	    (sp->s_rx_pkt_mem_alloc + size >= sp->s_rx_pkt_mem_max)) {
+		DTRACE_PROBE(rx_pkt_mem_max);
+		srp->sr_kstat.srk_rx_pkt_mem_limit++;
+		return (NULL);
+	}
+
+	/* Allocate a new packet */
+	if ((srpp = kmem_cache_alloc(sp->s_rpc, KM_NOSLEEP)) == NULL) {
+		srp->sr_kstat.srk_kcache_alloc_nomem++;
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	srpp->srp_srp = srp;
+	srpp->srp_putp = srfppp->srfpp_putp;
+
+	/* Allocate some DMA memory */
+	err = ddi_dma_mem_alloc(srpp->srp_dma_handle, size,
+	    &sfxge_rx_packet_devacc, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
+	    NULL, &base, &unit, &(srpp->srp_acc_handle));
+	switch (err) {
+	case DDI_SUCCESS:
+		break;
+
+	case DDI_FAILURE:
+		srp->sr_kstat.srk_dma_alloc_nomem++;
+		rc = ENOMEM;
+		goto fail2;
+
+	default:
+		srp->sr_kstat.srk_dma_alloc_fail++;
+		rc = EFAULT;
+		goto fail2;
+	}
+
+	/* Adjust the buffer to align the start of the DMA area correctly */
+	base += sp->s_rx_buffer_align;
+	size -= sp->s_rx_buffer_align;
+
+	/* Bind the DMA memory to the DMA handle */
+	err = ddi_dma_addr_bind_handle(srpp->srp_dma_handle, NULL,
+	    base, size, DDI_DMA_READ | DDI_DMA_STREAMING,
+	    DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies);
+	switch (err) {
+	case DDI_DMA_MAPPED:
+		break;
+
+	case DDI_DMA_INUSE:
+		srp->sr_kstat.srk_dma_bind_fail++;
+		rc = EEXIST;
+		goto fail3;
+
+	case DDI_DMA_NORESOURCES:
+		srp->sr_kstat.srk_dma_bind_nomem++;
+		rc = ENOMEM;
+		goto fail3;
+
+	case DDI_DMA_NOMAPPING:
+		srp->sr_kstat.srk_dma_bind_fail++;
+		rc = ENOTSUP;
+		goto fail3;
+
+	case DDI_DMA_TOOBIG:
+		srp->sr_kstat.srk_dma_bind_fail++;
+		rc = EFBIG;
+		goto fail3;
+
+	default:
+		srp->sr_kstat.srk_dma_bind_fail++;
+		rc = EFAULT;
+		goto fail3;
+	}
+	ASSERT3U(ncookies, ==, 1);
+
+	srpp->srp_addr = dmac.dmac_laddress;
+
+	srpp->srp_base = (unsigned char *)base;
+	srpp->srp_mblksize = size;
+
+	/*
+	 * Allocate a STREAMS block: We use size 1 so that the allocator will
+	 * use the first (and smallest) dblk cache.
+	 */
+	freep = &(srpp->srp_free);
+	freep->free_func = sfxge_rx_qpacket_free;
+	freep->free_arg  = (caddr_t)srpp;
+
+#ifdef _USE_XESBALLOC
+	if ((mp = xesballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) {
+		srp->sr_kstat.srk_xesballoc_fail++;
+		rc = ENOMEM;
+		goto fail4;
+	}
+#endif	/* _USE_XESBALLOC */
+
+#ifdef _USE_DESBALLOC
+	if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) {
+		srp->sr_kstat.srk_desballoc_fail++;
+		rc = ENOMEM;
+		goto fail4;
+	}
+#endif	/* _USE_DESBALLOC */
+
+	srpp->srp_mp = mp;
+	srpp->srp_recycle = B_TRUE;
+
+	if (sp->s_rx_pkt_mem_max) {
+		int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
+		atomic_add_64(&sp->s_rx_pkt_mem_alloc, delta);
+	}
+
+	return (srpp);
+
+fail4:
+	DTRACE_PROBE(fail4);
+
+	bzero(&(srpp->srp_free), sizeof (frtn_t));
+
+	srpp->srp_mblksize = 0;
+	srpp->srp_base = NULL;
+
+	/* Unbind the DMA memory from the DMA handle */
+	srpp->srp_addr = 0;
+	(void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	/* Free the DMA memory */
+	ddi_dma_mem_free(&(srpp->srp_acc_handle));
+	srpp->srp_acc_handle = NULL;
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	srpp->srp_putp = NULL;
+	srpp->srp_srp = NULL;
+
+	kmem_cache_free(sp->s_rpc, srpp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (NULL);
+}
+
+#define	SFXGE_REFILL_BATCH  64
+
+/* Try to refill the RX descriptor ring from the associated free pkt pool */
+static void
+sfxge_rx_qrefill(sfxge_rxq_t *srp, unsigned int target)
+{
+	sfxge_t *sp = srp->sr_sp;
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
+	mblk_t *mp;
+	int ntodo;
+	unsigned int count;
+	unsigned int batch;
+	unsigned int rxfill;
+	unsigned int mblksize;
+
+	prefetch_read_many(sp->s_enp);
+	prefetch_read_many(srp->sr_erp);
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	if (srp->sr_state != SFXGE_RXQ_STARTED)
+		return;
+
+	rxfill = srp->sr_added - srp->sr_completed;
+	ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
+	ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
+	ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
+
+	if (ntodo == 0)
+		goto out;
+
+	(void) sfxge_rx_qfpp_swizzle(srp);
+
+	mp = srfppp->srfpp_get;
+	count = srfppp->srfpp_count;
+	mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
+
+	batch = 0;
+	while (ntodo-- > 0) {
+		mblk_t *next;
+		frtn_t *freep;
+		sfxge_rx_packet_t *srpp;
+		unsigned int id;
+
+		if (mp == NULL)
+			break;
+
+		next = mp->b_next;
+		mp->b_next = NULL;
+
+		if (next != NULL)
+			prefetch_read_many(next);
+
+		freep = DB_FRTNP(mp);
+		/*LINTED*/
+		srpp = (sfxge_rx_packet_t *)(freep->free_arg);
+		ASSERT3P(srpp->srp_mp, ==, mp);
+
+		/* The MTU may have changed since the packet was allocated */
+		if (MBLKSIZE(mp) != mblksize) {
+			srpp->srp_recycle = B_FALSE;
+
+			freeb(mp);
+
+			--count;
+			mp = next;
+			continue;
+		}
+
+		srpp->srp_off = 0;
+		srpp->srp_thp = NULL;
+		srpp->srp_iphp = NULL;
+		srpp->srp_etherhp = NULL;
+		srpp->srp_size = 0;
+		srpp->srp_flags = EFX_DISCARD;
+
+		id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
+		ASSERT(srp->sr_srpp[id] == NULL);
+		srp->sr_srpp[id] = srpp;
+
+		addr[batch++] = srpp->srp_addr;
+		if (batch == SFXGE_REFILL_BATCH) {
+			efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
+			    srp->sr_completed, srp->sr_added);
+			srp->sr_added += batch;
+			batch = 0;
+		}
+
+		--count;
+		mp = next;
+	}
+
+	srfppp->srfpp_get = mp;
+	srfppp->srfpp_count = count;
+
+	if (batch != 0) {
+		efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
+		    srp->sr_completed, srp->sr_added);
+		srp->sr_added += batch;
+	}
+
+	/* Make the descriptors visible to the hardware */
+	(void) ddi_dma_sync(srp->sr_mem.esm_dma_handle,
+	    0,
+	    EFX_RXQ_SIZE(sp->s_rxq_size),
+	    DDI_DMA_SYNC_FORDEV);
+
+	efx_rx_qpush(srp->sr_erp, srp->sr_added);
+
+out:
+	if (srfppp->srfpp_count < srfppp->srfpp_min)
+		srfppp->srfpp_min = srfppp->srfpp_count;
+}
+
+/* Preallocate packets and put them in the free packet pool */
+static int
+sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc)
+{
+	sfxge_rx_fpp_t *srfppp = &((srp)->sr_fpp);
+	srfppp->srfpp_lowat = nprealloc;
+	while (nprealloc-- > 0) {
+		sfxge_rx_packet_t *srpp;
+
+		if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
+			break;
+		sfxge_rx_qfpp_put(srp, srpp);
+	}
+	return (0);
+}
+
+/* Try to refill the RX descriptor ring by allocating new packets */
+static void
+sfxge_rx_qfill(sfxge_rxq_t *srp, unsigned int target)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	unsigned int batch;
+	unsigned int rxfill;
+	unsigned int mblksize;
+	int ntodo;
+	efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
+	mblk_t *mp = NULL;
+
+	prefetch_read_many(sp->s_enp);
+	prefetch_read_many(srp->sr_erp);
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	if (srp->sr_state != SFXGE_RXQ_STARTED)
+		return;
+
+	rxfill = srp->sr_added - srp->sr_completed;
+	ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
+	ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
+	ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
+
+	if (ntodo == 0)
+		return;
+
+	mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
+
+	batch = 0;
+	while (ntodo-- > 0) {
+		sfxge_rx_packet_t *srpp;
+		unsigned int id;
+
+		if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
+			break;
+
+		mp = srpp->srp_mp;
+
+		ASSERT3U(MBLKSIZE(mp), ==, mblksize);
+
+		ASSERT3U(srpp->srp_off, ==, 0);
+		ASSERT3P(srpp->srp_thp, ==, NULL);
+		ASSERT3P(srpp->srp_iphp, ==, NULL);
+		ASSERT3P(srpp->srp_etherhp, ==, NULL);
+		ASSERT3U(srpp->srp_size, ==, 0);
+
+		srpp->srp_flags = EFX_DISCARD;
+
+		id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
+		ASSERT(srp->sr_srpp[id] == NULL);
+		srp->sr_srpp[id] = srpp;
+
+		addr[batch++] = srpp->srp_addr;
+		if (batch == SFXGE_REFILL_BATCH) {
+			efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
+			    srp->sr_completed, srp->sr_added);
+			srp->sr_added += batch;
+			batch = 0;
+		}
+	}
+
+	if (batch != 0) {
+		efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
+		    srp->sr_completed, srp->sr_added);
+		srp->sr_added += batch;
+	}
+
+	/* Make the descriptors visible to the hardware */
+	(void) ddi_dma_sync(srp->sr_mem.esm_dma_handle,
+	    0,
+	    EFX_RXQ_SIZE(sp->s_rxq_size),
+	    DDI_DMA_SYNC_FORDEV);
+
+	efx_rx_qpush(srp->sr_erp, srp->sr_added);
+}
+
+void
+sfxge_rx_qfpp_trim(sfxge_rxq_t *srp)
+{
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	mblk_t *p;
+	mblk_t **pp;
+	int count;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	if (srp->sr_state != SFXGE_RXQ_STARTED)
+		goto done;
+
+	/* Make sure the queue is full */
+	sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
+
+	/* The refill may have emptied the pool */
+	if (srfppp->srfpp_min == 0)
+		goto done;
+
+	/* Don't trim below the pool's low water mark */
+	if (srfppp->srfpp_count <= srfppp->srfpp_lowat)
+		goto done;
+
+	ASSERT(srfppp->srfpp_min <= srfppp->srfpp_count);
+
+	/* Trim to the largest of srfppp->srfpp_min and srfpp->srfpp_lowat */
+	if (srfppp->srfpp_lowat > srfppp->srfpp_min)
+		count = srfppp->srfpp_count - srfppp->srfpp_lowat;
+	else
+		count = srfppp->srfpp_count - srfppp->srfpp_min;
+
+	/* Walk the get list */
+	pp = &(srfppp->srfpp_get);
+	while (--count >= 0) {
+		ASSERT(pp);
+		p = *pp;
+		ASSERT(p != NULL);
+
+		pp = &(p->b_next);
+	}
+	ASSERT(pp);
+	p = *pp;
+
+	/* Truncate the get list */
+	*pp = NULL;
+
+	/* Free the remainder */
+	while (p != NULL) {
+		mblk_t *next;
+		frtn_t *freep;
+		sfxge_rx_packet_t *srpp;
+
+		next = p->b_next;
+		p->b_next = NULL;
+
+		ASSERT3U(srfppp->srfpp_min, >, 0);
+		srfppp->srfpp_min--;
+		srfppp->srfpp_count--;
+
+		freep = DB_FRTNP(p);
+		/*LINTED*/
+		srpp = (sfxge_rx_packet_t *)(freep->free_arg);
+		ASSERT3P(srpp->srp_mp, ==, p);
+
+		srpp->srp_recycle = B_FALSE;
+
+		freeb(p);
+
+		p = next;
+	}
+
+done:
+	srfppp->srfpp_min = srfppp->srfpp_count;
+}
+
+static void
+sfxge_rx_qpoll(void *arg)
+{
+	sfxge_rxq_t *srp = arg;
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	uint16_t magic;
+
+	/*
+	 * man timeout(9f) states that this code should adhere to the
+	 * same requirements as a softirq handler - DO NOT BLOCK
+	 */
+
+	/*
+	 * Post an event to the event queue to cause the free packet pool to be
+	 * trimmed if it is oversize.
+	 */
+	magic = SFXGE_MAGIC_RX_QFPP_TRIM | index;
+
+#if defined(DEBUG)
+	/* This is guaranteed due to the start/stop order of rx and ev */
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
+#else
+	/*
+	 * Bug22691 WORKAROUND:
+	 * This handler has been observed in the field to be invoked for a
+	 * queue in the INITIALIZED state, which should never happen.
+	 * Until the mechanism for this is properly understood, add defensive
+	 * checks.
+	 */
+	if ((sep->se_state != SFXGE_EVQ_STARTED) ||
+	    (srp->sr_state != SFXGE_RXQ_STARTED) ||
+	    (!sep->se_eep)) {
+		cmn_err(CE_WARN, SFXGE_CMN_ERR
+			"[%s%d] RXQ[%d] bad state in sfxge_rx_qpoll %d %d %p",
+			ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip),
+			index, sep->se_state, srp->sr_state, sep->se_eep);
+		return;
+	}
+#endif
+	efx_ev_qpost(sep->se_eep, magic);
+
+	srp->sr_tid = timeout(sfxge_rx_qpoll, srp,
+		drv_usectohz(sp->s_rxq_poll_usec));
+}
+
+static void
+sfxge_rx_qpoll_start(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
+
+	/* Schedule a poll */
+	ASSERT3P(srp->sr_tid, ==, 0);
+	srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 0);
+}
+
+static void
+sfxge_rx_qpoll_stop(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	timeout_id_t tid;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
+
+	/*
+	 * Cancel the qpoll timer. Care is needed as this function
+	 * can race with sfxge_rx_qpoll() for timeout id updates.
+	 *
+	 * Do not hold locks used by any timeout(9f) handlers across
+	 * calls to untimeout(9f) as this will deadlock.
+	 */
+	tid = 0;
+	while ((srp->sr_tid != 0) && (srp->sr_tid != tid)) {
+		tid = srp->sr_tid;
+		(void) untimeout(tid);
+	}
+	srp->sr_tid = 0;
+}
+
+static int
+sfxge_rx_kstat_update(kstat_t *ksp, int rw)
+{
+	sfxge_rxq_t *srp = ksp->ks_private;
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	kstat_named_t *knp;
+	int rc;
+
+	if (rw != KSTAT_READ) {
+		rc = EACCES;
+		goto fail1;
+	}
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+	if (srp->sr_state != SFXGE_RXQ_STARTED)
+		goto done;
+
+	knp = ksp->ks_data;
+	/* NB pointer post-increment below */
+	knp++->value.ui32 = srp->sr_kstat.srk_rx_pkt_mem_limit;
+	knp++->value.ui32 = srp->sr_kstat.srk_kcache_alloc_nomem;
+	knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_nomem;
+	knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_fail;
+	knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_nomem;
+	knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_fail;
+#ifdef _USE_XESBALLOC
+	knp++->value.ui32 = srp->sr_kstat.srk_xesballoc_fail;
+#endif
+#ifdef _USE_DESBALLOC
+	knp++->value.ui32 = srp->sr_kstat.srk_desballoc_fail;
+#endif
+	knp++->value.ui32 = srp->sr_kstat.srk_rxq_empty_discard;
+
+done:
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_rx_kstat_init(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	dev_info_t *dip = sp->s_dip;
+	char name[MAXNAMELEN];
+	kstat_t *ksp;
+	kstat_named_t *knp;
+	int rc;
+
+	/* Create the set */
+	(void) snprintf(name, MAXNAMELEN - 1, "%s_rxq%04d",
+	    ddi_driver_name(dip), index);
+
+	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
+	    ddi_get_instance(dip), name, "rxq", KSTAT_TYPE_NAMED,
+	    SFXGE_RX_NSTATS, 0)) == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	srp->sr_ksp = ksp;
+
+	ksp->ks_update = sfxge_rx_kstat_update;
+	ksp->ks_private = srp;
+	ksp->ks_lock = &(sep->se_lock);
+
+	/* Initialise the named stats */
+	knp = ksp->ks_data;
+	kstat_named_init(knp, "rx_pkt_mem_limit", KSTAT_DATA_UINT32);
+	knp++;
+	kstat_named_init(knp, "kcache_alloc_nomem", KSTAT_DATA_UINT32);
+	knp++;
+	kstat_named_init(knp, "dma_alloc_nomem", KSTAT_DATA_UINT32);
+	knp++;
+	kstat_named_init(knp, "dma_alloc_fail", KSTAT_DATA_UINT32);
+	knp++;
+	kstat_named_init(knp, "dma_bind_nomem", KSTAT_DATA_UINT32);
+	knp++;
+	kstat_named_init(knp, "dma_bind_fail", KSTAT_DATA_UINT32);
+	knp++;
+#ifdef _USE_XESBALLOC
+	kstat_named_init(knp, "xesballoc_fail", KSTAT_DATA_UINT32);
+#endif
+#ifdef _USE_DESBALLOC
+	kstat_named_init(knp, "desballoc_fail", KSTAT_DATA_UINT32);
+#endif
+	kstat_named_init(knp, "rxq_empty_discard", KSTAT_DATA_UINT32);
+
+	kstat_install(ksp);
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_rx_qinit(sfxge_t *sp, unsigned int index)
+{
+	sfxge_rxq_t *srp;
+	int rc;
+
+	ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
+
+	srp = kmem_cache_alloc(sp->s_rqc, KM_SLEEP);
+
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_UNINITIALIZED);
+
+	srp->sr_index = index;
+	sp->s_srp[index] = srp;
+
+	if ((rc = sfxge_rx_kstat_init(srp)) != 0)
+		goto fail1;
+
+	srp->sr_state = SFXGE_RXQ_INITIALIZED;
+
+	return (0);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+	kmem_cache_free(sp->s_rqc, srp);
+
+	return (rc);
+}
+
+static int
+sfxge_rx_qstart(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	sfxge_rxq_t *srp;
+	efsys_mem_t *esmp;
+	efx_nic_t *enp;
+	unsigned int level;
+	int rc;
+
+	mutex_enter(&(sep->se_lock));
+	srp = sp->s_srp[index];
+	enp = sp->s_enp;
+	esmp = &(srp->sr_mem);
+
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
+
+	/* Zero the memory */
+	(void) memset(esmp->esm_base, 0, EFX_RXQ_SIZE(sp->s_rxq_size));
+
+	/* Program the buffer table */
+	if ((rc = sfxge_sram_buf_tbl_set(sp, srp->sr_id, esmp,
+	    EFX_RXQ_NBUFS(sp->s_rxq_size))) != 0)
+		goto fail1;
+
+	/* Create the receive queue */
+	if ((rc = efx_rx_qcreate(enp, index, index, EFX_RXQ_TYPE_DEFAULT,
+	    esmp, sp->s_rxq_size, srp->sr_id, sep->se_eep, &(srp->sr_erp)))
+	    != 0)
+		goto fail2;
+
+	/* Enable the receive queue */
+	efx_rx_qenable(srp->sr_erp);
+
+	/* Set the water marks */
+	srp->sr_hiwat = EFX_RXQ_LIMIT(sp->s_rxq_size) * 9 / 10;
+	srp->sr_lowat = srp->sr_hiwat / 2;
+
+	srp->sr_state = SFXGE_RXQ_STARTED;
+
+	sfxge_rx_qpoll_start(srp);
+
+	/* Try to fill the queue from the pool */
+	sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
+
+	/*
+	 * If there were insufficient buffers in the pool to reach the at
+	 * least a batch then allocate some.
+	 */
+	level = srp->sr_added - srp->sr_completed;
+	if (level < SFXGE_RX_BATCH)
+		sfxge_rx_qfill(srp, SFXGE_RX_BATCH);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	/* Clear entries from the buffer table */
+	sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
+	    EFX_RXQ_NBUFS(sp->s_rxq_size));
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (rc);
+}
+
+static void
+sfxge_rx_qflow_complete(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp)
+{
+	mblk_t *mp;
+	struct ether_header *etherhp;
+	struct ip *iphp;
+	struct tcphdr *thp;
+
+	if (srfp->srf_mp == NULL)
+		return;
+
+	mp = srfp->srf_mp;
+	etherhp = srfp->srf_etherhp;
+	iphp = srfp->srf_iphp;
+	thp = srfp->srf_last_thp;
+
+	ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
+	    sizeof (struct ether_vlan_header) :
+	    sizeof (struct ether_header)) +
+	    srfp->srf_len, ==, msgdsize(mp));
+
+	ASSERT3U(srfp->srf_len & 0xffff, ==, srfp->srf_len);
+	iphp->ip_len = htons(srfp->srf_len);
+
+	srfp->srf_first_thp->th_ack = thp->th_ack;
+	srfp->srf_first_thp->th_win = thp->th_win;
+	srfp->srf_first_thp->th_flags = thp->th_flags;
+
+	DTRACE_PROBE2(flow_complete, uint32_t, srfp->srf_tag,
+	    size_t, srfp->srf_len);
+
+	srfp->srf_mp = NULL;
+	srfp->srf_len = 0;
+
+	ASSERT(mp->b_next == NULL);
+	*(srp->sr_mpp) = mp;
+	srp->sr_mpp = &(mp->b_next);
+}
+
+static boolean_t
+sfxge_rx_qflow_add(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp,
+    sfxge_rx_packet_t *srpp, clock_t now)
+{
+	sfxge_t *sp = srp->sr_sp;
+	struct ether_header *etherhp = srpp->srp_etherhp;
+	struct ip *iphp = srpp->srp_iphp;
+	struct tcphdr *thp = srpp->srp_thp;
+	size_t off = srpp->srp_off;
+	size_t size = (size_t)(srpp->srp_size);
+	mblk_t *mp = srpp->srp_mp;
+	uint32_t seq;
+	unsigned int shift;
+
+	ASSERT3U(MBLKL(mp), ==, off + size);
+	ASSERT3U(DB_CKSUMFLAGS(mp), ==,
+	    HCK_FULLCKSUM | HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM);
+
+	seq = htonl(thp->th_seq);
+
+	/*
+	 * If the time between this segment and the last is greater than RTO
+	 * then consider this a new flow.
+	 */
+	if (now - srfp->srf_lbolt > srp->sr_rto) {
+		srfp->srf_count = 1;
+		srfp->srf_seq = seq + size;
+
+		goto fail1;
+	}
+
+	if (seq != srfp->srf_seq) {
+		if (srfp->srf_count > SFXGE_SLOW_START)
+			srfp->srf_count = SFXGE_SLOW_START;
+
+		srfp->srf_count >>= 1;
+
+		srfp->srf_count++;
+		srfp->srf_seq = seq + size;
+
+		goto fail2;
+	}
+
+	/* Update the in-order segment count and sequence number */
+	srfp->srf_count++;
+	srfp->srf_seq = seq + size;
+
+	/* Don't merge across pure ACK, URG, SYN or RST segments */
+	if (size == 0 || thp->th_flags & (TH_URG | TH_SYN | TH_RST) ||
+	    thp->th_urp != 0)
+		goto fail3;
+
+	/*
+	 * If the in-order segment count has not yet reached the slow-start
+	 * threshold then we cannot coalesce.
+	 */
+	if (srfp->srf_count < SFXGE_SLOW_START)
+		goto fail4;
+
+	/* Scale up the packet size from 4k (the maximum being 64k) */
+	ASSERT3U(srfp->srf_count, >=, SFXGE_SLOW_START);
+	shift = MIN(srfp->srf_count - SFXGE_SLOW_START + 12, 16);
+	if (srfp->srf_len + size >= (1 << shift))
+		sfxge_rx_qflow_complete(srp, srfp);
+
+	ASSERT(mp->b_cont == NULL);
+
+#ifdef _USE_GLD_V3_SOL10
+	/*
+	 * The IP and UDP layers in Solaris 10 have slow paths for
+	 * handling mblks with more than 2 fragments.
+	 * UDP: see OpenSolaris CR 6305037
+	 * IP: see <http://www.mail-archive.com/networking-discuss@
+	 *   opensolaris.org/msg07366.html>
+	 */
+	if (srfp->srf_mp && srfp->srf_mp->b_cont) {
+		sfxge_rx_qflow_complete(srp, srfp);
+	}
+#endif
+
+	if (srfp->srf_mp == NULL) {
+		/* First packet in this flow */
+		srfp->srf_etherhp = etherhp;
+		srfp->srf_iphp = iphp;
+		srfp->srf_first_thp = srfp->srf_last_thp = thp;
+
+		ASSERT3P(mp->b_cont, ==, NULL);
+		srfp->srf_mp = mp;
+		srfp->srf_mpp = &(mp->b_cont);
+
+		srfp->srf_len = ntohs(iphp->ip_len);
+
+		/*
+		 * If the flow is not already in the list of occupied flows then
+		 * add it.
+		 */
+		if (srfp->srf_next == NULL &&
+		    srp->sr_srfpp != &(srfp->srf_next)) {
+			*(srp->sr_srfpp) = srfp;
+			srp->sr_srfpp = &(srfp->srf_next);
+		}
+	} else {
+		/* Later packet in this flow - skip TCP header */
+		srfp->srf_last_thp = thp;
+
+		mp->b_rptr += off;
+		ASSERT3U(MBLKL(mp), ==, size);
+
+		ASSERT3P(mp->b_cont, ==, NULL);
+		*(srfp->srf_mpp) = mp;
+		srfp->srf_mpp = &(mp->b_cont);
+
+		srfp->srf_len += size;
+
+		ASSERT(srfp->srf_next != NULL ||
+		    srp->sr_srfpp == &(srfp->srf_next));
+	}
+
+	DTRACE_PROBE2(flow_add, uint32_t, srfp->srf_tag, size_t, size);
+
+	/*
+	 * Try to align coalesced segments on push boundaries, unless they
+	 * are too frequent.
+	 */
+	if (sp->s_rx_coalesce_mode == SFXGE_RX_COALESCE_ALLOW_PUSH &&
+	    thp->th_flags & TH_PUSH)
+		sfxge_rx_qflow_complete(srp, srfp);
+
+	srfp->srf_lbolt = now;
+	return (B_TRUE);
+
+fail4:
+fail3:
+fail2:
+fail1:
+	sfxge_rx_qflow_complete(srp, srfp);
+
+	srfp->srf_lbolt = now;
+	return (B_FALSE);
+}
+
+void
+sfxge_rx_qpacket_coalesce(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	clock_t now;
+	mblk_t *mp;
+	sfxge_rx_flow_t *srfp;
+
+	ASSERT(sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF);
+
+	now = ddi_get_lbolt();
+
+	mp = srp->sr_mp;
+
+	srp->sr_mp = NULL;
+	srp->sr_mpp = &(srp->sr_mp);
+
+	/* Start with the last flow to be appended to */
+	srfp = *(srp->sr_srfpp);
+
+	while (mp != NULL) {
+		frtn_t *freep;
+		sfxge_rx_packet_t *srpp;
+		struct ether_header *etherhp;
+		struct ip *iphp;
+		struct tcphdr *thp;
+		size_t off;
+		size_t size;
+		uint16_t ether_tci;
+		uint16_t hash;
+		uint32_t tag;
+		mblk_t *next;
+
+		next = mp->b_next;
+		mp->b_next = NULL;
+
+		if (next != NULL)
+			prefetch_read_many(next);
+
+		freep = DB_FRTNP(mp);
+		/*LINTED*/
+		srpp = (sfxge_rx_packet_t *)(freep->free_arg);
+		ASSERT3P(srpp->srp_mp, ==, mp);
+
+		/* If the packet is not TCP then we cannot coalesce it */
+		if (~(srpp->srp_flags) & EFX_PKT_TCP)
+			goto reject;
+
+		/*
+		 * If the packet is not fully checksummed then we cannot
+		 * coalesce it.
+		 */
+		if (~(srpp->srp_flags) & (EFX_CKSUM_TCPUDP | EFX_CKSUM_IPV4))
+			goto reject;
+
+		/* Parse the TCP header */
+		sfxge_tcp_parse(mp, &etherhp, &iphp, &thp, &off,
+		    &size);
+		ASSERT(etherhp != NULL);
+		ASSERT(iphp != NULL);
+		ASSERT(thp != NULL);
+		ASSERT(off != 0);
+
+		if ((iphp->ip_off & ~htons(IP_DF)) != 0)
+			goto reject;
+
+		if (etherhp->ether_type == htons(ETHERTYPE_VLAN)) {
+			struct ether_vlan_header *ethervhp;
+
+			ethervhp = (struct ether_vlan_header *)etherhp;
+			ether_tci = ethervhp->ether_tci;
+		} else {
+			ether_tci = 0;
+		}
+
+		/*
+		 * Make sure any minimum length padding is stripped
+		 * before we try to add the packet to a flow.
+		 */
+		ASSERT3U(sp->s_rx_prefix_size + MBLKL(mp), ==,
+		    (size_t)(srpp->srp_size));
+		ASSERT3U(sp->s_rx_prefix_size + off + size, <=,
+		    (size_t)(srpp->srp_size));
+
+		if (sp->s_rx_prefix_size + off + size <
+		    (size_t)(srpp->srp_size))
+			mp->b_wptr = mp->b_rptr + off + size;
+
+		/*
+		 * If there is no current flow, or the segment does not match
+		 * the current flow then we must attempt to look up the
+		 * correct flow in the table.
+		 */
+		if (srfp == NULL)
+			goto lookup;
+
+		if (srfp->srf_saddr != iphp->ip_src.s_addr ||
+		    srfp->srf_daddr != iphp->ip_dst.s_addr)
+			goto lookup;
+
+		if (srfp->srf_sport != thp->th_sport ||
+		    srfp->srf_dport != thp->th_dport)
+			goto lookup;
+
+		if (srfp->srf_tci != ether_tci)
+			goto lookup;
+
+add:
+		ASSERT(srfp != NULL);
+
+		srpp->srp_etherhp = etherhp;
+		srpp->srp_iphp = iphp;
+		srpp->srp_thp = thp;
+		srpp->srp_off = off;
+
+		ASSERT3U(size, <, (1 << 16));
+		srpp->srp_size = (uint16_t)size;
+
+		/* Try to append the packet to the flow */
+		if (!sfxge_rx_qflow_add(srp, srfp, srpp, now))
+			goto reject;
+
+		mp = next;
+		continue;
+
+lookup:
+		/*
+		 * If there is a prefix area then read the hash from that,
+		 * otherwise calculate it.
+		 */
+		if (sp->s_rx_prefix_size != 0) {
+			hash = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_LFSR,
+			    DB_BASE(mp));
+		} else {
+			SFXGE_TCP_HASH(
+			    ntohl(iphp->ip_src.s_addr),
+			    ntohs(thp->th_sport),
+			    ntohl(iphp->ip_dst.s_addr),
+			    ntohs(thp->th_dport),
+			    hash);
+		}
+
+		srfp = &(srp->sr_flow[(hash >> 6) % SFXGE_MAX_FLOW]);
+		tag = (uint32_t)hash + 1; /* Make sure it's not zero */
+
+		/*
+		 * If the flow we have found does not match the hash then
+		 * it may be an unused flow, or it may be stale.
+		 */
+		if (tag != srfp->srf_tag) {
+			if (srfp->srf_count != 0) {
+				if (now - srfp->srf_lbolt <= srp->sr_rto)
+					goto reject;
+			}
+
+			if (srfp->srf_mp != NULL)
+				goto reject;
+
+			/* Start a new flow */
+			ASSERT(srfp->srf_next == NULL);
+
+			srfp->srf_tag = tag;
+
+			srfp->srf_saddr = iphp->ip_src.s_addr;
+			srfp->srf_daddr = iphp->ip_dst.s_addr;
+			srfp->srf_sport = thp->th_sport;
+			srfp->srf_dport = thp->th_dport;
+			srfp->srf_tci = ether_tci;
+
+			srfp->srf_count = 0;
+			srfp->srf_seq = ntohl(thp->th_seq);
+
+			srfp->srf_lbolt = now;
+			goto add;
+		}
+
+		/*
+		 * If the flow we have found does match the hash then it could
+		 * still be an alias.
+		 */
+		if (srfp->srf_saddr != iphp->ip_src.s_addr ||
+		    srfp->srf_daddr != iphp->ip_dst.s_addr)
+			goto reject;
+
+		if (srfp->srf_sport != thp->th_sport ||
+		    srfp->srf_dport != thp->th_dport)
+			goto reject;
+
+		if (srfp->srf_tci != ether_tci)
+			goto reject;
+
+		goto add;
+
+reject:
+		*(srp->sr_mpp) = mp;
+		srp->sr_mpp = &(mp->b_next);
+
+		mp = next;
+	}
+}
+
+void
+sfxge_rx_qcomplete(sfxge_rxq_t *srp, boolean_t eop)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+	unsigned int completed;
+	sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
+	unsigned int level;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	ASSERT(srp->sr_mp == NULL);
+	ASSERT(srp->sr_mpp == &(srp->sr_mp));
+
+	completed = srp->sr_completed;
+	while (completed != srp->sr_pending) {
+		unsigned int id;
+		sfxge_rx_packet_t *srpp;
+		mblk_t *mp;
+		size_t size;
+		uint16_t flags;
+
+		id = completed++ & (sp->s_rxq_size - 1);
+
+		if (srp->sr_pending - completed >= 4) {
+			unsigned int prefetch;
+
+			prefetch = (id + 4) & (sp->s_rxq_size - 1);
+
+			srpp = srp->sr_srpp[prefetch];
+			ASSERT(srpp != NULL);
+
+			mp = srpp->srp_mp;
+			prefetch_read_many(mp->b_datap);
+		} else if (completed == srp->sr_pending) {
+			prefetch_read_many(srp->sr_mp);
+		}
+
+		srpp = srp->sr_srpp[id];
+		ASSERT(srpp != NULL);
+
+		srp->sr_srpp[id] = NULL;
+
+		mp = srpp->srp_mp;
+		ASSERT(mp->b_cont == NULL);
+
+		/* when called from sfxge_rx_qstop() */
+		if (srp->sr_state != SFXGE_RXQ_STARTED)
+			goto discard;
+
+		if (srpp->srp_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
+			goto discard;
+
+		/* Set up the packet length */
+		ASSERT3P(mp->b_rptr, ==, DB_BASE(mp));
+		mp->b_rptr += sp->s_rx_prefix_size;
+
+		prefetch_read_many(mp->b_rptr);
+
+		ASSERT3P(mp->b_wptr, ==, DB_BASE(mp));
+		mp->b_wptr += (size_t)(srpp->srp_size);
+		ASSERT3P(mp->b_wptr, <=, DB_LIM(mp));
+
+		/* Calculate the maximum packet size */
+		size = sp->s_mtu;
+		size += (srpp->srp_flags & EFX_PKT_VLAN_TAGGED) ?
+		    sizeof (struct ether_vlan_header) :
+		    sizeof (struct ether_header);
+
+		if (MBLKL(mp) > size)
+			goto discard;
+
+		/* Make the data visible to the kernel */
+		(void) ddi_dma_sync(srpp->srp_dma_handle, 0,
+		    (size_t)(srpp->srp_size), DDI_DMA_SYNC_FORKERNEL);
+
+		/* Check for loopback packets */
+		if (!(srpp->srp_flags & EFX_PKT_IPV4) &&
+		    !(srpp->srp_flags & EFX_PKT_IPV6)) {
+			struct ether_header *etherhp;
+
+			/*LINTED*/
+			etherhp = (struct ether_header *)(mp->b_rptr);
+
+			if (etherhp->ether_type ==
+			    htons(SFXGE_ETHERTYPE_LOOPBACK)) {
+				DTRACE_PROBE(loopback);
+
+				srp->sr_loopback++;
+				goto discard;
+			}
+		}
+
+		/* Set up the checksum information */
+		flags = 0;
+
+		if (srpp->srp_flags & EFX_CKSUM_IPV4) {
+			ASSERT(srpp->srp_flags & EFX_PKT_IPV4);
+			flags |= HCK_IPV4_HDRCKSUM;
+		}
+
+		if (srpp->srp_flags & EFX_CKSUM_TCPUDP) {
+			ASSERT(srpp->srp_flags & EFX_PKT_TCP ||
+			    srpp->srp_flags & EFX_PKT_UDP);
+			flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
+		}
+
+		DB_CKSUMSTART(mp) = 0;
+		DB_CKSUMSTUFF(mp) = 0;
+		DB_CKSUMEND(mp) = 0;
+		DB_CKSUMFLAGS(mp) = flags;
+		DB_CKSUM16(mp) = 0;
+
+		/* Add the packet to the tail of the chain */
+		srfppp->srfpp_loaned++;
+
+		ASSERT(mp->b_next == NULL);
+		*(srp->sr_mpp) = mp;
+		srp->sr_mpp = &(mp->b_next);
+
+		continue;
+
+discard:
+		/* Return the packet to the pool */
+		srfppp->srfpp_loaned++;
+		freeb(mp); /* Equivalent to freemsg() as b_cont==0 */
+	}
+	srp->sr_completed = completed;
+
+	/* Attempt to coalesce any TCP packets */
+	if (sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF)
+		sfxge_rx_qpacket_coalesce(srp);
+
+	/*
+	 * If there are any pending flows and this is the end of the
+	 * poll then they must be completed.
+	 */
+	if (srp->sr_srfp != NULL && eop) {
+		sfxge_rx_flow_t *srfp;
+
+		srfp = srp->sr_srfp;
+
+		srp->sr_srfp = NULL;
+		srp->sr_srfpp = &(srp->sr_srfp);
+
+		do {
+			sfxge_rx_flow_t *next;
+
+			next = srfp->srf_next;
+			srfp->srf_next = NULL;
+
+			sfxge_rx_qflow_complete(srp, srfp);
+
+			srfp = next;
+		} while (srfp != NULL);
+	}
+
+	level = srp->sr_added - srp->sr_completed;
+
+	/* If there are any packets then pass them up the stack */
+	if (srp->sr_mp != NULL) {
+		mblk_t *mp;
+
+		mp = srp->sr_mp;
+
+		srp->sr_mp = NULL;
+		srp->sr_mpp = &(srp->sr_mp);
+
+		if (level == 0) {
+			/* Try to refill ASAP */
+			sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
+			level = srp->sr_added - srp->sr_completed;
+		}
+
+		/*
+		 * If the RXQ is still empty, discard and recycle the
+		 * current entry to ensure that the ring always
+		 * contains at least one descriptor. This ensures that
+		 * the next hardware RX will trigger an event
+		 * (possibly delayed by interrupt moderation) and
+		 * trigger another refill/fill attempt.
+		 *
+		 * Note this drops a complete LRO fragment from the
+		 * start of the batch.
+		 *
+		 * Note also that copymsgchain() does not help with
+		 * resource starvation here, unless we are short of DMA
+		 * mappings.
+		 */
+		if (level == 0) {
+			mblk_t *nmp;
+
+			srp->sr_kstat.srk_rxq_empty_discard++;
+			DTRACE_PROBE1(rxq_empty_discard, int, index);
+			nmp = mp->b_next;
+			if (nmp)
+				sfxge_gld_rx_post(sp, index, nmp);
+			/* as level==0 will swizzle,rxpost below */
+			freemsg(mp);
+		} else {
+			sfxge_gld_rx_post(sp, index, mp);
+		}
+	}
+
+	/* Top up the queue if necessary */
+	if (level < srp->sr_hiwat) {
+		sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
+
+		level = srp->sr_added - srp->sr_completed;
+		if (level < srp->sr_lowat)
+			sfxge_rx_qfill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
+	}
+}
+
+static unsigned int
+sfxge_rx_qloopback(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	sfxge_rxq_t *srp;
+	unsigned int count;
+
+	mutex_enter(&(sep->se_lock));
+	srp = sp->s_srp[index];
+	count = srp->sr_loopback;
+	srp->sr_loopback = 0;
+	mutex_exit(&(sep->se_lock));
+
+	return (count);
+}
+
+void
+sfxge_rx_qflush_done(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* SFCbug22989: events may be delayed. EVQs are stopped after RXQs */
+	if ((srp->sr_state != SFXGE_RXQ_INITIALIZED) ||
+	    (srp->sr_flush == SFXGE_FLUSH_DONE))
+		return;
+
+	/* Flush successful: wakeup sfxge_rx_qstop() */
+	srp->sr_flush = SFXGE_FLUSH_DONE;
+	cv_broadcast(&(srp->sr_flush_kv));
+}
+
+void
+sfxge_rx_qflush_failed(sfxge_rxq_t *srp)
+{
+	sfxge_t *sp = srp->sr_sp;
+	unsigned int index = srp->sr_index;
+	sfxge_evq_t *sep = sp->s_sep[index];
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* SFCbug22989: events may be delayed. EVQs are stopped after RXQs */
+	if ((srp->sr_state != SFXGE_RXQ_INITIALIZED) ||
+	    (srp->sr_flush == SFXGE_FLUSH_DONE))
+		return;
+
+	/* SFCbug22989: events may be delayed. EVQs are stopped after RXQs */
+	if (srp->sr_state != SFXGE_RXQ_STARTED)
+		return;
+
+	/* Flush failed, so retry until timeout in sfxge_rx_qstop() */
+	srp->sr_flush = SFXGE_FLUSH_FAILED;
+	efx_rx_qflush(srp->sr_erp);
+}
+
+static void
+sfxge_rx_qstop(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	sfxge_rxq_t *srp;
+	clock_t timeout;
+
+	mutex_enter(&(sep->se_lock));
+
+	srp = sp->s_srp[index];
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
+
+	sfxge_rx_qpoll_stop(srp);
+
+	srp->sr_state = SFXGE_RXQ_INITIALIZED;
+
+	if (sp->s_hw_err == SFXGE_HW_OK) {
+		/* Wait upto 2sec for queue flushing to complete */
+		srp->sr_flush = SFXGE_FLUSH_PENDING;
+		efx_rx_qflush(srp->sr_erp);
+	} else {
+		/* Do not attempt flush if indication of H/W failure */
+		srp->sr_flush = SFXGE_FLUSH_DONE;
+	}
+
+	timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_RX_QFLUSH_USEC);
+
+	while (srp->sr_flush != SFXGE_FLUSH_DONE) {
+		if (cv_timedwait(&(srp->sr_flush_kv), &(sep->se_lock),
+			timeout) < 0) {
+			/* Timeout waiting for successful flush */
+			dev_info_t *dip = sp->s_dip;
+
+				ddi_driver_name(sp->s_dip),
+			cmn_err(CE_NOTE,
+			    SFXGE_CMN_ERR "[%s%d] rxq[%d] flush timeout",
+			    ddi_driver_name(dip), ddi_get_instance(dip), index);
+			break;
+		}
+	}
+
+	DTRACE_PROBE1(flush, sfxge_flush_state_t, srp->sr_flush);
+	srp->sr_flush = SFXGE_FLUSH_DONE;
+
+	/* Destroy the receive queue */
+	efx_rx_qdestroy(srp->sr_erp);
+	srp->sr_erp = NULL;
+
+	/* Clear entries from the buffer table */
+	sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
+	    EFX_RXQ_NBUFS(sp->s_rxq_size));
+
+	/*
+	 * Free any unused RX packets which had descriptors on the RXQ
+	 * Packets will be discard as state != STARTED
+	 */
+	srp->sr_pending = srp->sr_added;
+	sfxge_rx_qcomplete(srp, B_TRUE);
+
+	ASSERT3U(srp->sr_completed, ==, srp->sr_pending);
+
+	srp->sr_added = 0;
+	srp->sr_pending = 0;
+	srp->sr_completed = 0;
+	srp->sr_loopback = 0;
+
+	srp->sr_lowat = 0;
+	srp->sr_hiwat = 0;
+
+	mutex_exit(&(sep->se_lock));
+}
+
+static void
+sfxge_rx_kstat_fini(sfxge_rxq_t *srp)
+{
+	kstat_delete(srp->sr_ksp);
+	srp->sr_ksp = NULL;
+}
+
+static void
+sfxge_rx_qfini(sfxge_t *sp, unsigned int index)
+{
+	sfxge_rxq_t *srp = sp->s_srp[index];
+
+	ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
+
+	sp->s_srp[index] = NULL;
+	srp->sr_state = SFXGE_RXQ_UNINITIALIZED;
+
+	sfxge_rx_kstat_fini(srp);
+
+	/* Empty the pool */
+	sfxge_rx_qfpp_empty(srp);
+
+	srp->sr_index = 0;
+
+	kmem_cache_free(sp->s_rqc, srp);
+}
+
+static int
+sfxge_rx_scale_kstat_update(kstat_t *ksp, int rw)
+{
+	sfxge_t *sp = ksp->ks_private;
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	sfxge_intr_t *sip = &(sp->s_intr);
+	kstat_named_t *knp;
+	unsigned int index;
+	unsigned int entry;
+	unsigned int *freq;
+	int rc;
+
+	ASSERT(mutex_owned(&(srsp->srs_lock)));
+
+	if (rw != KSTAT_READ) {
+		rc = EACCES;
+		goto fail1;
+	}
+
+	if ((freq = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
+				KM_NOSLEEP)) == NULL) {
+		rc = ENOMEM;
+		goto fail2;
+	}
+
+	for (index = 0; index < sip->si_nalloc; index++)
+		freq[index] = 0;
+
+	for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
+		index = srsp->srs_tbl[entry];
+
+		freq[index]++;
+	}
+
+	knp = ksp->ks_data;
+	for (index = 0; index < sip->si_nalloc; index++) {
+		knp->value.ui64 = freq[index];
+		knp++;
+	}
+
+	knp->value.ui64 = srsp->srs_count;
+
+	kmem_free(freq, sizeof (unsigned int) * sip->si_nalloc);
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+static int
+sfxge_rx_scale_kstat_init(sfxge_t *sp)
+{
+	dev_info_t *dip = sp->s_dip;
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	sfxge_intr_t *sip = &(sp->s_intr);
+	char name[MAXNAMELEN];
+	kstat_t *ksp;
+	kstat_named_t *knp;
+	unsigned int index;
+	int rc;
+
+	/* Create the set */
+	(void) snprintf(name, MAXNAMELEN - 1, "%s_rss", ddi_driver_name(dip));
+
+	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
+	    ddi_get_instance(dip), name, "rss", KSTAT_TYPE_NAMED,
+	    sip->si_nalloc + 1, 0)) == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	srsp->srs_ksp = ksp;
+
+	ksp->ks_update = sfxge_rx_scale_kstat_update;
+	ksp->ks_private = sp;
+	ksp->ks_lock = &(srsp->srs_lock);
+
+	/* Initialise the named stats */
+	knp = ksp->ks_data;
+	for (index = 0; index < sip->si_nalloc; index++) {
+		char name[MAXNAMELEN];
+
+		(void) snprintf(name, MAXNAMELEN - 1, "evq%04d_count", index);
+		kstat_named_init(knp, name, KSTAT_DATA_UINT64);
+		knp++;
+	}
+
+	kstat_named_init(knp, "scale", KSTAT_DATA_UINT64);
+
+	kstat_install(ksp);
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static void
+sfxge_rx_scale_kstat_fini(sfxge_t *sp)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+
+	/* Destroy the set */
+	kstat_delete(srsp->srs_ksp);
+	srsp->srs_ksp = NULL;
+}
+
+
+unsigned int
+sfxge_rx_scale_prop_get(sfxge_t *sp)
+{
+	int rx_scale;
+
+	rx_scale = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+				    DDI_PROP_DONTPASS, "rx_scale_count",
+				    SFXGE_RX_SCALE_MAX);
+	/* 0 and all -ve numbers sets to number of logical CPUs */
+	if (rx_scale <= 0)
+		rx_scale = ncpus;
+
+	return (rx_scale);
+}
+
+
+static int
+sfxge_rx_scale_init(sfxge_t *sp)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	sfxge_intr_t *sip = &(sp->s_intr);
+	int rc;
+
+	ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_UNINITIALIZED);
+
+	/* Create tables for CPU, core, cache and chip counts */
+	srsp->srs_cpu = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
+#ifdef	_USE_CPU_PHYSID
+	srsp->srs_core = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
+	srsp->srs_cache = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
+	srsp->srs_chip = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
+#endif
+
+	mutex_init(&(srsp->srs_lock), NULL, MUTEX_DRIVER, NULL);
+
+	/* We need at least one event queue */
+	srsp->srs_count = sfxge_rx_scale_prop_get(sp);
+	if (srsp->srs_count > sip->si_nalloc)
+		srsp->srs_count = sip->si_nalloc;
+	if (srsp->srs_count < 1)
+		srsp->srs_count = 1;
+
+	/* Set up the kstats */
+	if ((rc = sfxge_rx_scale_kstat_init(sp)) != 0)
+		goto fail1;
+
+	srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+	mutex_destroy(&(srsp->srs_lock));
+
+	return (rc);
+}
+
+void
+sfxge_rx_scale_update(void *arg)
+{
+	sfxge_t *sp = arg;
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	sfxge_intr_t *sip;
+	processorid_t id;
+	unsigned int count;
+	unsigned int *tbl;
+	unsigned int *rating;
+	unsigned int entry;
+	int rc;
+
+	mutex_enter(&(srsp->srs_lock));
+
+	if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
+		rc = EFAULT;
+		goto fail1;
+	}
+
+	if ((tbl =  kmem_zalloc(sizeof (unsigned int) * SFXGE_RX_SCALE_MAX,
+			    KM_NOSLEEP)) == NULL) {
+		rc = ENOMEM;
+		goto fail2;
+	}
+
+	sip = &(sp->s_intr);
+	if ((rating = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
+			    KM_NOSLEEP)) == NULL) {
+		rc = ENOMEM;
+		goto fail3;
+	}
+
+	mutex_enter(&cpu_lock);
+
+	/*
+	 * Substract any current CPU, core, cache and chip usage from the
+	 * global contention tables.
+	 */
+	for (id = 0; id < NCPU; id++) {
+		ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
+		sfxge_cpu[id] -= srsp->srs_cpu[id];
+		srsp->srs_cpu[id] = 0;
+
+#ifdef	_USE_CPU_PHYSID
+		ASSERT3U(sfxge_core[id], >=, srsp->srs_core[id]);
+		sfxge_core[id] -= srsp->srs_core[id];
+		srsp->srs_core[id] = 0;
+
+		ASSERT3U(sfxge_cache[id], >=, srsp->srs_cache[id]);
+		sfxge_cache[id] -= srsp->srs_cache[id];
+		srsp->srs_cache[id] = 0;
+
+		ASSERT3U(sfxge_chip[id], >=, srsp->srs_chip[id]);
+		sfxge_chip[id] -= srsp->srs_chip[id];
+		srsp->srs_chip[id] = 0;
+#endif
+	}
+
+	ASSERT(srsp->srs_count != 0);
+
+	/* Choose as many event queues as we need */
+	for (count = 0; count < srsp->srs_count; count++) {
+		unsigned int index;
+		sfxge_evq_t *sep;
+		unsigned int choice;
+		unsigned int choice_rating;
+
+		bzero(rating, sizeof (unsigned int) * sip->si_nalloc);
+
+		/*
+		 * Rate each event queue on its global level of CPU
+		 * contention.
+		 */
+		for (index = 0; index < sip->si_nalloc; index++) {
+			sep = sp->s_sep[index];
+
+			id = sep->se_cpu_id;
+			rating[index] += sfxge_cpu[id];
+
+#ifdef	_USE_CPU_PHYSID
+			id = sep->se_core_id;
+			rating[index] += sfxge_core[id];
+
+			id = sep->se_cache_id;
+			rating[index] += sfxge_cache[id];
+
+			id = sep->se_chip_id;
+			rating[index] += sfxge_chip[id];
+#endif
+		}
+
+		/* Choose the queue with the lowest CPU contention */
+		choice = 0;
+		choice_rating = rating[0];
+
+		for (index = 1; index < sip->si_nalloc; index++) {
+			if (rating[index] < choice_rating) {
+				choice = index;
+				choice_rating = rating[index];
+			}
+		}
+
+		/* Add our choice to the condensed RSS table */
+		tbl[count] = choice;
+
+		/* Add information to the global contention tables */
+		sep = sp->s_sep[choice];
+
+		id = sep->se_cpu_id;
+		srsp->srs_cpu[id]++;
+		sfxge_cpu[id]++;
+
+#ifdef	_USE_CPU_PHYSID
+		id = sep->se_core_id;
+		srsp->srs_core[id]++;
+		sfxge_core[id]++;
+
+		id = sep->se_cache_id;
+		srsp->srs_cache[id]++;
+		sfxge_cache[id]++;
+
+		id = sep->se_chip_id;
+		srsp->srs_chip[id]++;
+		sfxge_chip[id]++;
+#endif
+	}
+
+	mutex_exit(&cpu_lock);
+
+	/* Build the expanded RSS table */
+	count = 0;
+	for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
+		unsigned int index;
+
+		index = tbl[count];
+		count = (count + 1) % srsp->srs_count;
+
+		srsp->srs_tbl[entry] = index;
+	}
+
+	/* Program the expanded RSS table into the hardware */
+	(void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
+	    SFXGE_RX_SCALE_MAX);
+
+	mutex_exit(&(srsp->srs_lock));
+	kmem_free(rating, sizeof (unsigned int) * sip->si_nalloc);
+	kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
+	return;
+
+fail3:
+	DTRACE_PROBE(fail3);
+	kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(srsp->srs_lock));
+}
+
+static int
+sfxge_rx_scale_start(sfxge_t *sp)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	const efx_nic_cfg_t *encp;
+	int rc;
+
+	mutex_enter(&(srsp->srs_lock));
+
+	ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
+
+	/* Clear down the RSS table */
+	bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
+
+	(void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
+	    SFXGE_RX_SCALE_MAX);
+
+	/* Make sure the LFSR hash is selected */
+	encp = efx_nic_cfg_get(sp->s_enp);
+	if ((rc = efx_rx_scale_mode_set(sp->s_enp, EFX_RX_HASHALG_LFSR, 0,
+	    (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT))) != 0)
+		goto fail1;
+
+	srsp->srs_state = SFXGE_RX_SCALE_STARTED;
+
+	mutex_exit(&(srsp->srs_lock));
+
+	/* sfxge_t->s_state_lock held */
+	(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
+	    DDI_SLEEP);
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(srsp->srs_lock));
+
+	return (rc);
+}
+
+int
+sfxge_rx_scale_count_get(sfxge_t *sp, unsigned int *countp)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	int rc;
+
+	mutex_enter(&(srsp->srs_lock));
+
+	if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
+	    srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	*countp = srsp->srs_count;
+
+	mutex_exit(&(srsp->srs_lock));
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(srsp->srs_lock));
+
+	return (rc);
+}
+
+int
+sfxge_rx_scale_count_set(sfxge_t *sp, unsigned int count)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	sfxge_intr_t *sip = &(sp->s_intr);
+	int dispatch = 1;
+	int rc;
+
+	if (count < 1 || count > sip->si_nalloc) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	mutex_enter(&(srsp->srs_lock));
+
+	if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
+	    srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
+		rc = ENOTSUP;
+		goto fail2;
+	}
+
+	srsp->srs_count = count;
+
+	if (srsp->srs_state != SFXGE_RX_SCALE_STARTED)
+		dispatch = 0;
+
+	mutex_exit(&(srsp->srs_lock));
+
+	if (dispatch)
+		/* no locks held */
+		(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
+		    DDI_SLEEP);
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	mutex_exit(&(srsp->srs_lock));
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static void
+sfxge_rx_scale_stop(sfxge_t *sp)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+	processorid_t id;
+
+	mutex_enter(&(srsp->srs_lock));
+
+	ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_STARTED);
+
+	srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
+
+	mutex_enter(&cpu_lock);
+
+	/*
+	 * Substract any current CPU, core, cache and chip usage from the
+	 * global contention tables.
+	 */
+	for (id = 0; id < NCPU; id++) {
+		ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
+		sfxge_cpu[id] -= srsp->srs_cpu[id];
+		srsp->srs_cpu[id] = 0;
+
+#ifdef	_USE_CPU_PHYSID
+		ASSERT3U(sfxge_core[id], >=, srsp->srs_core[id]);
+		sfxge_core[id] -= srsp->srs_core[id];
+		srsp->srs_core[id] = 0;
+
+		ASSERT3U(sfxge_cache[id], >=, srsp->srs_cache[id]);
+		sfxge_cache[id] -= srsp->srs_cache[id];
+		srsp->srs_cache[id] = 0;
+
+		ASSERT3U(sfxge_chip[id], >=, srsp->srs_chip[id]);
+		sfxge_chip[id] -= srsp->srs_chip[id];
+		srsp->srs_chip[id] = 0;
+#endif
+	}
+
+	mutex_exit(&cpu_lock);
+
+	/* Clear down the RSS table */
+	bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
+
+	(void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
+	    SFXGE_RX_SCALE_MAX);
+
+	mutex_exit(&(srsp->srs_lock));
+}
+
+static void
+sfxge_rx_scale_fini(sfxge_t *sp)
+{
+	sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+
+	ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
+
+	srsp->srs_state = SFXGE_RX_SCALE_UNINITIALIZED;
+
+	/* Tear down the kstats */
+	sfxge_rx_scale_kstat_fini(sp);
+
+	srsp->srs_count = 0;
+
+	mutex_destroy(&(srsp->srs_lock));
+
+	/* Destroy tables */
+#ifdef	_USE_CPU_PHYSID
+	kmem_free(srsp->srs_chip, sizeof (unsigned int) * NCPU);
+	srsp->srs_chip = NULL;
+
+	kmem_free(srsp->srs_cache, sizeof (unsigned int) * NCPU);
+	srsp->srs_cache = NULL;
+
+	kmem_free(srsp->srs_core, sizeof (unsigned int) * NCPU);
+	srsp->srs_core = NULL;
+#endif
+	kmem_free(srsp->srs_cpu, sizeof (unsigned int) * NCPU);
+	srsp->srs_cpu = NULL;
+}
+
+int
+sfxge_rx_init(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	const efx_nic_cfg_t *encp;
+	char name[MAXNAMELEN];
+	int index;
+	int rc;
+
+	if (sip->si_state == SFXGE_INTR_UNINITIALIZED) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	encp = efx_nic_cfg_get(sp->s_enp);
+	if ((rc = sfxge_rx_scale_init(sp)) != 0)
+		goto fail2;
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_rx_packet_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
+
+	sp->s_rpc = kmem_cache_create(name, sizeof (sfxge_rx_packet_t),
+	    SFXGE_CPU_CACHE_SIZE, sfxge_rx_packet_ctor, sfxge_rx_packet_dtor,
+	    NULL, sp, NULL, 0);
+	ASSERT(sp->s_rpc != NULL);
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_rxq_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
+
+	sp->s_rqc = kmem_cache_create(name, sizeof (sfxge_rxq_t),
+	    SFXGE_CPU_CACHE_SIZE, sfxge_rx_qctor, sfxge_rx_qdtor, NULL, sp,
+	    NULL, 0);
+	ASSERT(sp->s_rqc != NULL);
+
+	sp->s_rx_pkt_mem_max = ddi_prop_get_int64(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "rx_pkt_mem_max", 0); /* disabled */
+
+	/* Initialize the receive queue(s) */
+	for (index = 0; index < sip->si_nalloc; index++) {
+		if ((rc = sfxge_rx_qinit(sp, index)) != 0)
+			goto fail3;
+	}
+
+	sp->s_rx_coalesce_mode = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "rx_coalesce_mode", SFXGE_RX_COALESCE_OFF);
+
+	return (0);
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	/* Tear down the receive queue(s) */
+	while (--index >= 0)
+		sfxge_rx_qfini(sp, index);
+
+	kmem_cache_destroy(sp->s_rqc);
+	sp->s_rqc = NULL;
+
+	kmem_cache_destroy(sp->s_rpc);
+	sp->s_rpc = NULL;
+
+	sfxge_rx_scale_fini(sp);
+
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+int
+sfxge_rx_start(sfxge_t *sp)
+{
+	sfxge_mac_t *smp = &(sp->s_mac);
+	sfxge_intr_t *sip;
+	const efx_nic_cfg_t *encp;
+	int index;
+	int rc;
+
+	mutex_enter(&(smp->sm_lock));
+
+	/* Calculate the receive packet buffer size and alignment */
+	sp->s_rx_buffer_size = EFX_MAC_PDU(sp->s_mtu);
+
+	encp = efx_nic_cfg_get(sp->s_enp);
+	if (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT) {
+		size_t align;
+
+		sp->s_rx_prefix_size = EFX_RX_PREFIX_SIZE;
+
+		/*
+		 * Place the start of the buffer a prefix length minus 2
+		 * before the start of a cache line. This ensures that the
+		 * last two bytes of the prefix (which is where the LFSR hash
+		 * is located) are in the same cache line as the headers, and
+		 * the IP header is 32-bit aligned.
+		 */
+		align = SFXGE_CPU_CACHE_SIZE + SFXGE_IP_ALIGN -
+		    EFX_RX_PREFIX_SIZE;
+
+		sp->s_rx_buffer_align = align;
+		sp->s_rx_buffer_size += align;
+	} else {
+		sp->s_rx_prefix_size = 0;
+
+		/*
+		 * Place the start of the buffer 2 bytes after a cache line
+		 * boundary so that the headers fit into the cache line and
+		 * the IP header is 32-bit aligned.
+		 */
+
+		sp->s_rx_buffer_align = SFXGE_IP_ALIGN;
+		sp->s_rx_buffer_size += SFXGE_IP_ALIGN;
+	}
+
+	/* Initialize the receive module */
+	if ((rc = efx_rx_init(sp->s_enp)) != 0)
+		goto fail1;
+
+	mutex_exit(&(smp->sm_lock));
+
+	if ((rc = sfxge_rx_scale_start(sp)) != 0)
+		goto fail2;
+
+	/* Start the receive queue(s) */
+	sip = &(sp->s_intr);
+	for (index = 0; index < sip->si_nalloc; index++) {
+		if ((rc = sfxge_rx_qstart(sp, index)) != 0)
+			goto fail3;
+	}
+
+	return (0);
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	/* Stop the receive queue(s) */
+	while (--index >= 0)
+		sfxge_rx_qstop(sp, index);
+
+	sfxge_rx_scale_stop(sp);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	mutex_enter(&(smp->sm_lock));
+
+	/* Tear down the receive module */
+	efx_rx_fini(sp->s_enp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(smp->sm_lock));
+
+	return (rc);
+}
+
+void
+sfxge_rx_coalesce_mode_get(sfxge_t *sp, sfxge_rx_coalesce_mode_t *modep)
+{
+	*modep = sp->s_rx_coalesce_mode;
+}
+
+int
+sfxge_rx_coalesce_mode_set(sfxge_t *sp, sfxge_rx_coalesce_mode_t mode)
+{
+	int rc;
+
+	switch (mode) {
+	case SFXGE_RX_COALESCE_OFF:
+	case SFXGE_RX_COALESCE_DISALLOW_PUSH:
+	case SFXGE_RX_COALESCE_ALLOW_PUSH:
+		break;
+
+	default:
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	sp->s_rx_coalesce_mode = mode;
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+void
+sfxge_rx_loopback(sfxge_t *sp, unsigned int *countp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	int index;
+
+	*countp = 0;
+	for (index = 0; index < sip->si_nalloc; index++)
+		*countp += sfxge_rx_qloopback(sp, index);
+}
+
+int
+sfxge_rx_ioctl(sfxge_t *sp, sfxge_rx_ioc_t *srip)
+{
+	int rc;
+
+	switch (srip->sri_op) {
+	case SFXGE_RX_OP_LOOPBACK: {
+		unsigned int count;
+
+		sfxge_rx_loopback(sp, &count);
+
+		srip->sri_data = count;
+
+		break;
+	}
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+void
+sfxge_rx_stop(sfxge_t *sp)
+{
+	sfxge_mac_t *smp = &(sp->s_mac);
+	sfxge_intr_t *sip = &(sp->s_intr);
+	efx_nic_t *enp = sp->s_enp;
+	const efx_nic_cfg_t *encp;
+	int index;
+
+	/* Stop the receive queue(s) */
+	index = sip->si_nalloc;
+	while (--index >= 0)
+		/* TBD: Flush RXQs in parallel; HW has limit + may need retry */
+		sfxge_rx_qstop(sp, index);
+
+	encp = efx_nic_cfg_get(sp->s_enp);
+	sfxge_rx_scale_stop(sp);
+
+	mutex_enter(&(smp->sm_lock));
+
+	/* Tear down the receive module */
+	efx_rx_fini(enp);
+
+	sp->s_rx_buffer_align = 0;
+	sp->s_rx_prefix_size = 0;
+	sp->s_rx_buffer_size = 0;
+
+	mutex_exit(&(smp->sm_lock));
+}
+
+unsigned int
+sfxge_rx_loaned(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	int index;
+	unsigned int loaned;
+
+	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
+
+	loaned = 0;
+	for (index = 0; index < sip->si_nalloc; index++) {
+		sfxge_rxq_t *srp = sp->s_srp[index];
+		sfxge_evq_t *sep = sp->s_sep[srp->sr_index];
+
+		mutex_enter(&(sep->se_lock));
+
+		loaned += sfxge_rx_qfpp_swizzle(srp);
+
+		mutex_exit(&(sep->se_lock));
+	}
+
+	return (loaned);
+}
+
+void
+sfxge_rx_fini(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	const efx_nic_cfg_t *encp;
+	int index;
+
+	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
+
+	sp->s_rx_coalesce_mode = SFXGE_RX_COALESCE_OFF;
+
+	/* Tear down the receive queue(s) */
+	index = sip->si_nalloc;
+	while (--index >= 0)
+		sfxge_rx_qfini(sp, index);
+
+	ASSERT3U(sp->s_rx_pkt_mem_alloc, ==, 0);
+
+	kmem_cache_destroy(sp->s_rqc);
+	sp->s_rqc = NULL;
+
+	kmem_cache_destroy(sp->s_rpc);
+	sp->s_rpc = NULL;
+
+	encp = efx_nic_cfg_get(sp->s_enp);
+	sfxge_rx_scale_fini(sp);
+}
