--- /dev/null	2013-08-22 18:57:35.000000000 -0400
+++ new/usr/src/uts/common/io/sfxge/sfxge_tx.c	2013-08-22 18:57:35.010504816 -0400
@@ -0,0 +1,3041 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2008-2013 Solarflare Communications Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/types.h>
+#include <sys/sysmacros.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/atomic.h>
+#include <sys/stream.h>
+#include <sys/strsun.h>
+#include <sys/strsubr.h>
+#include <sys/pattr.h>
+#include <sys/cpu.h>
+
+#include <sys/ethernet.h>
+#include <inet/ip.h>
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+
+#include "sfxge.h"
+
+#include "efx.h"
+
+/* TXQ flush response timeout (in microseconds) */
+#define	SFXGE_TX_QFLUSH_USEC	(2000000)
+#define	EVQ_0 0
+
+/* See sfxge.conf.private for descriptions */
+#define	SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT 4096
+#define	SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT 256
+
+
+/* Transmit buffer DMA attributes */
+static ddi_device_acc_attr_t sfxge_tx_buffer_devacc = {
+
+	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
+	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
+	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
+};
+
+static ddi_dma_attr_t sfxge_tx_buffer_dma_attr = {
+	DMA_ATTR_V0,		/* dma_attr_version	*/
+	0,			/* dma_attr_addr_lo	*/
+	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
+	0xffffffffffffffffull,	/* dma_attr_count_max	*/
+	SFXGE_TX_BUFFER_SIZE,	/* dma_attr_align	*/
+	0xffffffff,		/* dma_attr_burstsizes	*/
+	1,			/* dma_attr_minxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_seg		*/
+	1,			/* dma_attr_sgllen	*/
+	1,			/* dma_attr_granular	*/
+	0			/* dma_attr_flags	*/
+};
+
+/* Transmit mapping DMA attributes */
+static ddi_dma_attr_t sfxge_tx_mapping_dma_attr = {
+	DMA_ATTR_V0,		/* dma_attr_version	*/
+	0,			/* dma_attr_addr_lo	*/
+	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
+	0xffffffffffffffffull,	/* dma_attr_count_max	*/
+	1,			/* dma_attr_align	*/
+	0xffffffff,		/* dma_attr_burstsizes	*/
+	1,			/* dma_attr_minxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_seg		*/
+	0x7fffffff,		/* dma_attr_sgllen	*/
+	1,			/* dma_attr_granular	*/
+	0			/* dma_attr_flags	*/
+};
+
+/* Transmit queue DMA attributes */
+static ddi_device_acc_attr_t sfxge_txq_devacc = {
+
+	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
+	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
+	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
+};
+
+static ddi_dma_attr_t sfxge_txq_dma_attr = {
+	DMA_ATTR_V0,		/* dma_attr_version	*/
+	0,			/* dma_attr_addr_lo	*/
+	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
+	0xffffffffffffffffull,	/* dma_attr_count_max	*/
+	EFX_BUF_SIZE,		/* dma_attr_align	*/
+	0xffffffff,		/* dma_attr_burstsizes	*/
+	1,			/* dma_attr_minxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_seg		*/
+	1,			/* dma_attr_sgllen	*/
+	1,			/* dma_attr_granular	*/
+	0			/* dma_attr_flags	*/
+};
+
+
+/*
+ * A sfxge_tx_qdpl_swizzle() can happen when the DPL get list is one packet
+ * under the limit, and must move all packets from the DPL put->get list
+ * Hence this is the real maximum length of the TX DPL get list.
+ */
+static int
+sfxge_tx_dpl_get_pkt_max(sfxge_txq_t *stp)
+{
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+	return (stdp->get_pkt_limit + stdp->put_pkt_limit - 1);
+}
+
+
+static int
+sfxge_tx_packet_ctor(void *buf, void *arg, int kmflags)
+{
+	_NOTE(ARGUNUSED(arg, kmflags))
+
+	bzero(buf, sizeof (sfxge_tx_packet_t));
+
+	return (0);
+}
+
+static void
+sfxge_tx_packet_dtor(void *buf, void *arg)
+{
+	sfxge_tx_packet_t *stpp = buf;
+
+	_NOTE(ARGUNUSED(arg))
+
+	SFXGE_OBJ_CHECK(stpp, sfxge_tx_packet_t);
+}
+
+static int
+sfxge_tx_buffer_ctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_tx_buffer_t *stbp = buf;
+	sfxge_t *sp = arg;
+	sfxge_dma_buffer_attr_t dma_attr;
+	int rc;
+
+	bzero(buf, sizeof (sfxge_tx_buffer_t));
+
+	dma_attr.sdba_dip	 = sp->s_dip;
+	dma_attr.sdba_dattrp	 = &sfxge_tx_buffer_dma_attr;
+	dma_attr.sdba_callback	 = ((kmflags == KM_SLEEP) ?
+	    DDI_DMA_SLEEP : DDI_DMA_DONTWAIT);
+	dma_attr.sdba_length	 = SFXGE_TX_BUFFER_SIZE;
+	dma_attr.sdba_memflags	 = DDI_DMA_STREAMING;
+	dma_attr.sdba_devaccp	 = &sfxge_tx_buffer_devacc;
+	dma_attr.sdba_bindflags	 = DDI_DMA_WRITE | DDI_DMA_STREAMING;
+	dma_attr.sdba_maxcookies = 1;
+	dma_attr.sdba_zeroinit	 = B_FALSE;
+
+	if ((rc = sfxge_dma_buffer_create(&(stbp->stb_esm), &dma_attr)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	SFXGE_OBJ_CHECK(stbp, sfxge_tx_buffer_t);
+
+	return (-1);
+}
+
+static void
+sfxge_tx_buffer_dtor(void *buf, void *arg)
+{
+	sfxge_tx_buffer_t *stbp = buf;
+
+	_NOTE(ARGUNUSED(arg))
+
+	sfxge_dma_buffer_destroy(&(stbp->stb_esm));
+
+	SFXGE_OBJ_CHECK(stbp, sfxge_tx_buffer_t);
+}
+
+static int
+sfxge_tx_mapping_ctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_tx_mapping_t *stmp = buf;
+	sfxge_t *sp = arg;
+	dev_info_t *dip = sp->s_dip;
+	int rc;
+
+	bzero(buf, sizeof (sfxge_tx_mapping_t));
+
+	stmp->stm_sp = sp;
+
+	/* Allocate DMA handle */
+	rc = ddi_dma_alloc_handle(dip, &sfxge_tx_mapping_dma_attr,
+	    (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
+	    NULL, &(stmp->stm_dma_handle));
+	if (rc != DDI_SUCCESS)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	stmp->stm_sp = NULL;
+
+	SFXGE_OBJ_CHECK(stmp, sfxge_tx_mapping_t);
+
+	return (-1);
+}
+
+static void
+sfxge_tx_mapping_dtor(void *buf, void *arg)
+{
+	sfxge_tx_mapping_t *stmp = buf;
+
+	_NOTE(ARGUNUSED(arg))
+
+	ASSERT3P(stmp->stm_sp, ==, arg);
+
+	/* Free the DMA handle */
+	ddi_dma_free_handle(&(stmp->stm_dma_handle));
+	stmp->stm_dma_handle = NULL;
+
+	stmp->stm_sp = NULL;
+
+	SFXGE_OBJ_CHECK(stmp, sfxge_tx_mapping_t);
+}
+
+static int
+sfxge_tx_qctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_txq_t *stp = buf;
+	efsys_mem_t *esmp = &(stp->st_mem);
+	sfxge_t *sp = arg;
+	sfxge_dma_buffer_attr_t dma_attr;
+	sfxge_tx_dpl_t *stdp;
+	int rc;
+
+	/* Compile-time structure layout checks */
+	EFX_STATIC_ASSERT(sizeof (stp->__st_u1.__st_s1) <=
+	    sizeof (stp->__st_u1.__st_pad));
+	EFX_STATIC_ASSERT(sizeof (stp->__st_u2.__st_s2) <=
+	    sizeof (stp->__st_u2.__st_pad));
+	EFX_STATIC_ASSERT(sizeof (stp->__st_u3.__st_s3) <=
+	    sizeof (stp->__st_u3.__st_pad));
+	EFX_STATIC_ASSERT(sizeof (stp->__st_u4.__st_s4) <=
+	    sizeof (stp->__st_u4.__st_pad));
+
+	bzero(buf, sizeof (sfxge_txq_t));
+
+	stp->st_sp = sp;
+
+	dma_attr.sdba_dip	 = sp->s_dip;
+	dma_attr.sdba_dattrp	 = &sfxge_txq_dma_attr;
+	dma_attr.sdba_callback	 = DDI_DMA_SLEEP;
+	dma_attr.sdba_length	 = EFX_TXQ_SIZE(SFXGE_TX_NDESCS);
+	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
+	dma_attr.sdba_devaccp	 = &sfxge_txq_devacc;
+	dma_attr.sdba_bindflags	 = DDI_DMA_READ | DDI_DMA_CONSISTENT;
+	dma_attr.sdba_maxcookies = EFX_TXQ_NBUFS(SFXGE_TX_NDESCS);
+	dma_attr.sdba_zeroinit	 = B_FALSE;
+
+	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
+		goto fail1;
+
+	/* Allocate some buffer table entries */
+	if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS),
+	    &(stp->st_id))) != 0)
+		goto fail2;
+
+	/* Allocate the descriptor array */
+	if ((stp->st_eb = kmem_zalloc(sizeof (efx_buffer_t) *
+	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS), kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail3;
+	}
+
+	/* Allocate the context arrays */
+	if ((stp->st_stmp = kmem_zalloc(sizeof (sfxge_tx_mapping_t *) *
+	    SFXGE_TX_NDESCS, kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail4;
+	}
+
+	if ((stp->st_stbp = kmem_zalloc(sizeof (sfxge_tx_buffer_t *) *
+	    SFXGE_TX_NDESCS, kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail5;
+	}
+
+	if ((stp->st_mp = kmem_zalloc(sizeof (mblk_t *) *
+	    SFXGE_TX_NDESCS, kmflags)) == NULL) {
+		rc = ENOMEM;
+		goto fail6;
+	}
+
+	/* Initialize the deferred packet list */
+	stdp = &(stp->st_dpl);
+	stdp->std_getp = &(stdp->std_get);
+
+	stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;
+
+	return (0);
+
+fail6:
+	DTRACE_PROBE(fail6);
+
+	kmem_free(stp->st_stbp, sizeof (sfxge_tx_buffer_t *) * SFXGE_TX_NDESCS);
+	stp->st_stbp = NULL;
+
+fail5:
+	DTRACE_PROBE(fail5);
+
+	kmem_free(stp->st_stmp,
+	    sizeof (sfxge_tx_mapping_t *) * SFXGE_TX_NDESCS);
+	stp->st_stmp = NULL;
+
+fail4:
+	DTRACE_PROBE(fail4);
+
+	/* Free the descriptor array */
+	kmem_free(stp->st_eb, sizeof (efx_buffer_t) *
+	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
+	stp->st_eb = NULL;
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	/* Free the buffer table entries */
+	sfxge_sram_buf_tbl_free(sp, stp->st_id, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));
+	stp->st_id = 0;
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	/* Tear down DMA setup */
+	sfxge_dma_buffer_destroy(esmp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	stp->st_sp = NULL;
+
+	SFXGE_OBJ_CHECK(stp, sfxge_txq_t);
+
+	return (-1);
+}
+
+static void
+sfxge_tx_qdtor(void *buf, void *arg)
+{
+	sfxge_txq_t *stp = buf;
+	efsys_mem_t *esmp = &(stp->st_mem);
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_dpl_t *stdp;
+
+	_NOTE(ARGUNUSED(arg))
+
+	stp->st_unblock = 0;
+
+	/* Tear down the deferred packet list */
+	stdp = &(stp->st_dpl);
+	ASSERT3P(stdp->std_getp, ==, &(stdp->std_get));
+	stdp->std_getp = NULL;
+
+	/* Free the context arrays */
+	kmem_free(stp->st_mp, sizeof (mblk_t *) * SFXGE_TX_NDESCS);
+	stp->st_mp = NULL;
+
+	kmem_free(stp->st_stbp, sizeof (sfxge_tx_buffer_t *) * SFXGE_TX_NDESCS);
+	stp->st_stbp = NULL;
+
+	kmem_free(stp->st_stmp,
+	    sizeof (sfxge_tx_mapping_t *) * SFXGE_TX_NDESCS);
+	stp->st_stmp = NULL;
+
+	/* Free the descriptor array */
+	kmem_free(stp->st_eb, sizeof (efx_buffer_t) *
+	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
+	stp->st_eb = NULL;
+
+	/* Free the buffer table entries */
+	sfxge_sram_buf_tbl_free(sp, stp->st_id, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));
+	stp->st_id = 0;
+
+	/* Tear down dma setup */
+	sfxge_dma_buffer_destroy(esmp);
+
+	stp->st_sp = NULL;
+
+	SFXGE_OBJ_CHECK(stp, sfxge_txq_t);
+}
+
+static void
+sfxge_tx_packet_destroy(sfxge_t *sp, sfxge_tx_packet_t *stpp)
+{
+	kmem_cache_free(sp->s_tpc, stpp);
+}
+
+static sfxge_tx_packet_t *
+sfxge_tx_packet_create(sfxge_t *sp)
+{
+	sfxge_tx_packet_t *stpp;
+
+	stpp = kmem_cache_alloc(sp->s_tpc, KM_NOSLEEP);
+
+	return (stpp);
+}
+
+static inline int
+sfxge_tx_qfpp_put(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp)
+{
+	sfxge_tx_fpp_t *stfp = &(stp->st_fpp);
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	ASSERT3P(stpp->stp_next, ==, NULL);
+	ASSERT3P(stpp->stp_mp, ==, NULL);
+	ASSERT3P(stpp->stp_etherhp, ==, NULL);
+	ASSERT3P(stpp->stp_iphp, ==, NULL);
+	ASSERT3P(stpp->stp_thp, ==, NULL);
+	ASSERT3U(stpp->stp_off, ==, 0);
+	ASSERT3U(stpp->stp_size, ==, 0);
+	ASSERT3U(stpp->stp_mss, ==, 0);
+	ASSERT3U(stpp->stp_dpl_put_len, ==, 0);
+
+	if (stfp->stf_count < SFXGE_TX_FPP_MAX) {
+		/* Add to the start of the list */
+		stpp->stp_next = stfp->stf_stpp;
+		stfp->stf_stpp = stpp;
+		stfp->stf_count++;
+
+		return (0);
+	}
+
+	DTRACE_PROBE(fpp_full);
+	return (ENOSPC);
+}
+
+static inline sfxge_tx_packet_t *
+sfxge_tx_qfpp_get(sfxge_txq_t *stp)
+{
+	sfxge_tx_packet_t *stpp;
+	sfxge_tx_fpp_t *stfp = &(stp->st_fpp);
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	stpp = stfp->stf_stpp;
+	if (stpp == NULL) {
+		ASSERT3U(stfp->stf_count, ==, 0);
+		return (NULL);
+	}
+
+	/* Remove item from the head of the list */
+	stfp->stf_stpp = stpp->stp_next;
+	stpp->stp_next = NULL;
+
+	ASSERT3U(stfp->stf_count, >, 0);
+	stfp->stf_count--;
+
+	if (stfp->stf_count != 0) {
+		ASSERT(stfp->stf_stpp != NULL);
+		prefetch_read_many(stfp->stf_stpp);
+	}
+	return (stpp);
+}
+
+static void
+sfxge_tx_qfpp_empty(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_fpp_t *stfp = &(stp->st_fpp);
+	sfxge_tx_packet_t *stpp;
+
+	mutex_enter(&(stp->st_lock));
+
+	stpp = stfp->stf_stpp;
+	stfp->stf_stpp = NULL;
+
+	while (stpp != NULL) {
+		sfxge_tx_packet_t *next;
+
+		next = stpp->stp_next;
+		stpp->stp_next = NULL;
+
+		ASSERT3U(stfp->stf_count, >, 0);
+		stfp->stf_count--;
+
+		sfxge_tx_packet_destroy(sp, stpp);
+
+		stpp = next;
+	}
+	ASSERT3U(stfp->stf_count, ==, 0);
+
+	mutex_exit(&(stp->st_lock));
+}
+
+static inline void
+sfxge_tx_qfbp_put(sfxge_txq_t *stp, sfxge_tx_buffer_t *stbp)
+{
+	sfxge_tx_fbp_t *stfp = &(stp->st_fbp);
+
+	ASSERT3P(stbp->stb_next, ==, NULL);
+	ASSERT3U(stbp->stb_off, ==, 0);
+	ASSERT3U(stbp->stb_esm.esm_size, ==, 0);
+
+	stbp->stb_next = stfp->stf_stbp;
+	stfp->stf_stbp = stbp;
+	stfp->stf_count++;
+}
+
+
+static inline sfxge_tx_buffer_t *
+sfxge_tx_qfbp_get(sfxge_txq_t *stp)
+{
+	sfxge_tx_buffer_t *stbp;
+	sfxge_tx_fbp_t *stfp = &(stp->st_fbp);
+
+	stbp = stfp->stf_stbp;
+	if (stbp == NULL) {
+		ASSERT3U(stfp->stf_count, ==, 0);
+		return (NULL);
+	}
+
+	stfp->stf_stbp = stbp->stb_next;
+	stbp->stb_next = NULL;
+
+	ASSERT3U(stfp->stf_count, >, 0);
+	stfp->stf_count--;
+
+	if (stfp->stf_count != 0) {
+		ASSERT(stfp->stf_stbp != NULL);
+		prefetch_read_many(stfp->stf_stbp);
+	}
+
+	return (stbp);
+}
+
+static void
+sfxge_tx_qfbp_empty(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_fbp_t *stfp = &(stp->st_fbp);
+	sfxge_tx_buffer_t *stbp;
+
+	mutex_enter(&(stp->st_lock));
+
+	stbp = stfp->stf_stbp;
+	stfp->stf_stbp = NULL;
+
+	while (stbp != NULL) {
+		sfxge_tx_buffer_t *next;
+
+		next = stbp->stb_next;
+		stbp->stb_next = NULL;
+
+		ASSERT3U(stfp->stf_count, >, 0);
+		stfp->stf_count--;
+
+		kmem_cache_free(sp->s_tbc, stbp);
+
+		stbp = next;
+	}
+	ASSERT3U(stfp->stf_count, ==, 0);
+
+	mutex_exit(&(stp->st_lock));
+}
+
+static inline void
+sfxge_tx_qfmp_put(sfxge_txq_t *stp, sfxge_tx_mapping_t *stmp)
+{
+	sfxge_tx_fmp_t *stfp = &(stp->st_fmp);
+
+	ASSERT3P(stmp->stm_next, ==, NULL);
+	ASSERT3P(stmp->stm_mp, ==, NULL);
+	ASSERT3P(stmp->stm_base, ==, NULL);
+	ASSERT3U(stmp->stm_off, ==, 0);
+	ASSERT3U(stmp->stm_size, ==, 0);
+
+	stmp->stm_next = stfp->stf_stmp;
+	stfp->stf_stmp = stmp;
+	stfp->stf_count++;
+}
+
+static inline sfxge_tx_mapping_t *
+sfxge_tx_qfmp_get(sfxge_txq_t *stp)
+{
+	sfxge_tx_mapping_t *stmp;
+	sfxge_tx_fmp_t *stfp = &(stp->st_fmp);
+
+	stmp = stfp->stf_stmp;
+	if (stmp == NULL) {
+		ASSERT3U(stfp->stf_count, ==, 0);
+		return (NULL);
+	}
+
+	stfp->stf_stmp = stmp->stm_next;
+	stmp->stm_next = NULL;
+
+	ASSERT3U(stfp->stf_count, >, 0);
+	stfp->stf_count--;
+
+	if (stfp->stf_count != 0) {
+		ASSERT(stfp->stf_stmp != NULL);
+		prefetch_read_many(stfp->stf_stmp);
+	}
+	return (stmp);
+}
+
+static void
+sfxge_tx_qfmp_empty(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_fmp_t *stfp = &(stp->st_fmp);
+	sfxge_tx_mapping_t *stmp;
+
+	mutex_enter(&(stp->st_lock));
+
+	stmp = stfp->stf_stmp;
+	stfp->stf_stmp = NULL;
+
+	while (stmp != NULL) {
+		sfxge_tx_mapping_t *next;
+
+		next = stmp->stm_next;
+		stmp->stm_next = NULL;
+
+		ASSERT3U(stfp->stf_count, >, 0);
+		stfp->stf_count--;
+
+		kmem_cache_free(sp->s_tmc, stmp);
+
+		stmp = next;
+	}
+	ASSERT3U(stfp->stf_count, ==, 0);
+
+	mutex_exit(&(stp->st_lock));
+}
+
+static void
+sfxge_tx_msgb_unbind(sfxge_tx_mapping_t *stmp)
+{
+	bzero(stmp->stm_addr, sizeof (uint64_t) * SFXGE_TX_MAPPING_NADDR);
+	stmp->stm_off = 0;
+
+	(void) ddi_dma_unbind_handle(stmp->stm_dma_handle);
+
+	stmp->stm_size = 0;
+	stmp->stm_base = NULL;
+
+	stmp->stm_mp = NULL;
+}
+
+#define	SFXGE_TX_DESCSHIFT	12
+#define	SFXGE_TX_DESCSIZE	(1 << 12)
+
+#define	SFXGE_TX_DESCOFFSET	(SFXGE_TX_DESCSIZE - 1)
+#define	SFXGE_TX_DESCMASK	(~SFXGE_TX_DESCOFFSET)
+
+static int
+sfxge_tx_msgb_bind(mblk_t *mp, sfxge_tx_mapping_t *stmp)
+{
+	ddi_dma_cookie_t dmac;
+	unsigned int ncookies;
+	size_t size;
+	unsigned int n;
+	int rc;
+
+	ASSERT(mp != NULL);
+	ASSERT3U(DB_TYPE(mp), ==, M_DATA);
+
+	ASSERT(stmp->stm_mp == NULL);
+	stmp->stm_mp = mp;
+
+	stmp->stm_base = (caddr_t)(mp->b_rptr);
+	stmp->stm_size = MBLKL(mp);
+
+	/* Bind the STREAMS block to the mapping */
+	rc = ddi_dma_addr_bind_handle(stmp->stm_dma_handle, NULL,
+	    stmp->stm_base, stmp->stm_size, DDI_DMA_WRITE | DDI_DMA_STREAMING,
+	    DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies);
+	if (rc != DDI_DMA_MAPPED)
+		goto fail1;
+
+	ASSERT3U(ncookies, <=, SFXGE_TX_MAPPING_NADDR);
+
+	/*
+	 * Construct an array of addresses and an initial
+	 * offset.
+	 */
+	n = 0;
+	stmp->stm_addr[n++] = dmac.dmac_laddress & SFXGE_TX_DESCMASK;
+	DTRACE_PROBE1(addr, uint64_t, dmac.dmac_laddress & SFXGE_TX_DESCMASK);
+
+	stmp->stm_off = dmac.dmac_laddress & SFXGE_TX_DESCOFFSET;
+
+	size = MIN(SFXGE_TX_DESCSIZE - stmp->stm_off, dmac.dmac_size);
+	dmac.dmac_laddress += size;
+	dmac.dmac_size -= size;
+
+	for (;;) {
+		ASSERT3U(n, <, SFXGE_TX_MAPPING_NADDR);
+
+		if (dmac.dmac_size == 0) {
+			if (--ncookies == 0)
+				break;
+
+			ddi_dma_nextcookie(stmp->stm_dma_handle, &dmac);
+		}
+
+		ASSERT((dmac.dmac_laddress & SFXGE_TX_DESCMASK) != 0);
+		ASSERT((dmac.dmac_laddress & SFXGE_TX_DESCOFFSET) == 0);
+		stmp->stm_addr[n++] = dmac.dmac_laddress;
+		DTRACE_PROBE1(addr, uint64_t, dmac.dmac_laddress);
+
+		size = MIN(SFXGE_TX_DESCSIZE, dmac.dmac_size);
+		dmac.dmac_laddress += size;
+		dmac.dmac_size -= size;
+	}
+	ASSERT3U(n, <=, SFXGE_TX_MAPPING_NADDR);
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	stmp->stm_size = 0;
+	stmp->stm_base = NULL;
+
+	stmp->stm_mp = NULL;
+
+	return (-1);
+}
+
+static void
+sfxge_tx_qreap(sfxge_txq_t *stp)
+{
+	unsigned int reaped;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	reaped = stp->st_reaped;
+	while (reaped != stp->st_completed) {
+		unsigned int id;
+		sfxge_tx_mapping_t *stmp;
+		sfxge_tx_buffer_t *stbp;
+
+		id = reaped++ & (SFXGE_TX_NDESCS - 1);
+
+		ASSERT3P(stp->st_mp[id], ==, NULL);
+
+		if ((stmp = stp->st_stmp[id]) != NULL) {
+			stp->st_stmp[id] = NULL;
+
+			/* Free all the mappings */
+			do {
+				sfxge_tx_mapping_t *next;
+
+				next = stmp->stm_next;
+				stmp->stm_next = NULL;
+
+				sfxge_tx_qfmp_put(stp, stmp);
+
+				stmp = next;
+			} while (stmp != NULL);
+		}
+
+		if ((stbp = stp->st_stbp[id]) != NULL) {
+			stp->st_stbp[id] = NULL;
+
+			/* Free all the buffers */
+			do {
+				sfxge_tx_buffer_t *next;
+
+				next = stbp->stb_next;
+				stbp->stb_next = NULL;
+
+				stbp->stb_esm.esm_size = 0;
+				stbp->stb_off = 0;
+
+				sfxge_tx_qfbp_put(stp, stbp);
+
+				stbp = next;
+			} while (stbp != NULL);
+		}
+	}
+	stp->st_reaped = reaped;
+}
+
+static void
+sfxge_tx_qlist_abort(sfxge_txq_t *stp)
+{
+	unsigned int id;
+	sfxge_tx_mapping_t *stmp;
+	sfxge_tx_buffer_t *stbp;
+	mblk_t *mp;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	id = stp->st_added & (SFXGE_TX_NDESCS - 1);
+
+	/* Clear the completion information */
+	stmp = stp->st_stmp[id];
+	stp->st_stmp[id] = NULL;
+
+	/* Free any mappings that were used */
+	while (stmp != NULL) {
+		sfxge_tx_mapping_t *next;
+
+		next = stmp->stm_next;
+		stmp->stm_next = NULL;
+
+		if (stmp->stm_mp != NULL)
+			sfxge_tx_msgb_unbind(stmp);
+
+		sfxge_tx_qfmp_put(stp, stmp);
+
+		stmp = next;
+	}
+
+	stbp = stp->st_stbp[id];
+	stp->st_stbp[id] = NULL;
+
+	/* Free any buffers that were used */
+	while (stbp != NULL) {
+		sfxge_tx_buffer_t *next;
+
+		next = stbp->stb_next;
+		stbp->stb_next = NULL;
+
+		stbp->stb_off = 0;
+		stbp->stb_esm.esm_size = 0;
+
+		sfxge_tx_qfbp_put(stp, stbp);
+
+		stbp = next;
+	}
+
+	mp = stp->st_mp[id];
+	stp->st_mp[id] = NULL;
+
+	if (mp != NULL)
+		freemsg(mp);
+
+	/* Clear the fragment list */
+	stp->st_n = 0;
+}
+
+/* Push descriptors to the TX ring setting blocked if no space */
+static void
+sfxge_tx_qlist_post(sfxge_txq_t *stp)
+{
+	unsigned int id;
+	unsigned int level;
+	unsigned int available;
+	int rc;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	ASSERT(stp->st_n != 0);
+
+again:
+	level = stp->st_added - stp->st_reaped;
+	available = EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) - level;
+
+	id = stp->st_added & (SFXGE_TX_NDESCS - 1);
+
+	if (available < stp->st_n) {
+		rc = ENOSPC;
+		goto fail1;
+	}
+
+	ASSERT3U(available, >=, stp->st_n);
+
+	/* Post the fragment list */
+	if ((rc = efx_tx_qpost(stp->st_etp, stp->st_eb, stp->st_n,
+	    stp->st_reaped, &(stp->st_added))) != 0)
+		goto fail2;
+
+	/*
+	 * If the list took more than a single descriptor then we need to
+	 * to move the completion information so it is referenced by the last
+	 * descriptor.
+	 */
+	if (((stp->st_added - 1) & (SFXGE_TX_NDESCS - 1)) != id) {
+		sfxge_tx_mapping_t *stmp;
+		sfxge_tx_buffer_t *stbp;
+		mblk_t *mp;
+
+		stmp = stp->st_stmp[id];
+		stp->st_stmp[id] = NULL;
+
+		stbp = stp->st_stbp[id];
+		stp->st_stbp[id] = NULL;
+
+		mp = stp->st_mp[id];
+		stp->st_mp[id] = NULL;
+
+		id = (stp->st_added - 1) & (SFXGE_TX_NDESCS - 1);
+
+		ASSERT(stp->st_stmp[id] == NULL);
+		stp->st_stmp[id] = stmp;
+
+		ASSERT(stp->st_stbp[id] == NULL);
+		stp->st_stbp[id] = stbp;
+
+		ASSERT(stp->st_mp[id] == NULL);
+		stp->st_mp[id] = mp;
+	}
+
+	/* Make the descriptors visible to the hardware */
+	(void) ddi_dma_sync(stp->st_mem.esm_dma_handle,
+	    0,
+	    EFX_TXQ_SIZE(SFXGE_TX_NDESCS),
+	    DDI_DMA_SYNC_FORDEV);
+
+	/* Clear the list */
+	stp->st_n = 0;
+
+	ASSERT3U(stp->st_unblock, ==, SFXGE_TXQ_NOT_BLOCKED);
+	return;
+
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	ASSERT(rc == ENOSPC);
+
+	level = stp->st_added - stp->st_completed;
+	available = EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) - level;
+
+	/*
+	 * If there would be enough space after we've reaped any completed
+	 * mappings and buffers, and we gain sufficient queue space by doing
+	 * so, then reap now and try posting again.
+	 */
+	if (stp->st_n <= available &&
+	    stp->st_completed - stp->st_reaped >= SFXGE_TX_BATCH) {
+		sfxge_tx_qreap(stp);
+
+		goto again;
+	}
+
+	/* Set the unblock level */
+	if (stp->st_unblock == SFXGE_TXQ_NOT_BLOCKED) {
+		stp->st_unblock = SFXGE_TXQ_UNBLOCK_LEVEL1;
+	} else {
+		ASSERT(stp->st_unblock == SFXGE_TXQ_UNBLOCK_LEVEL1);
+
+		stp->st_unblock = SFXGE_TXQ_UNBLOCK_LEVEL2;
+	}
+
+	/*
+	 * Avoid a race with completion interrupt handling that could leave the
+	 * queue blocked.
+	 *
+	 * NOTE: The use of st_pending rather than st_completed is intentional
+	 *	 as st_pending is updated per-event rather than per-batch and
+	 *	 therefore avoids needless deferring.
+	 */
+	if (stp->st_pending == stp->st_added) {
+		sfxge_tx_qreap(stp);
+
+		stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;
+		goto again;
+	}
+
+	ASSERT(stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED);
+}
+
+static int
+sfxge_tx_kstat_update(kstat_t *ksp, int rw)
+{
+	sfxge_txq_t *stp = ksp->ks_private;
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+	kstat_named_t *knp;
+	int rc;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	if (rw != KSTAT_READ) {
+		rc = EACCES;
+		goto fail1;
+	}
+
+	if (stp->st_state != SFXGE_TXQ_STARTED)
+		goto done;
+
+	efx_tx_qstats_update(stp->st_etp, stp->st_stat);
+	knp = (kstat_named_t *)ksp->ks_data + TX_NQSTATS;
+	knp->value.ui64 = stdp->get_pkt_limit;
+	knp++;
+	knp->value.ui64 = stdp->put_pkt_limit;
+	knp++;
+	knp->value.ui64 = stdp->get_full_count;
+	knp++;
+	knp->value.ui64 = stdp->put_full_count;
+
+done:
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_tx_kstat_init(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	unsigned int index = stp->st_index;
+	dev_info_t *dip = sp->s_dip;
+	kstat_t *ksp;
+	kstat_named_t *knp;
+	char name[MAXNAMELEN];
+	unsigned int id;
+	int rc;
+
+	/* Create the set */
+	(void) snprintf(name, MAXNAMELEN - 1, "%s_txq%04d",
+	    ddi_driver_name(dip), index);
+
+	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
+	    ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
+	    TX_NQSTATS + 4, 0)) == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	stp->st_ksp = ksp;
+
+	ksp->ks_update = sfxge_tx_kstat_update;
+	ksp->ks_private = stp;
+	ksp->ks_lock = &(stp->st_lock);
+
+	/* Initialise the named stats */
+	stp->st_stat = knp = ksp->ks_data;
+	for (id = 0; id < TX_NQSTATS; id++) {
+		kstat_named_init(knp, (char *)efx_tx_qstat_name(sp->s_enp, id),
+		    KSTAT_DATA_UINT64);
+		knp++;
+	}
+	kstat_named_init(knp, "dpl_get_pkt_limit", KSTAT_DATA_UINT64);
+	knp++;
+	kstat_named_init(knp, "dpl_put_pkt_limit", KSTAT_DATA_UINT64);
+	knp++;
+	kstat_named_init(knp, "dpl_get_full_count", KSTAT_DATA_UINT64);
+	knp++;
+	kstat_named_init(knp, "dpl_put_full_count", KSTAT_DATA_UINT64);
+
+	kstat_install(ksp);
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static void
+sfxge_tx_kstat_fini(sfxge_txq_t *stp)
+{
+	/* Destroy the set */
+	kstat_delete(stp->st_ksp);
+	stp->st_ksp = NULL;
+	stp->st_stat = NULL;
+}
+
+static int
+sfxge_tx_qinit(sfxge_t *sp, unsigned int index, sfxge_txq_type_t type,
+    unsigned int evq)
+{
+	sfxge_txq_t *stp;
+	sfxge_tx_dpl_t *stdp;
+	int rc;
+
+	ASSERT3U(index, <, SFXGE_TXQ_NTYPES + SFXGE_RX_SCALE_MAX);
+	ASSERT3U(type, <, SFXGE_TXQ_NTYPES);
+	ASSERT3U(evq, <, SFXGE_RX_SCALE_MAX);
+
+	stp = kmem_cache_alloc(sp->s_tqc, KM_SLEEP);
+	stdp = &(stp->st_dpl);
+
+	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_UNINITIALIZED);
+
+	stp->st_index = index;
+	stp->st_type = type;
+	stp->st_evq = evq;
+
+	mutex_init(&(stp->st_lock), NULL, MUTEX_DRIVER,
+	    DDI_INTR_PRI(sp->s_intr.si_intr_pri));
+
+	/* Initialize the statistics */
+	if ((rc = sfxge_tx_kstat_init(stp)) != 0)
+		goto fail1;
+
+	stdp->get_pkt_limit = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "tx_dpl_get_pkt_limit",
+	    SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT);
+
+	stdp->put_pkt_limit = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "tx_dpl_put_pkt_limit",
+	    SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT);
+
+	stp->st_state = SFXGE_TXQ_INITIALIZED;
+
+	/* Attach the TXQ to the driver */
+	ASSERT3P(sp->s_stp[index], ==, NULL);
+	sp->s_stp[index] = stp;
+	sp->s_tx_qcount++;
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	stp->st_evq = 0;
+	stp->st_type = 0;
+	stp->st_index = 0;
+
+	mutex_destroy(&(stp->st_lock));
+
+	kmem_cache_free(sp->s_tqc, stp);
+
+	return (rc);
+}
+
+static int
+sfxge_tx_qstart(sfxge_t *sp, unsigned int index)
+{
+	sfxge_txq_t *stp = sp->s_stp[index];
+	efx_nic_t *enp = sp->s_enp;
+	efsys_mem_t *esmp;
+	sfxge_evq_t *sep;
+	unsigned int evq;
+	unsigned int flags;
+	int rc;
+
+	mutex_enter(&(stp->st_lock));
+
+	esmp = &(stp->st_mem);
+	evq = stp->st_evq;
+	sep = sp->s_sep[evq];
+
+	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED);
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
+
+	/* Zero the memory */
+	(void) memset(esmp->esm_base, 0, EFX_TXQ_SIZE(SFXGE_TX_NDESCS));
+
+	/* Program the buffer table */
+	if ((rc = sfxge_sram_buf_tbl_set(sp, stp->st_id, esmp,
+	    EFX_TXQ_NBUFS(SFXGE_TX_NDESCS))) != 0)
+		goto fail1;
+
+	switch (stp->st_type) {
+	case SFXGE_TXQ_NON_CKSUM:
+		flags = 0;
+		break;
+
+	case SFXGE_TXQ_IP_CKSUM:
+		flags = EFX_CKSUM_IPV4;
+		break;
+
+	case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
+		flags = EFX_CKSUM_IPV4 | EFX_CKSUM_TCPUDP;
+		break;
+
+	default:
+		ASSERT(B_FALSE);
+
+		flags = 0;
+		break;
+	}
+
+	/* Create the transmit queue */
+	if ((rc = efx_tx_qcreate(enp, index, index, esmp, SFXGE_TX_NDESCS,
+	    stp->st_id, flags, sep->se_eep, &(stp->st_etp))) != 0)
+		goto fail2;
+
+	/* Enable the transmit queue */
+	efx_tx_qenable(stp->st_etp);
+
+	stp->st_state = SFXGE_TXQ_STARTED;
+
+	mutex_exit(&(stp->st_lock));
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	/* Clear entries from the buffer table */
+	sfxge_sram_buf_tbl_clear(sp, stp->st_id,
+	    EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(stp->st_lock));
+
+	return (rc);
+}
+
+static inline int
+sfxge_tx_qmapping_add(sfxge_txq_t *stp, sfxge_tx_mapping_t *stmp,
+size_t *offp, size_t *limitp)
+{
+	mblk_t *mp;
+	size_t mapping_off;
+	size_t mapping_size;
+	int rc;
+
+	ASSERT3U(*offp, <, stmp->stm_size);
+	ASSERT(*limitp != 0);
+
+	mp = stmp->stm_mp;
+
+	ASSERT3P(stmp->stm_base, ==, mp->b_rptr);
+	ASSERT3U(stmp->stm_size, ==, MBLKL(mp));
+
+	mapping_off = stmp->stm_off + *offp;
+	mapping_size = stmp->stm_size - *offp;
+
+	while (mapping_size != 0 && *limitp != 0) {
+		size_t page =
+		    mapping_off >> SFXGE_TX_DESCSHIFT;
+		size_t page_off =
+		    mapping_off & SFXGE_TX_DESCOFFSET;
+		size_t page_size =
+		    SFXGE_TX_DESCSIZE - page_off;
+		efx_buffer_t *ebp;
+
+		ASSERT3U(page, <, SFXGE_TX_MAPPING_NADDR);
+		ASSERT((stmp->stm_addr[page] &
+			SFXGE_TX_DESCMASK) != 0);
+
+		page_size = MIN(page_size, mapping_size);
+		page_size = MIN(page_size, *limitp);
+
+		ASSERT3U(stp->st_n, <=,
+		    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
+		if (stp->st_n ==
+		    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS)) {
+			rc = ENOSPC;
+			goto fail1;
+		}
+
+		ebp = &(stp->st_eb[stp->st_n++]);
+		ebp->eb_addr = stmp->stm_addr[page] +
+		    page_off;
+		ebp->eb_size = page_size;
+
+		*offp += page_size;
+		*limitp -= page_size;
+
+		mapping_off += page_size;
+		mapping_size -= page_size;
+
+		ebp->eb_eop = (*limitp == 0 ||
+		    (mapping_size == 0 && mp->b_cont == NULL));
+
+		DTRACE_PROBE5(tx_mapping_add,
+		    unsigned int, stp->st_index,
+		    unsigned int, stp->st_n - 1,
+		    uint64_t, ebp->eb_addr,
+		    size_t, ebp->eb_size,
+		    boolean_t, ebp->eb_eop);
+	}
+
+	ASSERT3U(*offp, <=, stmp->stm_size);
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static inline int
+sfxge_tx_qbuffer_add(sfxge_txq_t *stp, sfxge_tx_buffer_t *stbp, boolean_t eop)
+{
+	efx_buffer_t *ebp;
+	int rc;
+
+	ASSERT3U(stp->st_n, <=,
+	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
+	if (stp->st_n == EFX_TXQ_LIMIT(SFXGE_TX_NDESCS)) {
+		rc = ENOSPC;
+		goto fail1;
+	}
+
+	ebp = &(stp->st_eb[stp->st_n++]);
+	ebp->eb_addr = stbp->stb_esm.esm_addr + stbp->stb_off;
+	ebp->eb_size = stbp->stb_esm.esm_size - stbp->stb_off;
+	ebp->eb_eop = eop;
+
+	(void) ddi_dma_sync(stbp->stb_esm.esm_dma_handle,
+	    stbp->stb_off, ebp->eb_size,
+	    DDI_DMA_SYNC_FORDEV);
+
+	stbp->stb_off = stbp->stb_esm.esm_size;
+
+	DTRACE_PROBE5(tx_buffer_add,
+	    unsigned int, stp->st_index,
+	    unsigned int, stp->st_n - 1,
+	    uint64_t, ebp->eb_addr, size_t, ebp->eb_size,
+	    boolean_t, ebp->eb_eop);
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static inline boolean_t
+sfxge_tx_msgb_copy(mblk_t *mp, sfxge_tx_buffer_t *stbp, size_t *offp,
+    size_t *limitp)
+{
+	size_t data_off;
+	size_t data_size;
+	size_t copy_off;
+	size_t copy_size;
+	boolean_t eop;
+
+	ASSERT3U(*offp, <=, MBLKL(mp));
+	ASSERT(*limitp != 0);
+
+	data_off = *offp;
+	data_size = MBLKL(mp) - *offp;
+
+	copy_off = stbp->stb_esm.esm_size;
+	copy_size = SFXGE_TX_BUFFER_SIZE - copy_off;
+
+	copy_size = MIN(copy_size, data_size);
+	copy_size = MIN(copy_size, *limitp);
+
+	bcopy(mp->b_rptr + data_off,
+	    stbp->stb_esm.esm_base + copy_off, copy_size);
+
+	stbp->stb_esm.esm_size += copy_size;
+	ASSERT3U(stbp->stb_esm.esm_size, <=,
+	    SFXGE_TX_BUFFER_SIZE);
+
+	*offp += copy_size;
+	*limitp -= copy_size;
+
+	data_off += copy_size;
+	data_size -= copy_size;
+
+	eop = (*limitp == 0 ||
+	    (data_size == 0 && mp->b_cont == NULL));
+
+	ASSERT3U(*offp, <=, MBLKL(mp));
+
+	return (eop);
+}
+
+static int
+sfxge_tx_qpayload_fragment(sfxge_txq_t *stp, unsigned int id, mblk_t **mpp,
+    size_t *offp, size_t size, boolean_t copy)
+{
+	sfxge_t *sp = stp->st_sp;
+	mblk_t *mp = *mpp;
+	size_t off = *offp;
+	sfxge_tx_buffer_t *stbp;
+	sfxge_tx_mapping_t *stmp;
+	int rc;
+
+	stbp = stp->st_stbp[id];
+	ASSERT(stbp == NULL || (stbp->stb_esm.esm_size == stbp->stb_off));
+
+	stmp = stp->st_stmp[id];
+
+	while (size != 0) {
+		boolean_t eop;
+
+		ASSERT(mp != NULL);
+
+		if (mp->b_cont != NULL)
+			prefetch_read_many(mp->b_cont);
+
+		ASSERT3U(off, <, MBLKL(mp));
+
+		if (copy)
+			goto copy;
+
+		/*
+		 * Check whether we have already mapped this data block for
+		 * DMA.
+		 */
+		if (stmp == NULL || stmp->stm_mp != mp) {
+			/*
+			 * If we are part way through copying a data block then
+			 * there's no point in trying to map it for DMA.
+			 */
+			if (off != 0)
+				goto copy;
+
+			/*
+			 * If the data block is too short then the cost of
+			 * mapping it for DMA would outweigh the cost of
+			 * copying it.
+			 */
+			if (MBLKL(mp) < SFXGE_TX_COPY_THRESHOLD)
+				goto copy;
+
+			/* Try to grab a transmit mapping from the pool */
+			stmp = sfxge_tx_qfmp_get(stp);
+			if (stmp == NULL) {
+				/*
+				 * The pool was empty so allocate a new
+				 * mapping.
+				 */
+				if ((stmp = kmem_cache_alloc(sp->s_tmc,
+				    KM_NOSLEEP)) == NULL)
+					goto copy;
+			}
+
+			/* Add the DMA mapping to the list */
+			stmp->stm_next = stp->st_stmp[id];
+			stp->st_stmp[id] = stmp;
+
+			/* Try to bind the data block to the mapping */
+			if (sfxge_tx_msgb_bind(mp, stmp) != 0)
+				goto copy;
+		}
+		ASSERT3P(stmp->stm_mp, ==, mp);
+
+		/*
+		 * If we have a partially filled buffer then we must add it to
+		 * the fragment list before adding the mapping.
+		 */
+		if (stbp != NULL && (stbp->stb_esm.esm_size > stbp->stb_off)) {
+			rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE);
+			if (rc != 0)
+				goto fail1;
+		}
+
+		/* Add the mapping to the fragment list */
+		rc = sfxge_tx_qmapping_add(stp, stmp, &off, &size);
+		if (rc != 0)
+			goto fail2;
+
+		ASSERT(off == MBLKL(mp) || size == 0);
+
+		/*
+		 * If the data block has been exhausted then Skip over the
+		 * control block and advance to the next data block.
+		 */
+		if (off == MBLKL(mp)) {
+			mp = mp->b_cont;
+			off = 0;
+		}
+
+		continue;
+
+copy:
+		if (stbp == NULL ||
+		    stbp->stb_esm.esm_size == SFXGE_TX_BUFFER_SIZE) {
+			/* Try to grab a buffer from the pool */
+			stbp = sfxge_tx_qfbp_get(stp);
+			if (stbp == NULL) {
+				/*
+				 * The pool was empty so allocate a new
+				 * buffer.
+				 */
+				if ((stbp = kmem_cache_alloc(sp->s_tbc,
+				    KM_NOSLEEP)) == NULL) {
+					rc = ENOMEM;
+					goto fail3;
+				}
+			}
+
+			/* Add it to the list */
+			stbp->stb_next = stp->st_stbp[id];
+			stp->st_stbp[id] = stbp;
+		}
+
+		/* Copy as much of the data block as we can into the buffer */
+		eop = sfxge_tx_msgb_copy(mp, stbp, &off, &size);
+
+		ASSERT(off == MBLKL(mp) || size == 0 ||
+		    stbp->stb_esm.esm_size == SFXGE_TX_BUFFER_SIZE);
+
+		/*
+		 * If we have reached the end of the packet, or the buffer is
+		 * full, then add the buffer to the fragment list.
+		 */
+		if (stbp->stb_esm.esm_size == SFXGE_TX_BUFFER_SIZE || eop) {
+			rc = sfxge_tx_qbuffer_add(stp, stbp, eop);
+			if (rc != 0)
+				goto fail4;
+		}
+
+		/*
+		 * If the data block has been exhaused then advance to the next
+		 * one.
+		 */
+		if (off == MBLKL(mp)) {
+			mp = mp->b_cont;
+			off = 0;
+		}
+	}
+
+	*mpp = mp;
+	*offp = off;
+
+	return (0);
+
+fail4:
+	DTRACE_PROBE(fail4);
+fail3:
+	DTRACE_PROBE(fail3);
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_tx_qlso_fragment(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp,
+    boolean_t copy)
+{
+	sfxge_t *sp = stp->st_sp;
+	mblk_t *mp = stpp->stp_mp;
+	struct ether_header *etherhp = stpp->stp_etherhp;
+	struct ip *iphp = stpp->stp_iphp;
+	struct tcphdr *thp = stpp->stp_thp;
+	size_t size = stpp->stp_size;
+	size_t off = stpp->stp_off;
+	size_t mss = stpp->stp_mss;
+	unsigned int id;
+	caddr_t hp;
+	size_t ehs, hs;
+	uint16_t start_len;
+	uint16_t start_id;
+	uint16_t ip_id;
+	uint8_t start_flags;
+	uint32_t start_seq;
+	uint32_t th_seq;
+	size_t lss;
+	sfxge_tx_buffer_t *stbp;
+	int rc;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	if ((DB_LSOFLAGS(mp) & HW_LSO) == 0) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	id = stp->st_added & (SFXGE_TX_NDESCS - 1);
+
+	ASSERT(stp->st_n == 0);
+	ASSERT(stp->st_stbp[id] == NULL);
+	ASSERT(stp->st_stmp[id] == NULL);
+
+	ehs = (etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
+	    sizeof (struct ether_vlan_header) :
+	    sizeof (struct ether_header);
+	if (msgdsize(mp) != ehs + ntohs(iphp->ip_len)) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	/* The payload offset is equivalent to the size of the headers */
+	hp = (caddr_t)(mp->b_rptr);
+	hs = off;
+
+	/*
+	 * If the initial data block only contains the headers then advance
+	 * to the next one.
+	 */
+	if (hs > MBLKL(mp)) {
+		rc = EINVAL;
+		goto fail3;
+	}
+	mp->b_rptr += hs;
+
+	if (MBLKL(mp) == 0)
+		mp = mp->b_cont;
+
+	off = 0;
+
+	/* Check IP and TCP headers are suitable for LSO */
+	if (((iphp->ip_off & ~htons(IP_DF)) != 0) ||
+	    ((thp->th_flags & (TH_URG | TH_SYN)) != 0) ||
+	    (thp->th_urp != 0)) {
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	if (size + (thp->th_off << 2) + (iphp->ip_hl << 2) !=
+	    ntohs(iphp->ip_len)) {
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	/*
+	 * Get the base IP id, The stack leaves enough of a gap in id space
+	 * for us to increment this for each segment we send out.
+	 */
+	start_len = ntohs(iphp->ip_len);
+	start_id = ip_id = ntohs(iphp->ip_id);
+
+	/* Get the base TCP sequence number and flags */
+	start_flags = thp->th_flags;
+	start_seq = th_seq = ntohl(thp->th_seq);
+
+	/* Adjust the header for interim segments */
+	iphp->ip_len = htons((iphp->ip_hl << 2) + (thp->th_off << 2) + mss);
+	thp->th_flags = start_flags & ~(TH_PUSH | TH_FIN);
+
+	lss = size;
+	if ((lss / mss) >= (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 2)) {
+		rc = EINVAL;
+		goto fail5;
+	}
+
+	stbp = NULL;
+	while (lss != 0) {
+		size_t ss = MIN(lss, mss);
+		boolean_t eol = (ss == lss);
+
+		/* Adjust the header for this segment */
+		iphp->ip_id = htons(ip_id);
+		ip_id++;
+
+		thp->th_seq = htonl(th_seq);
+		th_seq += ss;
+
+		/* If this is the final segment then do some extra adjustment */
+		if (eol) {
+			iphp->ip_len = htons((iphp->ip_hl << 2) +
+			    (thp->th_off << 2) + ss);
+			thp->th_flags = start_flags;
+		}
+
+		if (stbp == NULL ||
+		    stbp->stb_esm.esm_size + hs > SFXGE_TX_BUFFER_SIZE) {
+			/* Try to grab a buffer from the pool */
+			stbp = sfxge_tx_qfbp_get(stp);
+			if (stbp == NULL) {
+				/*
+				 * The pool was empty so allocate a new
+				 * buffer.
+				 */
+				if ((stbp = kmem_cache_alloc(sp->s_tbc,
+				    KM_NOSLEEP)) == NULL) {
+					rc = ENOMEM;
+					goto fail6;
+				}
+			}
+
+			/* Add it to the list */
+			stbp->stb_next = stp->st_stbp[id];
+			stp->st_stbp[id] = stbp;
+		}
+
+		/* Copy in the headers */
+		ASSERT3U(stbp->stb_off, ==, stbp->stb_esm.esm_size);
+		bcopy(hp, stbp->stb_esm.esm_base + stbp->stb_off, hs);
+		stbp->stb_esm.esm_size += hs;
+
+		/* Add the buffer to the fragment list */
+		rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE);
+		if (rc != 0)
+			goto fail7;
+
+		/* Add the payload to the fragment list */
+		if ((rc = sfxge_tx_qpayload_fragment(stp, id, &mp, &off,
+		    ss, copy)) != 0)
+			goto fail8;
+
+		lss -= ss;
+	}
+	ASSERT3U(off, ==, 0);
+	ASSERT3P(mp, ==, NULL);
+
+	ASSERT3U(th_seq - start_seq, ==, size);
+
+	/*
+	 * If no part of the packet has been mapped for DMA then we can free
+	 * it now, otherwise it can only be freed on completion.
+	 */
+	if (stp->st_stmp[id] == NULL)
+		freemsg(stpp->stp_mp);
+	else
+		stp->st_mp[id] = stpp->stp_mp;
+
+	stpp->stp_mp = NULL;
+
+	return (0);
+
+fail8:
+	DTRACE_PROBE(fail8);
+fail7:
+	DTRACE_PROBE(fail7);
+fail6:
+	DTRACE_PROBE(fail6);
+fail5:
+	DTRACE_PROBE(fail5);
+
+	/* Restore the header */
+	thp->th_seq = htonl(start_seq);
+	thp->th_flags = start_flags;
+
+	iphp->ip_len = htons(start_len);
+	iphp->ip_id = htons(start_id);
+
+fail4:
+	DTRACE_PROBE(fail4);
+
+	mp = stpp->stp_mp;
+	mp->b_rptr -= hs;
+
+	ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
+	    sizeof (struct ether_vlan_header) :
+	    sizeof (struct ether_header)) +
+	    ntohs(iphp->ip_len), ==, msgdsize(mp));
+
+	ASSERT(stp->st_mp[id] == NULL);
+
+fail3:
+	DTRACE_PROBE(fail3);
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_tx_qpacket_fragment(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp,
+    boolean_t copy)
+{
+	sfxge_t *sp = stp->st_sp;
+	mblk_t *mp = stpp->stp_mp;
+	unsigned int id;
+	size_t off;
+	size_t size;
+	sfxge_tx_mapping_t *stmp;
+	sfxge_tx_buffer_t *stbp;
+	int rc;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	ASSERT(stp->st_n == 0);
+
+	id = stp->st_added & (SFXGE_TX_NDESCS - 1);
+
+	ASSERT(stp->st_stbp[id] == NULL);
+	ASSERT(stp->st_stmp[id] == NULL);
+
+	off = 0;
+	size = LONG_MAX;	/* must be larger than the packet */
+
+	stbp = NULL;
+	stmp = NULL;
+
+	while (mp != NULL) {
+		boolean_t eop;
+
+		ASSERT(mp != NULL);
+
+		if (mp->b_cont != NULL)
+			prefetch_read_many(mp->b_cont);
+
+		ASSERT(stmp == NULL || stmp->stm_mp != mp);
+
+		if (copy)
+			goto copy;
+
+		/*
+		 * If we are part way through copying a data block then there's
+		 * no point in trying to map it for DMA.
+		 */
+		if (off != 0)
+			goto copy;
+
+		/*
+		 * If the data block is too short then the cost of mapping it
+		 * for DMA would outweigh the cost of copying it.
+		 *
+		 * TX copy break
+		 */
+		if (MBLKL(mp) < SFXGE_TX_COPY_THRESHOLD)
+			goto copy;
+
+		/* Try to grab a transmit mapping from the pool */
+		stmp = sfxge_tx_qfmp_get(stp);
+		if (stmp == NULL) {
+			/*
+			 * The pool was empty so allocate a new
+			 * mapping.
+			 */
+			if ((stmp = kmem_cache_alloc(sp->s_tmc,
+			    KM_NOSLEEP)) == NULL)
+				goto copy;
+		}
+
+		/* Add the DMA mapping to the list */
+		stmp->stm_next = stp->st_stmp[id];
+		stp->st_stmp[id] = stmp;
+
+		/* Try to bind the data block to the mapping */
+		if (sfxge_tx_msgb_bind(mp, stmp) != 0)
+			goto copy;
+
+		/*
+		 * If we have a partially filled buffer then we must add it to
+		 * the fragment list before adding the mapping.
+		 */
+		if (stbp != NULL && (stbp->stb_esm.esm_size > stbp->stb_off)) {
+			rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE);
+			if (rc != 0)
+				goto fail1;
+		}
+
+		/* Add the mapping to the fragment list */
+		rc = sfxge_tx_qmapping_add(stp, stmp, &off, &size);
+		if (rc != 0)
+			goto fail2;
+
+		ASSERT3U(off, ==, MBLKL(mp));
+
+		/* Advance to the next data block */
+		mp = mp->b_cont;
+		off = 0;
+		continue;
+
+copy:
+		if (stbp == NULL ||
+		    stbp->stb_esm.esm_size == SFXGE_TX_BUFFER_SIZE) {
+			/* Try to grab a buffer from the pool */
+			stbp = sfxge_tx_qfbp_get(stp);
+			if (stbp == NULL) {
+				/*
+				 * The pool was empty so allocate a new
+				 * buffer.
+				 */
+				if ((stbp = kmem_cache_alloc(sp->s_tbc,
+				    KM_NOSLEEP)) == NULL) {
+					rc = ENOMEM;
+					goto fail3;
+				}
+			}
+
+			/* Add it to the list */
+			stbp->stb_next = stp->st_stbp[id];
+			stp->st_stbp[id] = stbp;
+		}
+
+		/* Copy as much of the data block as we can into the buffer */
+		eop = sfxge_tx_msgb_copy(mp, stbp, &off, &size);
+
+		ASSERT(off == MBLKL(mp) ||
+		    stbp->stb_esm.esm_size == SFXGE_TX_BUFFER_SIZE);
+
+		/*
+		 * If we have reached the end of the packet, or the buffer is
+		 * full, then add the buffer to the fragment list.
+		 */
+		if (stbp->stb_esm.esm_size == SFXGE_TX_BUFFER_SIZE || eop) {
+			rc = sfxge_tx_qbuffer_add(stp, stbp, eop);
+			if (rc != 0)
+				goto fail4;
+		}
+
+		/*
+		 * If the data block has been exhaused then advance to the next
+		 * one.
+		 */
+		if (off == MBLKL(mp)) {
+			mp = mp->b_cont;
+			off = 0;
+		}
+	}
+	ASSERT3U(off, ==, 0);
+	ASSERT3P(mp, ==, NULL);
+	ASSERT3U(size, !=, 0);
+
+	/*
+	 * If no part of the packet has been mapped for DMA then we can free
+	 * it now, otherwise it can only be freed on completion.
+	 */
+	if (stp->st_stmp[id] == NULL)
+		freemsg(stpp->stp_mp);
+	else
+		stp->st_mp[id] = stpp->stp_mp;
+
+	stpp->stp_mp = NULL;
+
+	return (0);
+
+fail4:
+	DTRACE_PROBE(fail4);
+fail3:
+	DTRACE_PROBE(fail3);
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	ASSERT(stp->st_stmp[id] == NULL);
+
+	return (rc);
+}
+
+
+#define	SFXGE_TX_QDPL_PUT_PENDING(_stp)					\
+	((_stp)->st_dpl.std_put != 0)
+
+static void
+sfxge_tx_qdpl_swizzle(sfxge_txq_t *stp)
+{
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+	volatile uintptr_t *putp;
+	uintptr_t put;
+	sfxge_tx_packet_t *stpp;
+	sfxge_tx_packet_t *p;
+	sfxge_tx_packet_t **pp;
+	unsigned int count;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	/*
+	 * Guaranteed that in flight TX packets will cause more TX completions
+	 * hence more swizzles must happen
+	 */
+	ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));
+	if (stdp->std_count >= stdp->get_pkt_limit)
+		return;
+
+	/* Acquire the put list - replacing with an empty list */
+	putp = &(stdp->std_put);
+	put = atomic_swap_ulong(putp, 0);
+	stpp = (void *)put;
+
+	if (stpp == NULL)
+		return;
+
+	/* Reverse the list */
+	pp = &(stpp->stp_next);
+	p = NULL;
+
+	count = 0;
+	do {
+		sfxge_tx_packet_t *next;
+
+		next = stpp->stp_next;
+
+		stpp->stp_next = p;
+		p = stpp;
+
+		count++;
+		stpp = next;
+	} while (stpp != NULL);
+
+	/* Add it to the tail of the get list */
+	ASSERT3P(*pp, ==, NULL);
+
+	*(stdp->std_getp) = p;
+	stdp->std_getp = pp;
+	stdp->std_count += count;
+	ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));
+
+	DTRACE_PROBE2(dpl_counts, int, stdp->std_count, int, count);
+}
+
+
+/*
+ * If TXQ locked, add the RX DPL put list and this packet to the TX DPL get list
+ * If TXQ unlocked, atomically add this packet to TX DPL put list
+ *
+ * The only possible error is ENOSPC (used for TX backpressure)
+ * For the TX DPL put or get list becoming full, in both cases there must be
+ * future TX completions (as represented by the packets on the DPL get lists).
+ *
+ * This ensures that in the future mac_tx_update() will be called from
+ * sfxge_tx_qcomplete()
+ */
+static inline int
+sfxge_tx_qdpl_add(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp, int locked)
+{
+	sfxge_tx_dpl_t *stdp = &stp->st_dpl;
+
+	ASSERT3P(stpp->stp_next, ==, NULL);
+
+	if (locked) {
+		ASSERT(mutex_owned(&stp->st_lock));
+
+		if (stdp->std_count >= stdp->get_pkt_limit) {
+			stdp->get_full_count++;
+			return (ENOSPC);
+		}
+
+		/* Reverse the put list onto the get list */
+		sfxge_tx_qdpl_swizzle(stp);
+
+		/* Add to the tail of the get list */
+		*(stdp->std_getp) = stpp;
+		stdp->std_getp = &stpp->stp_next;
+		stdp->std_count++;
+		ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));
+
+	} else {
+		volatile uintptr_t *putp;
+		uintptr_t old;
+		uintptr_t new;
+		sfxge_tx_packet_t *old_pkt;
+
+		putp = &(stdp->std_put);
+		new = (uintptr_t)stpp;
+
+		/* Add to the head of the put list, keeping a list length */
+		do {
+			old = *putp;
+			old_pkt =  (sfxge_tx_packet_t *)old;
+
+			stpp->stp_dpl_put_len = old ?
+			    old_pkt->stp_dpl_put_len + 1 : 1;
+
+			if (stpp->stp_dpl_put_len >= stdp->put_pkt_limit) {
+				stpp->stp_next = 0;
+				stpp->stp_dpl_put_len = 0;
+				stdp->put_full_count++;
+				return (ENOSPC);
+			}
+
+			stpp->stp_next = (void *)old;
+		} while (atomic_cas_ulong(putp, old, new) != old);
+	}
+	return (0);
+}
+
+
+/* Take all packets from DPL get list and try to send to HW */
+static void
+sfxge_tx_qdpl_drain(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+	unsigned int pushed = stp->st_added;
+	sfxge_tx_packet_t *stpp;
+	unsigned int count;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	prefetch_read_many(sp->s_enp);
+	prefetch_read_many(stp->st_etp);
+
+	stpp = stdp->std_get;
+	count = stdp->std_count;
+
+	while (count != 0) {
+		sfxge_tx_packet_t *next;
+		boolean_t copy;
+		int rc;
+
+		ASSERT(stpp != NULL);
+
+		/* Split stpp off */
+		next = stpp->stp_next;
+		stpp->stp_next = NULL;
+
+		if (next != NULL)
+			prefetch_read_many(next);
+
+		if (stp->st_state != SFXGE_TXQ_STARTED)
+			goto reject;
+
+		copy = B_FALSE;
+
+again:
+		/* Fragment the packet */
+		if (stpp->stp_mss != 0) {
+			rc = sfxge_tx_qlso_fragment(stp, stpp, copy);
+		} else {
+			rc = sfxge_tx_qpacket_fragment(stp, stpp, copy);
+		}
+
+		switch (rc) {
+		case 0:
+			break;
+
+		case ENOSPC:
+			if (!copy)
+				goto copy;
+
+		/*FALLTHRU*/
+		default:
+			goto reject;
+		}
+
+		/* Free the packet structure */
+		stpp->stp_etherhp = NULL;
+		stpp->stp_iphp = NULL;
+		stpp->stp_thp = NULL;
+		stpp->stp_off = 0;
+		stpp->stp_size = 0;
+		stpp->stp_mss = 0;
+		stpp->stp_dpl_put_len = 0;
+
+		ASSERT3P(stpp->stp_mp, ==, NULL);
+
+		if (sfxge_tx_qfpp_put(stp, stpp) != 0) {
+			sfxge_tx_packet_destroy(sp, stpp);
+			stpp = NULL;
+		}
+
+		--count;
+		stpp = next;
+
+		/* Post the packet */
+		sfxge_tx_qlist_post(stp);
+
+		if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED)
+			goto defer;
+
+		if (stp->st_added - pushed >= SFXGE_TX_BATCH) {
+			efx_tx_qpush(stp->st_etp, stp->st_added);
+			pushed = stp->st_added;
+		}
+
+		continue;
+
+copy:
+		/* Abort the current fragment list */
+		sfxge_tx_qlist_abort(stp);
+
+		/* Try copying the packet to flatten it */
+		ASSERT(!copy);
+		copy = B_TRUE;
+
+		goto again;
+
+reject:
+		/* Abort the current fragment list */
+		sfxge_tx_qlist_abort(stp);
+
+		/* Discard the packet */
+		freemsg(stpp->stp_mp);
+		stpp->stp_mp = NULL;
+
+		/* Free the packet structure */
+		stpp->stp_etherhp = NULL;
+		stpp->stp_iphp = NULL;
+		stpp->stp_thp = NULL;
+		stpp->stp_off = 0;
+		stpp->stp_size = 0;
+		stpp->stp_mss = 0;
+		stpp->stp_dpl_put_len = 0;
+
+		if (sfxge_tx_qfpp_put(stp, stpp) != 0) {
+			sfxge_tx_packet_destroy(sp, stpp);
+			stpp = NULL;
+		}
+
+		--count;
+		stpp = next;
+		continue;
+defer:
+		DTRACE_PROBE1(defer, unsigned int, stp->st_index);
+		break;
+	}
+
+	if (count == 0) {
+		/* New empty get list */
+		ASSERT3P(stpp, ==, NULL);
+		stdp->std_get = NULL;
+		stdp->std_count = 0;
+
+		stdp->std_getp = &(stdp->std_get);
+	} else {
+		/* shorten the list by moving the head */
+		stdp->std_get = stpp;
+		stdp->std_count = count;
+		ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));
+	}
+
+	if (stp->st_added != pushed)
+		efx_tx_qpush(stp->st_etp, stp->st_added);
+
+	ASSERT(stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED ||
+	    stdp->std_count == 0);
+}
+
+/* Swizzle deferred packet list, try and push to HW */
+static inline void
+sfxge_tx_qdpl_service(sfxge_txq_t *stp)
+{
+	do {
+		ASSERT(mutex_owned(&(stp->st_lock)));
+
+		if (SFXGE_TX_QDPL_PUT_PENDING(stp))
+			sfxge_tx_qdpl_swizzle(stp);
+
+		if (stp->st_unblock == SFXGE_TXQ_NOT_BLOCKED)
+			sfxge_tx_qdpl_drain(stp);
+
+		mutex_exit(&(stp->st_lock));
+
+		if (!SFXGE_TX_QDPL_PUT_PENDING(stp))
+			break;
+	} while (mutex_tryenter(&(stp->st_lock)));
+}
+
+static void
+sfxge_tx_qdpl_flush_locked(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+	sfxge_tx_packet_t *stpp;
+	unsigned int count;
+
+	ASSERT(mutex_owned(&(stp->st_lock)));
+
+	/* Swizzle put list to the get list */
+	sfxge_tx_qdpl_swizzle(stp);
+
+	stpp = stdp->std_get;
+	count = stdp->std_count;
+
+	while (count != 0) {
+		sfxge_tx_packet_t *next;
+
+		next = stpp->stp_next;
+		stpp->stp_next = NULL;
+
+		/* Discard the packet */
+		freemsg(stpp->stp_mp);
+		stpp->stp_mp = NULL;
+
+		/* Free the packet structure */
+		stpp->stp_etherhp = NULL;
+		stpp->stp_iphp = NULL;
+		stpp->stp_thp = NULL;
+		stpp->stp_off = 0;
+		stpp->stp_size = 0;
+		stpp->stp_mss = 0;
+		stpp->stp_dpl_put_len = 0;
+
+		sfxge_tx_packet_destroy(sp, stpp);
+
+		--count;
+		stpp = next;
+	}
+
+	ASSERT3P(stpp, ==, NULL);
+
+	/* Empty list */
+	stdp->std_get = NULL;
+	stdp->std_count = 0;
+	stdp->std_getp = &(stdp->std_get);
+}
+
+
+void
+sfxge_tx_qdpl_flush(sfxge_txq_t *stp)
+{
+	mutex_enter(&(stp->st_lock));
+	sfxge_tx_qdpl_flush_locked(stp);
+	mutex_exit(&(stp->st_lock));
+}
+
+
+static void
+sfxge_tx_qunblock(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	unsigned int evq = stp->st_evq;
+	sfxge_evq_t *sep = sp->s_sep[evq];
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	if (stp->st_state != SFXGE_TXQ_STARTED)
+		return;
+
+	mutex_enter(&(stp->st_lock));
+
+	if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) {
+		unsigned int level;
+
+		level = stp->st_added - stp->st_completed;
+		if (level <= stp->st_unblock) {
+			stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;
+			sfxge_tx_qlist_post(stp);
+		}
+	}
+
+	sfxge_tx_qdpl_service(stp);
+	/* lock has been dropped */
+}
+
+void
+sfxge_tx_qcomplete(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+	unsigned int evq = stp->st_evq;
+	sfxge_evq_t *sep = sp->s_sep[evq];
+	unsigned int completed;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	completed = stp->st_completed;
+	while (completed != stp->st_pending) {
+		unsigned int id;
+		sfxge_tx_mapping_t *stmp;
+
+		id = completed++ & (SFXGE_TX_NDESCS - 1);
+
+		if ((stmp = stp->st_stmp[id]) != NULL) {
+			mblk_t *mp;
+
+			/* Unbind all the mappings */
+			do {
+				ASSERT(stmp->stm_mp != NULL);
+				sfxge_tx_msgb_unbind(stmp);
+
+				stmp = stmp->stm_next;
+			} while (stmp != NULL);
+
+			/*
+			 * Now that the packet is no longer mapped for DMA it
+			 * can be freed.
+			 */
+			mp = stp->st_mp[id];
+			stp->st_mp[id] = NULL;
+
+			ASSERT(mp != NULL);
+			freemsg(mp);
+		}
+	}
+	stp->st_completed = completed;
+
+	/* Check whether we need to unblock the queue */
+	if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) {
+		unsigned int level;
+
+		level = stp->st_added - stp->st_completed;
+		if (level <= stp->st_unblock)
+			sfxge_tx_qunblock(stp);
+	}
+
+	/* Release TX backpressure from the TX DPL put/get list being full */
+	if (stdp->std_count < stdp->get_pkt_limit)
+		mac_tx_update(sp->s_mh);
+}
+
+void
+sfxge_tx_qflush_done(sfxge_txq_t *stp)
+{
+	sfxge_t *sp = stp->st_sp;
+
+	ASSERT(mutex_owned(&(sp->s_sep[stp->st_evq]->se_lock)));
+
+	mutex_enter(&(stp->st_lock));
+
+	if (stp->st_flush == SFXGE_FLUSH_PENDING)
+		stp->st_flush = SFXGE_FLUSH_DONE;
+
+	mutex_exit(&(stp->st_lock));
+
+	mutex_enter(&(sp->s_tx_flush_lock));
+	sp->s_tx_flush_pending--;
+	if (sp->s_tx_flush_pending <= 0) {
+		/* All queues flushed: wakeup sfxge_tx_stop() */
+		cv_signal(&(sp->s_tx_flush_kv));
+	}
+	mutex_exit(&(sp->s_tx_flush_lock));
+}
+
+static void
+sfxge_tx_qflush(sfxge_t *sp, unsigned int index, boolean_t do_flush)
+{
+	sfxge_txq_t *stp = sp->s_stp[index];
+
+	ASSERT(mutex_owned(&(sp->s_state_lock)));
+
+	mutex_enter(&(stp->st_lock));
+
+	/* Prepare to flush and stop the queue */
+	if (stp->st_state == SFXGE_TXQ_STARTED)
+		stp->st_state = SFXGE_TXQ_INITIALIZED;
+	else
+		do_flush = B_FALSE; /* No hardware ring, so don't flush */
+
+	if (do_flush)
+		stp->st_flush = SFXGE_FLUSH_PENDING;
+	else
+		stp->st_flush = SFXGE_FLUSH_INACTIVE;
+
+	mutex_exit(&(stp->st_lock));
+
+	/* Flush the transmit queue */
+	if (do_flush)
+		efx_tx_qflush(stp->st_etp);
+}
+
+static void
+sfxge_tx_qstop(sfxge_t *sp, unsigned int index)
+{
+	sfxge_txq_t *stp = sp->s_stp[index];
+	unsigned int evq = stp->st_evq;
+	sfxge_evq_t *sep = sp->s_sep[evq];
+
+	mutex_enter(&(sep->se_lock));
+	mutex_enter(&(stp->st_lock));
+	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED);
+
+	/* All queues should have been flushed */
+	ASSERT3S(stp->st_sp->s_tx_flush_pending, ==, 0);
+	ASSERT(stp->st_flush != SFXGE_FLUSH_FAILED);
+
+	/* in case of TX flush timeout */
+	stp->st_flush = SFXGE_FLUSH_DONE;
+
+	/* Destroy the transmit queue */
+	efx_tx_qdestroy(stp->st_etp);
+	stp->st_etp = NULL;
+
+	/* Clear entries from the buffer table */
+	sfxge_sram_buf_tbl_clear(sp, stp->st_id,
+	    EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));
+
+	sfxge_tx_qlist_abort(stp);
+	ASSERT3U(stp->st_n, ==, 0);
+
+	stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;
+
+	stp->st_pending = stp->st_added;
+
+	sfxge_tx_qcomplete(stp);
+	ASSERT3U(stp->st_completed, ==, stp->st_pending);
+
+	sfxge_tx_qreap(stp);
+	ASSERT3U(stp->st_reaped, ==, stp->st_completed);
+
+	/*
+	 * Ensure the deferred packet list is cleared
+	 * Can race with sfxge_tx_packet_add() adding to the put list
+	 */
+	sfxge_tx_qdpl_flush_locked(stp);
+
+	stp->st_added = 0;
+	stp->st_pending = 0;
+	stp->st_completed = 0;
+	stp->st_reaped = 0;
+
+	mutex_exit(&(stp->st_lock));
+	mutex_exit(&(sep->se_lock));
+}
+
+static void
+sfxge_tx_qfini(sfxge_t *sp, unsigned int index)
+{
+	sfxge_txq_t *stp = sp->s_stp[index];
+	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
+
+	/* Detach the TXQ from the driver */
+	sp->s_stp[index] = NULL;
+	ASSERT(sp->s_tx_qcount > 0);
+	sp->s_tx_qcount--;
+
+	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED);
+	stp->st_state = SFXGE_TXQ_UNINITIALIZED;
+
+	/* Tear down the statistics */
+	sfxge_tx_kstat_fini(stp);
+
+	/* Ensure the deferred packet list is empty */
+	ASSERT3U(stdp->std_count, ==, 0);
+	ASSERT3P(stdp->std_get, ==, NULL);
+	ASSERT3U(stdp->std_put, ==, 0);
+
+	/* Clear the free buffer pool */
+	sfxge_tx_qfbp_empty(stp);
+
+	/* Clear the free mapping pool */
+	sfxge_tx_qfmp_empty(stp);
+
+	/* Clear the free packet pool */
+	sfxge_tx_qfpp_empty(stp);
+
+	mutex_destroy(&(stp->st_lock));
+
+	stp->st_evq = 0;
+	stp->st_type = 0;
+	stp->st_index = 0;
+
+	kmem_cache_free(sp->s_tqc, stp);
+}
+
+int
+sfxge_tx_init(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	char name[MAXNAMELEN];
+	int index;
+	int rc;
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_tx_packet_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
+
+	sp->s_tpc = kmem_cache_create(name, sizeof (sfxge_tx_packet_t),
+	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_packet_ctor, sfxge_tx_packet_dtor,
+	    NULL, sp, NULL, 0);
+	ASSERT(sp->s_tpc != NULL);
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_tx_buffer_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
+
+	sp->s_tbc = kmem_cache_create(name, sizeof (sfxge_tx_buffer_t),
+	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_buffer_ctor, sfxge_tx_buffer_dtor,
+	    NULL, sp, NULL, 0);
+	ASSERT(sp->s_tbc != NULL);
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_tx_mapping_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
+
+	sp->s_tmc = kmem_cache_create(name, sizeof (sfxge_tx_mapping_t),
+	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_mapping_ctor, sfxge_tx_mapping_dtor,
+	    NULL, sp, NULL, 0);
+	ASSERT(sp->s_tmc != NULL);
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_txq_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
+
+	sp->s_tqc = kmem_cache_create(name, sizeof (sfxge_txq_t),
+	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_qctor, sfxge_tx_qdtor, NULL, sp,
+	    NULL, 0);
+	ASSERT(sp->s_tqc != NULL);
+
+	/* Initialize the special non-checksummed transmit queues */
+
+	/* NB sfxge_ev_qinit() is sensitive to using EVQ_0 */
+	if ((rc = sfxge_tx_qinit(sp, SFXGE_TXQ_NON_CKSUM,
+		    SFXGE_TXQ_NON_CKSUM, EVQ_0)) != 0)
+		goto fail1;
+
+	/* NB sfxge_ev_qinit() is sensitive to using EVQ_0 */
+	if ((rc = sfxge_tx_qinit(sp, SFXGE_TXQ_IP_CKSUM,
+	    SFXGE_TXQ_IP_CKSUM, EVQ_0)) != 0)
+		goto fail2;
+
+	/* Initialize the normal transmit queues */
+	for (index = 0; index < sip->si_nalloc; index++) {
+		if ((rc = sfxge_tx_qinit(sp, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index,
+		    SFXGE_TXQ_IP_TCP_UDP_CKSUM, index)) != 0)
+			goto fail3;
+	}
+
+	return (0);
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	while (--index >= 0)
+		sfxge_tx_qfini(sp, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index);
+
+	sfxge_tx_qfini(sp, SFXGE_TXQ_IP_CKSUM);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	sfxge_tx_qfini(sp, SFXGE_TXQ_NON_CKSUM);
+
+	kmem_cache_destroy(sp->s_tqc);
+	sp->s_tqc = NULL;
+
+	kmem_cache_destroy(sp->s_tmc);
+	sp->s_tmc = NULL;
+
+	kmem_cache_destroy(sp->s_tbc);
+	sp->s_tbc = NULL;
+
+	kmem_cache_destroy(sp->s_tpc);
+	sp->s_tpc = NULL;
+
+	return (rc);
+}
+
+int
+sfxge_tx_start(sfxge_t *sp)
+{
+	efx_nic_t *enp = sp->s_enp;
+	int index;
+	int rc;
+
+	/* Initialize the transmit module */
+	if ((rc = efx_tx_init(enp)) != 0)
+		goto fail1;
+
+	for (index = 0; index < sp->s_tx_qcount; index++) {
+		if ((rc = sfxge_tx_qstart(sp, index)) != 0)
+			goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	while (--index >= 0)
+		sfxge_tx_qstop(sp, index);
+
+	/* Tear down the transmit module */
+	efx_tx_fini(enp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+
+/*
+ * Add a packet to the TX Deferred Packet List and if the TX queue lock
+ * can be acquired then call sfxge_tx_qdpl_service() to fragment and push
+ * to the H/W transmit descriptor ring
+ *
+ * If ENOSPC is returned then the DPL is full or the packet create failed, but
+ * the mblk isn't freed so that the caller can return this mblk from mc_tx() to
+ * back-pressure the OS stack.
+ *
+ * For all other errors the mblk is freed
+ */
+int
+sfxge_tx_packet_add(sfxge_t *sp, mblk_t *mp)
+{
+	struct ether_header *etherhp;
+	struct ip *iphp;
+	struct tcphdr *thp;
+	size_t off;
+	size_t size;
+	size_t mss;
+	sfxge_txq_t *stp;
+	boolean_t locked;
+	sfxge_tx_packet_t *stpp;
+	int rc = 0;
+
+	ASSERT3P(mp->b_next, ==, NULL);
+	ASSERT(!(DB_CKSUMFLAGS(mp) & HCK_PARTIALCKSUM));
+
+	/*
+	 * Do not enqueue packets during startup/shutdown;
+	 *
+	 * NOTE: This access to the state is NOT protected by the state lock. It
+	 * is an imperfect test and anything further getting onto the get/put
+	 * deferred packet lists is cleaned up in (possibly repeated) calls to
+	 * sfxge_can_destroy().
+	 */
+	if (sp->s_state != SFXGE_STARTED) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	etherhp = NULL;
+	iphp = NULL;
+	thp = NULL;
+	off = 0;
+	size = 0;
+	mss = 0;
+
+	/* Check whether we need the header pointers for LSO segmentation */
+	if (DB_LSOFLAGS(mp) & HW_LSO) {
+		/* LSO segmentation relies on hardware checksum offload */
+		DB_CKSUMFLAGS(mp) |= HCK_FULLCKSUM;
+
+		if ((mss = DB_LSOMSS(mp)) == 0) {
+			rc = EINVAL;
+			goto fail1;
+		}
+
+		sfxge_tcp_parse(mp, &etherhp, &iphp, &thp, &off, &size);
+
+		if (etherhp == NULL ||
+		    iphp == NULL ||
+		    thp == NULL ||
+		    off == 0) {
+			rc = EINVAL;
+			goto fail2;
+		}
+	}
+
+	/* Choose the appropriate transit queue */
+	if (DB_CKSUMFLAGS(mp) & HCK_FULLCKSUM) {
+		sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
+
+		if (srsp->srs_state == SFXGE_RX_SCALE_STARTED) {
+			uint16_t hash;
+			int index;
+
+			if (srsp->srs_count > 1) {
+				/*
+				 * If we have not already parsed the headers
+				 * for LSO segmentation then we need to do it
+				 * now so we can calculate the hash.
+				 */
+				if (thp == NULL)
+					sfxge_tcp_parse(mp, &etherhp, &iphp,
+					    &thp, &off, &size);
+
+				if (thp != NULL) {
+					SFXGE_TCP_HASH(
+					    ntohl(iphp->ip_dst.s_addr),
+					    ntohs(thp->th_dport),
+					    ntohl(iphp->ip_src.s_addr),
+					    ntohs(thp->th_sport), hash);
+
+					index = srsp->srs_tbl[hash %
+					    SFXGE_RX_SCALE_MAX];
+				} else {
+					/*
+					 * Non-TCP traffix always goes to the
+					 * the queue in the zero-th entry of
+					 * the RSS table.
+					 */
+					index = srsp->srs_tbl[0];
+				}
+			} else {
+				/*
+				 * It does not matter what the hash is
+				 * because all the RSS table entries will be
+				 * the same.
+				 */
+				index = srsp->srs_tbl[0];
+			}
+
+			/*
+			 * Find the event queue corresponding to the hash in
+			 * the RSS table.
+			 */
+			stp = sp->s_stp[SFXGE_TXQ_IP_TCP_UDP_CKSUM + index];
+			ASSERT3U(stp->st_evq, ==, index);
+		} else {
+			stp = sp->s_stp[SFXGE_TXQ_IP_TCP_UDP_CKSUM];
+		}
+	} else if (DB_CKSUMFLAGS(mp) & HCK_IPV4_HDRCKSUM) {
+		stp = sp->s_stp[SFXGE_TXQ_IP_CKSUM];
+	} else {
+		if ((stp = sp->s_stp[SFXGE_TXQ_NON_CKSUM]) == NULL)
+			stp = sp->s_stp[SFXGE_TXQ_IP_CKSUM];
+	}
+	ASSERT(stp != NULL);
+
+	ASSERT(mss == 0 || (DB_LSOFLAGS(mp) & HW_LSO));
+
+	/* Try to grab the lock */
+	locked = mutex_tryenter(&(stp->st_lock));
+
+	if (locked) {
+		/* Try to grab a packet from the pool */
+		stpp = sfxge_tx_qfpp_get(stp);
+	} else {
+		stpp = NULL;
+	}
+
+	if (stpp == NULL) {
+		/*
+		 * Either the pool was empty or we don't have the lock so
+		 * allocate a new packet.
+		 */
+		if ((stpp = sfxge_tx_packet_create(sp)) == NULL) {
+			rc = ENOSPC;
+			goto fail3;
+		}
+	}
+
+	stpp->stp_mp = mp;
+	stpp->stp_etherhp = etherhp;
+	stpp->stp_iphp = iphp;
+	stpp->stp_thp = thp;
+	stpp->stp_off = off;
+	stpp->stp_size = size;
+	stpp->stp_mss = mss;
+	stpp->stp_dpl_put_len = 0;
+
+	rc = sfxge_tx_qdpl_add(stp, stpp, locked);
+	if (rc != 0) {
+		/* ENOSPC can happen for DPL get or put list is full */
+		ASSERT3U(rc, ==, ENOSPC);
+
+		/*
+		 * Note; if this is the unlocked DPL put list full case there is
+		 * no need to worry about a race with locked
+		 * sfxge_tx_qdpl_swizzle() as we know that the TX DPL put list
+		 * was full and would have been swizzle'd to the TX DPL get
+		 * list; hence guaranteeing future TX completions and calls
+		 * to mac_tx_update() via sfxge_tx_qcomplete()
+		 */
+		goto fail4;
+	}
+
+	/* Try to grab the lock again */
+	if (!locked)
+		locked = mutex_tryenter(&(stp->st_lock));
+
+	if (locked) {
+		/* Try to service the list */
+		sfxge_tx_qdpl_service(stp);
+		/* lock has been dropped */
+	}
+
+	return (0);
+
+fail4:
+	DTRACE_PROBE(fail4);
+	sfxge_tx_packet_destroy(sp, stpp);
+fail3:
+	DTRACE_PROBE(fail3);
+	if (locked)
+		mutex_exit(&(stp->st_lock));
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	if (rc != ENOSPC)
+		freemsg(mp);
+	return (rc);
+}
+
+int
+sfxge_tx_loopback(sfxge_t *sp, unsigned int count)
+{
+	uint8_t unicst[ETHERADDRL];
+	size_t mtu;
+	mblk_t *mp;
+	struct ether_header *etherhp;
+	unsigned int byte;
+	int rc;
+
+	if (count == 0) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	rc = sfxge_mac_unicst_get(sp, SFXGE_UNICST_LAA, unicst);
+
+	if (rc == ENOENT)
+		rc = sfxge_mac_unicst_get(sp, SFXGE_UNICST_BIA, unicst);
+
+	if (rc != 0)
+		goto fail2;
+
+	mtu = sp->s_mtu;
+
+	if ((mp = allocb(sizeof (struct ether_header) + mtu,
+	    BPRI_HI)) == NULL) {
+		rc = ENOMEM;
+		goto fail3;
+	}
+
+	mp->b_wptr = mp->b_rptr + sizeof (struct ether_header);
+	bzero(mp->b_rptr, MBLKL(mp));
+
+	/*LINTED*/
+	etherhp = (struct ether_header *)(mp->b_rptr);
+	bcopy(sfxge_brdcst, &(etherhp->ether_dhost), ETHERADDRL);
+	bcopy(unicst, &(etherhp->ether_shost), ETHERADDRL);
+	etherhp->ether_type = htons(SFXGE_ETHERTYPE_LOOPBACK);
+
+	for (byte = 0; byte < 30; byte++)
+		*(mp->b_wptr++) = (byte & 1) ? 0xaa : 0x55;
+
+	do {
+		mblk_t *nmp;
+
+		if ((nmp = dupb(mp)) == NULL) {
+			rc = ENOMEM;
+			goto fail4;
+		}
+
+		rc = sfxge_tx_packet_add(sp, nmp);
+		if (rc != 0) {
+			freeb(nmp);
+			goto fail5;
+		}
+
+	} while (--count != 0);
+
+	freeb(mp);
+	return (0);
+
+fail5:
+	DTRACE_PROBE(fail5);
+fail4:
+	DTRACE_PROBE(fail4);
+
+	freeb(mp);
+
+fail3:
+	DTRACE_PROBE(fail3);
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+int
+sfxge_tx_ioctl(sfxge_t *sp, sfxge_tx_ioc_t *stip)
+{
+	int rc;
+
+	switch (stip->sti_op) {
+	case SFXGE_TX_OP_LOOPBACK: {
+		unsigned int count = stip->sti_data;
+
+		if ((rc = sfxge_tx_loopback(sp, count)) != 0)
+			goto fail1;
+
+		break;
+	}
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+void
+sfxge_tx_stop(sfxge_t *sp)
+{
+	efx_nic_t *enp = sp->s_enp;
+	clock_t timeout;
+	boolean_t do_flush;
+	int index;
+
+	ASSERT(mutex_owned(&(sp->s_state_lock)));
+
+	mutex_enter(&(sp->s_tx_flush_lock));
+
+	/* Flush all the queues */
+	if (sp->s_hw_err == SFXGE_HW_OK) {
+		sp->s_tx_flush_pending = sp->s_tx_qcount;
+		do_flush = B_TRUE;
+	} else {
+		sp->s_tx_flush_pending = 0;
+		do_flush = B_FALSE;
+	}
+
+	/* Prepare queues to stop and flush the hardware ring */
+	for (index = 0; index < sp->s_tx_qcount; index++)
+		sfxge_tx_qflush(sp, index, do_flush);
+
+	if (do_flush == B_FALSE)
+		goto flush_done;
+
+	/* Wait upto 2sec for queue flushing to complete */
+	timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_TX_QFLUSH_USEC);
+
+	while (sp->s_tx_flush_pending > 0) {
+		if (cv_timedwait(&(sp->s_tx_flush_kv), &(sp->s_tx_flush_lock),
+			timeout) < 0) {
+			/* Timeout waiting for queues to flush */
+			dev_info_t *dip = sp->s_dip;
+
+			DTRACE_PROBE(timeout);
+			cmn_err(CE_NOTE,
+			    SFXGE_CMN_ERR "[%s%d] tx qflush timeout",
+			    ddi_driver_name(dip), ddi_get_instance(dip));
+			break;
+		}
+	}
+	sp->s_tx_flush_pending = 0;
+
+flush_done:
+	mutex_exit(&(sp->s_tx_flush_lock));
+
+	/* Stop all the queues */
+	for (index = 0; index < sp->s_tx_qcount; index++)
+		sfxge_tx_qstop(sp, index);
+
+	/* Tear down the transmit module */
+	efx_tx_fini(enp);
+}
+
+void
+sfxge_tx_fini(sfxge_t *sp)
+{
+	int index;
+
+	index = sp->s_tx_qcount;
+	while (--index >= 0)
+		sfxge_tx_qfini(sp, index);
+
+	kmem_cache_destroy(sp->s_tqc);
+	sp->s_tqc = NULL;
+
+	kmem_cache_destroy(sp->s_tmc);
+	sp->s_tmc = NULL;
+
+	kmem_cache_destroy(sp->s_tbc);
+	sp->s_tbc = NULL;
+
+	kmem_cache_destroy(sp->s_tpc);
+	sp->s_tpc = NULL;
+}
