--- /dev/null	Tue May  3 20:52:16 2016
+++ new/usr/src/uts/common/io/sfxge/sfxge_ev.c	Tue May  3 20:52:16 2016
@@ -0,0 +1,1354 @@
+/*
+ * Copyright (c) 2008-2015 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the FreeBSD Project.
+ */
+
+#include <sys/types.h>
+#include <sys/sysmacros.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/stream.h>
+#include <sys/strsun.h>
+#include <sys/strsubr.h>
+#include <sys/cpu.h>
+#include <sys/pghw.h>
+
+#include "sfxge.h"
+
+#include "efx.h"
+
+
+/* Timeout to wait for DRIVER_EV_START event at EVQ startup */
+#define	SFXGE_EV_QSTART_TIMEOUT_USEC	(2000000)
+
+
+/* Event queue DMA attributes */
+static ddi_device_acc_attr_t sfxge_evq_devacc = {
+
+	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
+	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
+	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
+};
+
+static ddi_dma_attr_t sfxge_evq_dma_attr = {
+	DMA_ATTR_V0,		/* dma_attr_version	*/
+	0,			/* dma_attr_addr_lo	*/
+	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
+	0xffffffffffffffffull,	/* dma_attr_count_max	*/
+	EFX_BUF_SIZE,		/* dma_attr_align	*/
+	0xffffffff,		/* dma_attr_burstsizes	*/
+	1,			/* dma_attr_minxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
+	0xffffffffffffffffull,	/* dma_attr_seg		*/
+	1,			/* dma_attr_sgllen	*/
+	1,			/* dma_attr_granular	*/
+	0			/* dma_attr_flags	*/
+};
+
+static int
+_sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size)
+{
+	efsys_mem_t *esmp = &(sep->se_mem);
+	sfxge_dma_buffer_attr_t dma_attr;
+	int rc;
+
+	/* Compile-time structure layout checks */
+	EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <=
+	    sizeof (sep->__se_u1.__se_pad));
+	EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <=
+	    sizeof (sep->__se_u2.__se_pad));
+	EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <=
+	    sizeof (sep->__se_u3.__se_pad));
+
+	bzero(sep, sizeof (sfxge_evq_t));
+
+	sep->se_sp = sp;
+
+	dma_attr.sdba_dip	 = sp->s_dip;
+	dma_attr.sdba_dattrp	 = &sfxge_evq_dma_attr;
+	dma_attr.sdba_callback	 = (kmflags == KM_SLEEP) ?
+	    DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
+	dma_attr.sdba_length	 = EFX_EVQ_SIZE(evq_size);
+	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
+	dma_attr.sdba_devaccp	 = &sfxge_evq_devacc;
+	dma_attr.sdba_bindflags	 = DDI_DMA_READ | DDI_DMA_CONSISTENT;
+	dma_attr.sdba_maxcookies = 1;
+	dma_attr.sdba_zeroinit	 = B_FALSE;
+
+	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
+		goto fail1;
+
+	/* Allocate some buffer table entries */
+	if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size),
+	    &(sep->se_id))) != 0)
+		goto fail2;
+
+	sep->se_stpp = &(sep->se_stp);
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	/* Tear down DMA setup */
+	esmp->esm_addr = 0;
+	sfxge_dma_buffer_destroy(esmp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	sep->se_sp = NULL;
+
+	SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
+
+	return (-1);
+}
+
+static int
+sfxge_ev_q0ctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_evq_t *sep = buf;
+	sfxge_t *sp = arg;
+	return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size));
+}
+
+static int
+sfxge_ev_qXctor(void *buf, void *arg, int kmflags)
+{
+	sfxge_evq_t *sep = buf;
+	sfxge_t *sp = arg;
+	return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size));
+}
+static void
+_sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size)
+{
+	efsys_mem_t *esmp = &(sep->se_mem);
+	ASSERT3P(sep->se_sp, ==, sp);
+	ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
+	sep->se_stpp = NULL;
+
+	/* Free the buffer table entries */
+	sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
+	sep->se_id = 0;
+
+	/* Tear down DMA setup */
+	sfxge_dma_buffer_destroy(esmp);
+
+	sep->se_sp = NULL;
+
+	SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
+}
+
+static void
+sfxge_ev_q0dtor(void *buf, void *arg)
+{
+	sfxge_evq_t *sep = buf;
+	sfxge_t *sp = arg;
+	_sfxge_ev_qdtor(sp, sep, sp->s_evq0_size);
+}
+
+static void
+sfxge_ev_qXdtor(void *buf, void *arg)
+{
+	sfxge_evq_t *sep = buf;
+	sfxge_t *sp = arg;
+	_sfxge_ev_qdtor(sp, sep, sp->s_evqX_size);
+}
+
+static boolean_t
+sfxge_ev_initialized(void *arg)
+{
+	sfxge_evq_t *sep = arg;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* Init done events may be duplicated on 7xxx (see SFCbug31631) */
+	if (sep->se_state == SFXGE_EVQ_STARTED)
+		goto done;
+
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING);
+	sep->se_state = SFXGE_EVQ_STARTED;
+
+	cv_broadcast(&(sep->se_init_kv));
+
+done:
+	return (B_FALSE);
+}
+
+static void
+sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop)
+{
+	sfxge_t *sp = sep->se_sp;
+	unsigned int index = sep->se_index;
+	sfxge_rxq_t *srp = sp->s_srp[index];
+	sfxge_txq_t *stp;
+
+	if ((stp = sep->se_stp) != NULL) {
+		sep->se_stp = NULL;
+		sep->se_stpp = &(sep->se_stp);
+
+		do {
+			sfxge_txq_t *next;
+
+			next = stp->st_next;
+			stp->st_next = NULL;
+
+			ASSERT3U(stp->st_evq, ==, index);
+
+			if (stp->st_pending != stp->st_completed)
+				sfxge_tx_qcomplete(stp);
+
+			stp = next;
+		} while (stp != NULL);
+	}
+
+	if (srp != NULL) {
+		if (srp->sr_pending != srp->sr_completed)
+			sfxge_rx_qcomplete(srp, eop);
+	}
+}
+
+static boolean_t
+sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
+    uint16_t flags)
+{
+	sfxge_evq_t *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+	sfxge_rxq_t *srp;
+	sfxge_rx_packet_t *srpp;
+	unsigned int prefetch;
+	unsigned int stop;
+	unsigned int delta;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	if (sep->se_exception)
+		goto done;
+
+	srp = sp->s_srp[label];
+	if (srp == NULL)
+		goto done;
+
+	ASSERT3U(sep->se_index, ==, srp->sr_index);
+	ASSERT3U(id, <, sp->s_rxq_size);
+
+	/*
+	 * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset
+	 * So the return missing srp->sr_pending increase is safe
+	 */
+	if (srp->sr_state != SFXGE_RXQ_STARTED)
+		goto done;
+
+	stop = (id + 1) & (sp->s_rxq_size - 1);
+	id = srp->sr_pending & (sp->s_rxq_size - 1);
+
+	delta = (stop >= id) ? (stop - id) : (sp->s_rxq_size - id + stop);
+	srp->sr_pending += delta;
+
+	if (delta != 1) {
+		if ((!efx_nic_cfg_get(sp->s_enp)->enc_rx_batching_enabled) ||
+		    (delta == 0) ||
+		    (delta > efx_nic_cfg_get(sp->s_enp)->enc_rx_batch_max)) {
+			/*
+			 * FIXME: This does not take into account scatter
+			 * aborts.  See Bug40811
+			 */
+			sep->se_exception = B_TRUE;
+
+			DTRACE_PROBE(restart_ev_rx_id);
+			/* sfxge_evq_t->se_lock held */
+			(void) sfxge_restart_dispatch(sp, DDI_SLEEP,
+			    SFXGE_HW_ERR, "Out of order RX event", delta);
+
+			goto done;
+		}
+	}
+
+	prefetch = (id + 4) & (sp->s_rxq_size - 1);
+	if ((srpp = srp->sr_srpp[prefetch]) != NULL)
+		prefetch_read_many(srpp);
+
+	srpp = srp->sr_srpp[id];
+	ASSERT(srpp != NULL);
+	prefetch_read_many(srpp->srp_mp);
+
+	for (; id != stop; id = (id + 1) & (sp->s_rxq_size - 1)) {
+		srpp = srp->sr_srpp[id];
+		ASSERT(srpp != NULL);
+
+		ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD);
+		srpp->srp_flags = flags;
+
+		ASSERT3U(size, <, (1 << 16));
+		srpp->srp_size = (uint16_t)size;
+	}
+
+	sep->se_rx++;
+
+	DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index,
+	    unsigned int, srp->sr_added - srp->sr_pending);
+
+	if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH)
+		sfxge_ev_qcomplete(sep, B_FALSE);
+
+done:
+	/* returning B_TRUE makes efx_ev_qpoll() stop processing events */
+	return (sep->se_rx >= sep->se_ev_batch);
+}
+
+static boolean_t
+sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
+{
+	sfxge_evq_t *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+
+	_NOTE(ARGUNUSED(code))
+	_NOTE(ARGUNUSED(data))
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+	sep->se_exception = B_TRUE;
+
+	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
+
+		DTRACE_PROBE(restart_ev_exception);
+
+		/* sfxge_evq_t->se_lock held */
+		(void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR,
+		    "Unknown EV", code);
+	}
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
+{
+	sfxge_evq_t *sep_targetq, *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+	sfxge_rxq_t *srp;
+	unsigned int index;
+	unsigned int label;
+	uint16_t magic;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
+	srp = sp->s_srp[rxq_index];
+	if (srp == NULL)
+		goto done;
+
+	/* Process right now if it is the correct event queue */
+	index = srp->sr_index;
+	if (index == sep->se_index) {
+		sfxge_rx_qflush_done(srp);
+		goto done;
+	}
+
+	/* Resend a software event on the correct queue */
+	sep_targetq = sp->s_sep[index];
+
+	if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
+		goto done; /* TBD: state test not under the lock */
+
+	label = rxq_index;
+	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
+	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
+
+	efx_ev_qpost(sep_targetq->se_eep, magic);
+
+done:
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
+{
+	sfxge_evq_t *sep_targetq, *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+	sfxge_rxq_t *srp;
+	unsigned int index;
+	unsigned int label;
+	uint16_t magic;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
+	srp = sp->s_srp[rxq_index];
+	if (srp == NULL)
+		goto done;
+
+	/* Process right now if it is the correct event queue */
+	index = srp->sr_index;
+	if (index == sep->se_index) {
+		sfxge_rx_qflush_failed(srp);
+		goto done;
+	}
+
+	/* Resend a software event on the correct queue */
+	sep_targetq = sp->s_sep[index];
+
+	label = rxq_index;
+	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
+	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
+
+	if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
+		goto done; /* TBD: state test not under the lock */
+
+	efx_ev_qpost(sep_targetq->se_eep, magic);
+
+done:
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
+{
+	sfxge_evq_t *sep = arg;
+	sfxge_txq_t *stp;
+	unsigned int stop;
+	unsigned int delta;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	stp = sep->se_label_stp[label];
+	if (stp == NULL)
+		goto done;
+
+	if (stp->st_state != SFXGE_TXQ_STARTED)
+		goto done;
+
+	ASSERT3U(sep->se_index, ==, stp->st_evq);
+
+	stop = (id + 1) & (SFXGE_TX_NDESCS - 1);
+	id = stp->st_pending & (SFXGE_TX_NDESCS - 1);
+
+	delta = (stop >= id) ? (stop - id) : (SFXGE_TX_NDESCS - id + stop);
+	stp->st_pending += delta;
+
+	sep->se_tx++;
+
+	if (stp->st_next == NULL &&
+	    sep->se_stpp != &(stp->st_next)) {
+		*(sep->se_stpp) = stp;
+		sep->se_stpp = &(stp->st_next);
+	}
+
+	DTRACE_PROBE2(qlevel, unsigned int, stp->st_index,
+	    unsigned int, stp->st_added - stp->st_pending);
+
+	if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH)
+		sfxge_tx_qcomplete(stp);
+
+done:
+	/* returning B_TRUE makes efx_ev_qpoll() stop processing events */
+	return (sep->se_tx >= sep->se_ev_batch);
+}
+
+static boolean_t
+sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
+{
+	sfxge_evq_t *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+	sfxge_txq_t *stp;
+	unsigned int evq;
+	unsigned int label;
+	uint16_t magic;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	/* Ensure TXQ exists, as events may arrive after TXQ was destroyed */
+	stp = sp->s_stp[txq_index];
+	if (stp == NULL)
+		goto done;
+
+	/* Process right now if it is the correct event queue */
+	evq = stp->st_evq;
+	if (evq == sep->se_index) {
+		sfxge_tx_qflush_done(stp);
+		goto done;
+	}
+
+	/* Resend a software event on the correct queue */
+	sep = sp->s_sep[evq];
+
+	label = stp->st_label;
+
+	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
+	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
+
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
+	efx_ev_qpost(sep->se_eep, magic);
+
+done:
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_software(void *arg, uint16_t magic)
+{
+	sfxge_evq_t *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+	dev_info_t *dip = sp->s_dip;
+	unsigned int label;
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
+	    FSF_AZ_RX_EV_Q_LABEL_WIDTH);
+	EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
+	    FSF_AZ_TX_EV_Q_LABEL_WIDTH);
+
+	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
+	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
+
+	switch (magic) {
+	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
+		sfxge_rxq_t *srp = sp->s_srp[label];
+
+		if (srp != NULL) {
+			ASSERT3U(sep->se_index, ==, srp->sr_index);
+
+			sfxge_rx_qflush_done(srp);
+		}
+		break;
+	}
+	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
+		sfxge_rxq_t *srp = sp->s_srp[label];
+
+		if (srp != NULL) {
+			ASSERT3U(sep->se_index, ==, srp->sr_index);
+
+			sfxge_rx_qflush_failed(srp);
+		}
+		break;
+	}
+	case SFXGE_MAGIC_RX_QFPP_TRIM: {
+		sfxge_rxq_t *srp = sp->s_srp[label];
+
+		if (srp != NULL) {
+			ASSERT3U(sep->se_index, ==, srp->sr_index);
+
+			sfxge_rx_qfpp_trim(srp);
+		}
+		break;
+	}
+	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
+		sfxge_txq_t *stp = sep->se_label_stp[label];
+
+		if (stp != NULL) {
+			ASSERT3U(sep->se_index, ==, stp->st_evq);
+
+			sfxge_tx_qflush_done(stp);
+		}
+		break;
+	}
+	default:
+		dev_err(dip, CE_NOTE,
+		    SFXGE_CMN_ERR "unknown software event 0x%x", magic);
+		break;
+	}
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_sram(void *arg, uint32_t code)
+{
+	_NOTE(ARGUNUSED(arg))
+
+	switch (code) {
+	case EFX_SRAM_UPDATE:
+		DTRACE_PROBE(sram_update);
+		break;
+
+	case EFX_SRAM_CLEAR:
+		DTRACE_PROBE(sram_clear);
+		break;
+
+	case EFX_SRAM_ILLEGAL_CLEAR:
+		DTRACE_PROBE(sram_illegal_clear);
+		break;
+
+	default:
+		ASSERT(B_FALSE);
+		break;
+	}
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_timer(void *arg, uint32_t index)
+{
+	_NOTE(ARGUNUSED(arg, index))
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_wake_up(void *arg, uint32_t index)
+{
+	_NOTE(ARGUNUSED(arg, index))
+
+	return (B_FALSE);
+}
+
+#if 0
+static boolean_t
+sfxge_ev_monitor(void)
+{
+	_NOTE(ARGUNUSED(arg, id, value))
+
+	return (B_FALSE);
+}
+#endif
+
+static boolean_t
+sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
+{
+	sfxge_evq_t *sep = arg;
+	sfxge_t *sp = sep->se_sp;
+
+	sfxge_mac_link_update(sp, link_mode);
+
+	return (B_FALSE);
+}
+
+static int
+sfxge_ev_kstat_update(kstat_t *ksp, int rw)
+{
+	sfxge_evq_t *sep = ksp->ks_private;
+	kstat_named_t *knp;
+	int rc;
+
+	if (rw != KSTAT_READ) {
+		rc = EACCES;
+		goto fail1;
+	}
+
+	ASSERT(mutex_owned(&(sep->se_lock)));
+
+	if (sep->se_state != SFXGE_EVQ_STARTED)
+		goto done;
+
+	efx_ev_qstats_update(sep->se_eep, sep->se_stat);
+
+	knp = ksp->ks_data;
+	knp += EV_NQSTATS;
+
+	knp->value.ui64 = sep->se_cpu_id;
+
+done:
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_ev_kstat_init(sfxge_evq_t *sep)
+{
+	sfxge_t *sp = sep->se_sp;
+	unsigned int index = sep->se_index;
+	dev_info_t *dip = sp->s_dip;
+	kstat_t *ksp;
+	kstat_named_t *knp;
+	char name[MAXNAMELEN];
+	unsigned int id;
+	int rc;
+
+	/* Determine the name */
+	(void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d",
+	    ddi_driver_name(dip), index);
+
+	/* Create the set */
+	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
+	    ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
+	    EV_NQSTATS + 1, 0)) == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	sep->se_ksp = ksp;
+
+	ksp->ks_update = sfxge_ev_kstat_update;
+	ksp->ks_private = sep;
+	ksp->ks_lock = &(sep->se_lock);
+
+	/* Initialise the named stats */
+	sep->se_stat = knp = ksp->ks_data;
+	for (id = 0; id < EV_NQSTATS; id++) {
+		kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id),
+		    KSTAT_DATA_UINT64);
+		knp++;
+	}
+
+	kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64);
+
+	kstat_install(ksp);
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static void
+sfxge_ev_kstat_fini(sfxge_evq_t *sep)
+{
+	/* Destroy the set */
+	kstat_delete(sep->se_ksp);
+	sep->se_ksp = NULL;
+	sep->se_stat = NULL;
+}
+
+inline unsigned pow2_ge(unsigned int n) {
+	unsigned int order = 0;
+	ASSERT3U(n, >, 0);
+	while ((1ul << order) < n) ++order;
+	return (1ul << (order));
+}
+
+static int
+sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch)
+{
+	sfxge_evq_t *sep;
+	int rc;
+
+	ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
+
+	sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP);
+	if (sep == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED);
+
+	sep->se_index = index;
+
+	mutex_init(&(sep->se_lock), NULL,
+	    MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri));
+
+	cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL);
+
+	/* Initialize the statistics */
+	if ((rc = sfxge_ev_kstat_init(sep)) != 0)
+		goto fail2;
+
+	sep->se_state = SFXGE_EVQ_INITIALIZED;
+	sep->se_ev_batch = (uint16_t)ev_batch;
+	sp->s_sep[index] = sep;
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	sep->se_index = 0;
+
+	cv_destroy(&(sep->se_init_kv));
+	mutex_destroy(&(sep->se_lock));
+
+	kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static int
+sfxge_ev_qstart(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	sfxge_intr_t *sip = &(sp->s_intr);
+	efx_nic_t *enp = sp->s_enp;
+	efx_ev_callbacks_t *eecp;
+	efsys_mem_t *esmp;
+	clock_t timeout;
+	int rc;
+	uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
+
+	mutex_enter(&(sep->se_lock));
+	esmp = &(sep->se_mem);
+
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
+
+	/* Set the memory to all ones */
+	(void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size));
+
+	/* Program the buffer table */
+	if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp,
+	    EFX_EVQ_NBUFS(evq_size))) != 0)
+		goto fail1;
+
+	/* Set up the event callbacks */
+	eecp = &(sep->se_eec);
+	eecp->eec_initialized = sfxge_ev_initialized;
+	eecp->eec_rx = sfxge_ev_rx;
+	eecp->eec_tx = sfxge_ev_tx;
+	eecp->eec_exception = sfxge_ev_exception;
+	eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done;
+	eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed;
+	eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done;
+	eecp->eec_software = sfxge_ev_software;
+	eecp->eec_sram = sfxge_ev_sram;
+	eecp->eec_wake_up = sfxge_ev_wake_up;
+	eecp->eec_timer = sfxge_ev_timer;
+	eecp->eec_link_change = sfxge_ev_link_change;
+#if 0
+	eecp->eec_monitor = (void *)sfxge_ev_monitor;
+#endif
+
+	/* Create the event queue */
+	if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id,
+	    &(sep->se_eep))) != 0)
+		goto fail2;
+
+	/* Set the default moderation */
+	if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0)
+		goto fail3;
+
+	/* Check that interrupts are enabled at the NIC */
+	if (sip->si_state != SFXGE_INTR_STARTED) {
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	sep->se_state = SFXGE_EVQ_STARTING;
+
+	/* Prime the event queue for interrupts */
+	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
+		goto fail5;
+
+	/* Wait for the initialization event */
+	timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_EV_QSTART_TIMEOUT_USEC);
+	while (sep->se_state != SFXGE_EVQ_STARTED) {
+		if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock),
+		    timeout) < 0) {
+			/* Timeout waiting for initialization */
+			dev_info_t *dip = sp->s_dip;
+
+			DTRACE_PROBE(timeout);
+			dev_err(dip, CE_NOTE,
+			    SFXGE_CMN_ERR "evq[%d] qstart timeout", index);
+
+			rc = ETIMEDOUT;
+			goto fail6;
+		}
+	}
+
+	mutex_exit(&(sep->se_lock));
+	return (0);
+
+fail6:
+	DTRACE_PROBE(fail6);
+
+fail5:
+	DTRACE_PROBE(fail5);
+
+	sep->se_state = SFXGE_EVQ_INITIALIZED;
+
+fail4:
+	DTRACE_PROBE(fail4);
+
+fail3:
+	DTRACE_PROBE(fail3);
+
+	/* Destroy the event queue */
+	efx_ev_qdestroy(sep->se_eep);
+	sep->se_eep = NULL;
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	/* Zero out the event handlers */
+	bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
+
+	/* Clear entries from the buffer table */
+	sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (rc);
+}
+
+int
+sfxge_ev_qpoll(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	processorid_t cpu_id;
+	int rc;
+	uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
+
+	mutex_enter(&(sep->se_lock));
+
+	if (sep->se_state != SFXGE_EVQ_STARTING &&
+	    sep->se_state != SFXGE_EVQ_STARTED) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	/* Make sure the CPU information is up to date */
+	cpu_id = CPU->cpu_id;
+
+	if (cpu_id != sep->se_cpu_id) {
+		sep->se_cpu_id = cpu_id;
+
+		/* sfxge_evq_t->se_lock held */
+		(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
+		    DDI_NOSLEEP);
+	}
+
+	/* Synchronize the DMA memory for reading */
+	(void) ddi_dma_sync(sep->se_mem.esm_dma_handle,
+	    0,
+	    EFX_EVQ_SIZE(evq_size),
+	    DDI_DMA_SYNC_FORKERNEL);
+
+	ASSERT3U(sep->se_rx, ==, 0);
+	ASSERT3U(sep->se_tx, ==, 0);
+	ASSERT3P(sep->se_stp, ==, NULL);
+	ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
+
+	/* Poll the queue */
+	efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec),
+	    sep);
+
+	sep->se_rx = 0;
+	sep->se_tx = 0;
+
+	/* Perform any pending completion processing */
+	sfxge_ev_qcomplete(sep, B_TRUE);
+
+	/* Re-prime the event queue for interrupts */
+	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
+		goto fail2;
+
+	mutex_exit(&(sep->se_lock));
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (rc);
+}
+
+int
+sfxge_ev_qprime(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	int rc;
+
+	mutex_enter(&(sep->se_lock));
+
+	if (sep->se_state != SFXGE_EVQ_STARTING &&
+	    sep->se_state != SFXGE_EVQ_STARTED) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
+		goto fail2;
+
+	mutex_exit(&(sep->se_lock));
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (rc);
+}
+
+
+int
+sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	efx_evq_t *eep = sep->se_eep;
+
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
+
+	return (efx_ev_qmoderate(eep, us));
+}
+
+static void
+sfxge_ev_qstop(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+	uint16_t evq_size;
+
+	mutex_enter(&(sep->se_lock));
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
+	sep->se_state = SFXGE_EVQ_INITIALIZED;
+	evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
+
+	/* Clear the CPU information */
+	sep->se_cpu_id = 0;
+
+	/* Clear the event count */
+	sep->se_count = 0;
+
+	/* Reset the exception flag */
+	sep->se_exception = B_FALSE;
+
+	/* Destroy the event queue */
+	efx_ev_qdestroy(sep->se_eep);
+	sep->se_eep = NULL;
+
+	mutex_exit(&(sep->se_lock));
+
+	/* Zero out the event handlers */
+	bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
+
+	/* Clear entries from the buffer table */
+	sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
+}
+
+static void
+sfxge_ev_qfini(sfxge_t *sp, unsigned int index)
+{
+	sfxge_evq_t *sep = sp->s_sep[index];
+
+	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
+
+	sp->s_sep[index] = NULL;
+	sep->se_state = SFXGE_EVQ_UNINITIALIZED;
+
+	/* Tear down the statistics */
+	sfxge_ev_kstat_fini(sep);
+
+	cv_destroy(&(sep->se_init_kv));
+	mutex_destroy(&(sep->se_lock));
+
+	sep->se_index = 0;
+
+	kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
+}
+
+int
+sfxge_ev_txlabel_alloc(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
+    unsigned int *labelp)
+{
+	sfxge_evq_t *sep = sp->s_sep[evq];
+	sfxge_txq_t **stpp;
+	unsigned int label;
+	int rc;
+
+	mutex_enter(&(sep->se_lock));
+
+	if (stp == NULL || labelp == NULL) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	stpp = NULL;
+	for (label = 0; label < SFXGE_TX_NLABELS; label++) {
+		if (sep->se_label_stp[label] == stp) {
+			rc = EEXIST;
+			goto fail2;
+		}
+		if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) {
+			stpp = &sep->se_label_stp[label];
+		}
+	}
+	if (stpp == NULL) {
+		rc = ENOSPC;
+		goto fail3;
+	}
+	*stpp = stp;
+	label = stpp - sep->se_label_stp;
+
+	ASSERT3U(label, <, SFXGE_TX_NLABELS);
+	*labelp = label;
+
+	mutex_exit(&(sep->se_lock));
+	return (0);
+
+fail3:
+	DTRACE_PROBE(fail3);
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (rc);
+}
+
+
+int
+sfxge_ev_txlabel_free(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
+    unsigned int label)
+{
+	sfxge_evq_t *sep = sp->s_sep[evq];
+	int rc;
+
+	mutex_enter(&(sep->se_lock));
+
+	if (stp == NULL || label > SFXGE_TX_NLABELS) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (sep->se_label_stp[label] != stp) {
+		rc = EINVAL;
+		goto fail2;
+	}
+	sep->se_label_stp[label] = NULL;
+
+	mutex_exit(&(sep->se_lock));
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	mutex_exit(&(sep->se_lock));
+
+	return (rc);
+}
+
+
+static 	kmem_cache_t *
+sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname,
+    int (*ctor)(void *, void *, int), void (*dtor)(void *, void *))
+{
+	char name[MAXNAMELEN];
+	kmem_cache_t *eqc;
+
+	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache",
+	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname);
+
+	eqc = kmem_cache_create(name, sizeof (sfxge_evq_t),
+	    SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0);
+	ASSERT(eqc != NULL);
+	return (eqc);
+}
+
+int
+sfxge_ev_init(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	unsigned int evq0_size;
+	unsigned int evqX_size;
+	unsigned int ev_batch;
+	int index;
+	int rc;
+
+	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
+
+	/*
+	 * Must account for RXQ, TXQ(s); MCDI not event completed at present
+	 * Note that common code does not completely fill descriptor queues
+	 */
+	evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS;
+	evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */
+	evq0_size += SFXGE_TX_NDESCS; /* no checksums */
+
+	ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS);
+	ASSERT3U(evq0_size, >, evqX_size);
+
+	if (evq0_size > EFX_EVQ_MAXNEVS) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	sp->s_evq0_size = pow2_ge(evq0_size);
+	sp->s_evqX_size = pow2_ge(evqX_size);
+
+	/* Read driver parameters */
+	sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION);
+
+	ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
+	    DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH);
+
+	/*
+	 * It is slightly peverse to have a cache for one item. But it allows
+	 * for simple alignment control without increasing the allocation size
+	 */
+	sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor,
+	    sfxge_ev_q0dtor);
+	sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor,
+	    sfxge_ev_qXdtor);
+
+	/* Initialize the event queue(s) */
+	for (index = 0; index < sip->si_nalloc; index++) {
+		if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0)
+			goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	while (--index >= 0)
+		sfxge_ev_qfini(sp, index);
+	sp->s_ev_moderation = 0;
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	kmem_cache_destroy(sp->s_eqXc);
+	kmem_cache_destroy(sp->s_eq0c);
+	sp->s_eqXc = NULL;
+	sp->s_eq0c = NULL;
+
+	return (rc);
+}
+
+int
+sfxge_ev_start(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	efx_nic_t *enp = sp->s_enp;
+	int index;
+	int rc;
+
+	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
+
+	/* Initialize the event module */
+	if ((rc = efx_ev_init(enp)) != 0)
+		goto fail1;
+
+	/* Start the event queues */
+	for (index = 0; index < sip->si_nalloc; index++) {
+		if ((rc = sfxge_ev_qstart(sp, index)) != 0)
+			goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	DTRACE_PROBE(fail2);
+
+	/* Stop the event queue(s) */
+	while (--index >= 0)
+		sfxge_ev_qstop(sp, index);
+
+	/* Tear down the event module */
+	efx_ev_fini(enp);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+void
+sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp)
+{
+	*usp = sp->s_ev_moderation;
+}
+
+int
+sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	int index;
+	int rc;
+
+	if (sip->si_state != SFXGE_INTR_STARTED)
+		return (ENODEV);
+
+	for (index = 0; index < sip->si_nalloc; index++) {
+		if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0)
+			goto fail1;
+	}
+
+	sp->s_ev_moderation = us;
+	return (0);
+
+fail1:
+	DTRACE_PROBE1(fail1, int, rc);
+
+	/*  The only error path is if the value to set to is invalid. */
+	ASSERT3U(index, ==, 0);
+
+	return (rc);
+}
+
+void
+sfxge_ev_stop(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	efx_nic_t *enp = sp->s_enp;
+	int index;
+
+	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
+
+	/* Stop the event queue(s) */
+	index = sip->si_nalloc;
+	while (--index >= 0)
+		sfxge_ev_qstop(sp, index);
+
+	/* Tear down the event module */
+	efx_ev_fini(enp);
+}
+
+void
+sfxge_ev_fini(sfxge_t *sp)
+{
+	sfxge_intr_t *sip = &(sp->s_intr);
+	int index;
+
+	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
+
+	sp->s_ev_moderation = 0;
+
+	/* Tear down the event queue(s) */
+	index = sip->si_nalloc;
+	while (--index >= 0)
+		sfxge_ev_qfini(sp, index);
+
+	kmem_cache_destroy(sp->s_eqXc);
+	kmem_cache_destroy(sp->s_eq0c);
+	sp->s_eqXc = NULL;
+	sp->s_eq0c = NULL;
+}
