1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008-2013 Solarflare Communications Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/types.h>
28 #include <sys/sysmacros.h>
29 #include <sys/ddi.h>
30 #include <sys/sunddi.h>
31 #include <sys/stream.h>
32 #include <sys/strsun.h>
33 #include <sys/strsubr.h>
34 #include <sys/cpu.h>
35 #include <sys/pghw.h>
36
37 #include "sfxge.h"
38
39 #include "efx.h"
40
41
42 /* Event queue DMA attributes */
43 static ddi_device_acc_attr_t sfxge_evq_devacc = {
44
45 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
46 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
47 DDI_STRICTORDER_ACC /* devacc_attr_dataorder */
48 };
49
50 static ddi_dma_attr_t sfxge_evq_dma_attr = {
51 DMA_ATTR_V0, /* dma_attr_version */
52 0, /* dma_attr_addr_lo */
53 0xffffffffffffffffull, /* dma_attr_addr_hi */
54 0xffffffffffffffffull, /* dma_attr_count_max */
55 EFX_BUF_SIZE, /* dma_attr_align */
56 0xffffffff, /* dma_attr_burstsizes */
57 1, /* dma_attr_minxfer */
58 0xffffffffffffffffull, /* dma_attr_maxxfer */
59 0xffffffffffffffffull, /* dma_attr_seg */
60 1, /* dma_attr_sgllen */
61 1, /* dma_attr_granular */
62 0 /* dma_attr_flags */
63 };
64
65 static int
66 _sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size)
67 {
68 efsys_mem_t *esmp = &(sep->se_mem);
69 sfxge_dma_buffer_attr_t dma_attr;
70 int rc;
71
72 /* Compile-time structure layout checks */
73 EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <=
74 sizeof (sep->__se_u1.__se_pad));
75 EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <=
76 sizeof (sep->__se_u2.__se_pad));
77
78 bzero(sep, sizeof (sfxge_evq_t));
79
80 sep->se_sp = sp;
81
82 dma_attr.sdba_dip = sp->s_dip;
83 dma_attr.sdba_dattrp = &sfxge_evq_dma_attr;
84 dma_attr.sdba_callback = (kmflags == KM_SLEEP) ?
85 DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
86 dma_attr.sdba_length = EFX_EVQ_SIZE(evq_size);
87 dma_attr.sdba_memflags = DDI_DMA_CONSISTENT;
88 dma_attr.sdba_devaccp = &sfxge_evq_devacc;
89 dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT;
90 dma_attr.sdba_maxcookies = 1;
91 dma_attr.sdba_zeroinit = B_FALSE;
92
93 if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
94 goto fail1;
95
96 /* Allocate some buffer table entries */
97 if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size),
98 &(sep->se_id))) != 0)
99 goto fail2;
100
101 sep->se_stpp = &(sep->se_stp);
102
103 return (0);
104
105 fail2:
106 DTRACE_PROBE(fail2);
107
108 /* Tear down DMA setup */
109 esmp->esm_addr = 0;
110 sfxge_dma_buffer_destroy(esmp);
111
112 fail1:
113 DTRACE_PROBE1(fail1, int, rc);
114
115 sep->se_sp = NULL;
116
117 SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
118
119 return (-1);
120 }
121
122 static int
123 sfxge_ev_q0ctor(void *buf, void *arg, int kmflags)
124 {
125 sfxge_evq_t *sep = buf;
126 sfxge_t *sp = arg;
127 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size));
128 }
129
130 static int
131 sfxge_ev_qXctor(void *buf, void *arg, int kmflags)
132 {
133 sfxge_evq_t *sep = buf;
134 sfxge_t *sp = arg;
135 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size));
136 }
137 static void
138 _sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size)
139 {
140 efsys_mem_t *esmp = &(sep->se_mem);
141 ASSERT3P(sep->se_sp, ==, sp);
142 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
143 sep->se_stpp = NULL;
144
145 /* Free the buffer table entries */
146 sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
147 sep->se_id = 0;
148
149 /* Tear down DMA setup */
150 sfxge_dma_buffer_destroy(esmp);
151
152 sep->se_sp = NULL;
153
154 SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
155 }
156
157 static void
158 sfxge_ev_q0dtor(void *buf, void *arg)
159 {
160 sfxge_evq_t *sep = buf;
161 sfxge_t *sp = arg;
162 _sfxge_ev_qdtor(sp, sep, sp->s_evq0_size);
163 }
164
165 static void
166 sfxge_ev_qXdtor(void *buf, void *arg)
167 {
168 sfxge_evq_t *sep = buf;
169 sfxge_t *sp = arg;
170 _sfxge_ev_qdtor(sp, sep, sp->s_evqX_size);
171 }
172
173 static boolean_t
174 sfxge_ev_initialized(void *arg)
175 {
176 sfxge_evq_t *sep = arg;
177
178 ASSERT(mutex_owned(&(sep->se_lock)));
179 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING);
180 sep->se_state = SFXGE_EVQ_STARTED;
181
182 cv_broadcast(&(sep->se_init_kv));
183
184 return (B_FALSE);
185 }
186
187 static void
188 sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop)
189 {
190 sfxge_t *sp = sep->se_sp;
191 unsigned int index = sep->se_index;
192 sfxge_rxq_t *srp = sp->s_srp[index];
193 sfxge_txq_t *stp;
194
195 if ((stp = sep->se_stp) != NULL) {
196 sep->se_stp = NULL;
197 sep->se_stpp = &(sep->se_stp);
198
199 do {
200 sfxge_txq_t *next;
201
202 next = stp->st_next;
203 stp->st_next = NULL;
204
205 ASSERT3U(stp->st_evq, ==, index);
206
207 if (stp->st_pending != stp->st_completed)
208 sfxge_tx_qcomplete(stp);
209
210 stp = next;
211 } while (stp != NULL);
212 }
213
214 if (srp != NULL) {
215 if (srp->sr_pending != srp->sr_completed)
216 sfxge_rx_qcomplete(srp, eop);
217 }
218 }
219
220 static boolean_t
221 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
222 uint16_t flags)
223 {
224 sfxge_evq_t *sep = arg;
225 sfxge_t *sp = sep->se_sp;
226 sfxge_rxq_t *srp;
227 sfxge_rx_packet_t *srpp;
228 unsigned int expected;
229 unsigned int prefetch;
230
231 ASSERT(mutex_owned(&(sep->se_lock)));
232
233 if (sep->se_exception)
234 goto done;
235
236 srp = sp->s_srp[label];
237 if (srp == NULL)
238 goto done;
239
240 ASSERT3U(sep->se_index, ==, srp->sr_index);
241 ASSERT3U(id, <, sp->s_rxq_size);
242
243 /*
244 * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset
245 * So the return missing srp->sr_pending++ is safe
246 */
247 if (srp->sr_state != SFXGE_RXQ_STARTED)
248 goto done;
249
250 expected = srp->sr_pending++ & (sp->s_rxq_size - 1);
251 if (id != expected) {
252 sep->se_exception = B_TRUE;
253
254 DTRACE_PROBE(restart_ev_rx_id);
255 /* sfxge_evq_t->se_lock held */
256 (void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR,
257 "Out of order RX event", id - expected);
258
259 goto done;
260 }
261
262 prefetch = (id + 4) & (sp->s_rxq_size - 1);
263 if ((srpp = srp->sr_srpp[prefetch]) != NULL)
264 prefetch_read_many(srpp);
265
266 srpp = srp->sr_srpp[id];
267 ASSERT(srpp != NULL);
268
269 ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD);
270 srpp->srp_flags = flags;
271
272 ASSERT3U(size, <, (1 << 16));
273 srpp->srp_size = (uint16_t)size;
274
275 prefetch_read_many(srpp->srp_mp);
276
277 sep->se_rx++;
278
279 DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index,
280 unsigned int, srp->sr_added - srp->sr_pending);
281
282 if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH)
283 sfxge_ev_qcomplete(sep, B_FALSE);
284
285 done:
286 /* returning B_TRUE makes efx_ev_qpoll() stop processing events */
287 return (sep->se_rx >= sep->se_ev_batch);
288 }
289
290 static boolean_t
291 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
292 {
293 sfxge_evq_t *sep = arg;
294 sfxge_t *sp = sep->se_sp;
295
296 _NOTE(ARGUNUSED(code))
297 _NOTE(ARGUNUSED(data))
298
299 ASSERT(mutex_owned(&(sep->se_lock)));
300 sep->se_exception = B_TRUE;
301
302 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
303
304 DTRACE_PROBE(restart_ev_exception);
305
306 /* sfxge_evq_t->se_lock held */
307 (void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR,
308 "Unknown EV", code);
309 }
310
311 return (B_FALSE);
312 }
313
314 static boolean_t
315 sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
316 {
317 sfxge_evq_t *sep_targetq, *sep = arg;
318 sfxge_t *sp = sep->se_sp;
319 sfxge_rxq_t *srp;
320 unsigned int index;
321 uint16_t magic;
322
323 ASSERT(mutex_owned(&(sep->se_lock)));
324
325 /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
326 srp = sp->s_srp[label];
327 if (srp == NULL)
328 goto done;
329
330 /* Resend a software event on the correct queue */
331 index = srp->sr_index;
332 sep_targetq = sp->s_sep[index];
333
334 if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
335 goto done; /* TBD: state test not under the lock */
336
337 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
338 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
339
340 efx_ev_qpost(sep_targetq->se_eep, magic);
341
342 done:
343 return (B_FALSE);
344 }
345
346 static boolean_t
347 sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
348 {
349 sfxge_evq_t *sep_targetq, *sep = arg;
350 sfxge_t *sp = sep->se_sp;
351 sfxge_rxq_t *srp;
352 unsigned int index;
353 uint16_t magic;
354
355 ASSERT(mutex_owned(&(sep->se_lock)));
356
357 /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
358 srp = sp->s_srp[label];
359 if (srp == NULL)
360 goto done;
361
362 /* Resend a software event on the correct queue */
363 index = srp->sr_index;
364 sep_targetq = sp->s_sep[index];
365
366 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
367 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
368
369 if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
370 goto done; /* TBD: state test not under the lock */
371
372 efx_ev_qpost(sep_targetq->se_eep, magic);
373
374 done:
375 return (B_FALSE);
376 }
377
378 static boolean_t
379 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
380 {
381 sfxge_evq_t *sep = arg;
382 sfxge_t *sp = sep->se_sp;
383 sfxge_txq_t *stp;
384 unsigned int stop;
385 unsigned int delta;
386
387 ASSERT(mutex_owned(&(sep->se_lock)));
388
389 stp = sp->s_stp[label];
390 if (stp == NULL)
391 goto done;
392
393 if (stp->st_state != SFXGE_TXQ_STARTED)
394 goto done;
395
396 ASSERT3U(sep->se_index, ==, stp->st_evq);
397
398 stop = (id + 1) & (SFXGE_NDESCS - 1);
399 id = stp->st_pending & (SFXGE_NDESCS - 1);
400
401 delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
402 stp->st_pending += delta;
403
404 sep->se_tx++;
405
406 if (stp->st_next == NULL &&
407 sep->se_stpp != &(stp->st_next)) {
408 *(sep->se_stpp) = stp;
409 sep->se_stpp = &(stp->st_next);
410 }
411
412 DTRACE_PROBE2(qlevel, unsigned int, stp->st_index,
413 unsigned int, stp->st_added - stp->st_pending);
414
415 if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH)
416 sfxge_tx_qcomplete(stp);
417
418 done:
419 /* returning B_TRUE makes efx_ev_qpoll() stop processing events */
420 return (sep->se_tx >= sep->se_ev_batch);
421 }
422
423 static boolean_t
424 sfxge_ev_txq_flush_done(void *arg, uint32_t label)
425 {
426 sfxge_evq_t *sep = arg;
427 sfxge_t *sp = sep->se_sp;
428 sfxge_txq_t *stp;
429 unsigned int evq;
430 uint16_t magic;
431
432 ASSERT(mutex_owned(&(sep->se_lock)));
433
434 /* Ensure TXQ exists, as events may arrive after TXQ was destroyed */
435 stp = sp->s_stp[label];
436 if (stp == NULL)
437 goto done;
438
439 ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED);
440
441 /* Resend a software event on the correct queue */
442 evq = stp->st_evq;
443 sep = sp->s_sep[evq];
444
445 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
446 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
447
448 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
449 efx_ev_qpost(sep->se_eep, magic);
450
451 done:
452 return (B_FALSE);
453 }
454
455 static boolean_t
456 sfxge_ev_software(void *arg, uint16_t magic)
457 {
458 sfxge_evq_t *sep = arg;
459 sfxge_t *sp = sep->se_sp;
460 dev_info_t *dip = sp->s_dip;
461 unsigned int label;
462
463 ASSERT(mutex_owned(&(sep->se_lock)));
464
465 EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
466 FSF_AZ_RX_EV_Q_LABEL_WIDTH);
467 EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
468 FSF_AZ_TX_EV_Q_LABEL_WIDTH);
469
470 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
471 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
472
473 switch (magic) {
474 case SFXGE_MAGIC_RX_QFLUSH_DONE: {
475 sfxge_rxq_t *srp = sp->s_srp[label];
476
477 if (srp != NULL) {
478 ASSERT3U(sep->se_index, ==, srp->sr_index);
479
480 sfxge_rx_qflush_done(srp);
481 }
482 break;
483 }
484 case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
485 sfxge_rxq_t *srp = sp->s_srp[label];
486
487 if (srp != NULL) {
488 ASSERT3U(sep->se_index, ==, srp->sr_index);
489
490 sfxge_rx_qflush_failed(srp);
491 }
492 break;
493 }
494 case SFXGE_MAGIC_RX_QFPP_TRIM: {
495 sfxge_rxq_t *srp = sp->s_srp[label];
496
497 if (srp != NULL) {
498 ASSERT3U(sep->se_index, ==, srp->sr_index);
499
500 sfxge_rx_qfpp_trim(srp);
501 }
502 break;
503 }
504 case SFXGE_MAGIC_TX_QFLUSH_DONE: {
505 sfxge_txq_t *stp = sp->s_stp[label];
506
507 if (stp != NULL) {
508 ASSERT3U(sep->se_index, ==, stp->st_evq);
509
510 sfxge_tx_qflush_done(stp);
511 }
512 break;
513 }
514 default:
515 cmn_err(CE_NOTE,
516 SFXGE_CMN_ERR "[%s%d] unknown software event 0x%x",
517 ddi_driver_name(dip), ddi_get_instance(dip), magic);
518 break;
519 }
520
521 return (B_FALSE);
522 }
523
524 static boolean_t
525 sfxge_ev_sram(void *arg, uint32_t code)
526 {
527 _NOTE(ARGUNUSED(arg))
528
529 switch (code) {
530 case EFX_SRAM_UPDATE:
531 DTRACE_PROBE(sram_update);
532 break;
533
534 case EFX_SRAM_CLEAR:
535 DTRACE_PROBE(sram_clear);
536 break;
537
538 case EFX_SRAM_ILLEGAL_CLEAR:
539 DTRACE_PROBE(sram_illegal_clear);
540 break;
541
542 default:
543 ASSERT(B_FALSE);
544 break;
545 }
546
547 return (B_FALSE);
548 }
549
550 static boolean_t
551 sfxge_ev_timer(void *arg, uint32_t index)
552 {
553 _NOTE(ARGUNUSED(arg, index))
554
555 return (B_FALSE);
556 }
557
558 static boolean_t
559 sfxge_ev_wake_up(void *arg, uint32_t index)
560 {
561 _NOTE(ARGUNUSED(arg, index))
562
563 return (B_FALSE);
564 }
565
566 static boolean_t
567 sfxge_ev_monitor(void *arg, efx_mon_stat_t id, efx_mon_stat_value_t value)
568 {
569 _NOTE(ARGUNUSED(arg, id, value))
570
571 return (B_FALSE);
572 }
573
574 static boolean_t
575 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
576 {
577 sfxge_evq_t *sep = arg;
578 sfxge_t *sp = sep->se_sp;
579
580 sfxge_mac_link_update(sp, link_mode);
581
582 return (B_FALSE);
583 }
584
585 static int
586 sfxge_ev_kstat_update(kstat_t *ksp, int rw)
587 {
588 sfxge_evq_t *sep = ksp->ks_private;
589 kstat_named_t *knp;
590 int rc;
591
592 if (rw != KSTAT_READ) {
593 rc = EACCES;
594 goto fail1;
595 }
596
597 ASSERT(mutex_owned(&(sep->se_lock)));
598
599 if (sep->se_state != SFXGE_EVQ_STARTED)
600 goto done;
601
602 efx_ev_qstats_update(sep->se_eep, sep->se_stat);
603
604 knp = ksp->ks_data;
605 knp += EV_NQSTATS;
606
607 knp->value.ui64 = sep->se_cpu_id;
608
609 done:
610 return (0);
611
612 fail1:
613 DTRACE_PROBE1(fail1, int, rc);
614
615 return (rc);
616 }
617
618 static int
619 sfxge_ev_kstat_init(sfxge_evq_t *sep)
620 {
621 sfxge_t *sp = sep->se_sp;
622 unsigned int index = sep->se_index;
623 dev_info_t *dip = sp->s_dip;
624 kstat_t *ksp;
625 kstat_named_t *knp;
626 char name[MAXNAMELEN];
627 unsigned int id;
628 int rc;
629
630 /* Determine the name */
631 (void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d",
632 ddi_driver_name(dip), index);
633
634 /* Create the set */
635 if ((ksp = kstat_create((char *)ddi_driver_name(dip),
636 ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
637 EV_NQSTATS + 1, 0)) == NULL) {
638 rc = ENOMEM;
639 goto fail1;
640 }
641
642 sep->se_ksp = ksp;
643
644 ksp->ks_update = sfxge_ev_kstat_update;
645 ksp->ks_private = sep;
646 ksp->ks_lock = &(sep->se_lock);
647
648 /* Initialise the named stats */
649 sep->se_stat = knp = ksp->ks_data;
650 for (id = 0; id < EV_NQSTATS; id++) {
651 kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id),
652 KSTAT_DATA_UINT64);
653 knp++;
654 }
655
656 kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64);
657
658 kstat_install(ksp);
659 return (0);
660
661 fail1:
662 DTRACE_PROBE1(fail1, int, rc);
663
664 return (rc);
665 }
666
667 static void
668 sfxge_ev_kstat_fini(sfxge_evq_t *sep)
669 {
670 /* Destroy the set */
671 kstat_delete(sep->se_ksp);
672 sep->se_ksp = NULL;
673 sep->se_stat = NULL;
674 }
675
676 inline unsigned pow2_ge(unsigned int n) {
677 unsigned int order = 0;
678 ASSERT3U(n, >, 0);
679 while ((1ul << order) < n) ++order;
680 return (1ul << (order));
681 }
682
683 static int
684 sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch)
685 {
686 sfxge_evq_t *sep;
687 int rc;
688
689 ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
690
691 sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP);
692 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED);
693
694 sep->se_index = index;
695
696 mutex_init(&(sep->se_lock), NULL,
697 MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri));
698
699 cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL);
700
701 /* Initialize the statistics */
702 if ((rc = sfxge_ev_kstat_init(sep)) != 0)
703 goto fail1;
704
705 sep->se_state = SFXGE_EVQ_INITIALIZED;
706 sep->se_ev_batch = ev_batch;
707 sp->s_sep[index] = sep;
708
709 return (0);
710
711 fail1:
712 DTRACE_PROBE1(fail1, int, rc);
713
714 sep->se_index = 0;
715
716 cv_destroy(&(sep->se_init_kv));
717 mutex_destroy(&(sep->se_lock));
718
719 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
720
721 return (rc);
722 }
723
724 static int
725 sfxge_ev_qstart(sfxge_t *sp, unsigned int index)
726 {
727 sfxge_evq_t *sep = sp->s_sep[index];
728 sfxge_intr_t *sip = &(sp->s_intr);
729 efx_nic_t *enp = sp->s_enp;
730 efx_ev_callbacks_t *eecp;
731 efsys_mem_t *esmp;
732 clock_t timeout;
733 int rc;
734 uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
735
736 mutex_enter(&(sep->se_lock));
737 esmp = &(sep->se_mem);
738
739 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
740
741 /* Set the memory to all ones */
742 (void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size));
743
744 /* Program the buffer table */
745 if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp,
746 EFX_EVQ_NBUFS(evq_size))) != 0)
747 goto fail1;
748
749 /* Set up the event callbacks */
750 eecp = &(sep->se_eec);
751 eecp->eec_initialized = sfxge_ev_initialized;
752 eecp->eec_rx = sfxge_ev_rx;
753 eecp->eec_tx = sfxge_ev_tx;
754 eecp->eec_exception = sfxge_ev_exception;
755 eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done;
756 eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed;
757 eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done;
758 eecp->eec_software = sfxge_ev_software;
759 eecp->eec_sram = sfxge_ev_sram;
760 eecp->eec_wake_up = sfxge_ev_wake_up;
761 eecp->eec_timer = sfxge_ev_timer;
762 eecp->eec_link_change = sfxge_ev_link_change;
763 eecp->eec_monitor = sfxge_ev_monitor;
764
765 /* Create the event queue */
766 if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id,
767 &(sep->se_eep))) != 0)
768 goto fail2;
769
770 /* Set the default moderation */
771 if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0)
772 goto fail3;
773
774 /* Check that interrupts are enabled at the NIC */
775 if (sip->si_state != SFXGE_INTR_STARTED) {
776 rc = EINVAL;
777 goto fail4;
778 }
779
780 sep->se_state = SFXGE_EVQ_STARTING;
781
782 /* Prime the event queue for interrupts */
783 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
784 goto fail5;
785
786 /* Wait for the initialization event */
787 timeout = ddi_get_lbolt() + drv_usectohz(2000000);
788 while (sep->se_state != SFXGE_EVQ_STARTED) {
789 if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock),
790 timeout) < 0) {
791 /* Timeout waiting for initialization */
792 dev_info_t *dip = sp->s_dip;
793
794 DTRACE_PROBE(timeout);
795 cmn_err(CE_NOTE,
796 SFXGE_CMN_ERR "[%s%d] ev qstart timeout",
797 ddi_driver_name(dip), ddi_get_instance(dip));
798 rc = ETIMEDOUT;
799 goto fail6;
800 }
801 }
802
803 mutex_exit(&(sep->se_lock));
804 return (0);
805
806 fail6:
807 DTRACE_PROBE(fail6);
808
809 fail5:
810 DTRACE_PROBE(fail5);
811
812 sep->se_state = SFXGE_EVQ_INITIALIZED;
813
814 fail4:
815 DTRACE_PROBE(fail4);
816
817 fail3:
818 DTRACE_PROBE(fail3);
819
820 /* Destroy the event queue */
821 efx_ev_qdestroy(sep->se_eep);
822 sep->se_eep = NULL;
823
824 fail2:
825 DTRACE_PROBE(fail2);
826
827 /* Zero out the event handlers */
828 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
829
830 /* Clear entries from the buffer table */
831 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
832
833 fail1:
834 DTRACE_PROBE1(fail1, int, rc);
835
836 mutex_exit(&(sep->se_lock));
837
838 return (rc);
839 }
840
841 int
842 sfxge_ev_qpoll(sfxge_t *sp, unsigned int index)
843 {
844 sfxge_evq_t *sep = sp->s_sep[index];
845 processorid_t cpu_id;
846 int rc;
847 uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
848
849 mutex_enter(&(sep->se_lock));
850
851 if (sep->se_state != SFXGE_EVQ_STARTING &&
852 sep->se_state != SFXGE_EVQ_STARTED) {
853 rc = EINVAL;
854 goto fail1;
855 }
856
857 /* Make sure the CPU information is up to date */
858 cpu_id = CPU->cpu_id;
859
860 if (cpu_id != sep->se_cpu_id) {
861 #ifdef _USE_CPU_PHYSID
862 cpu_physid_t *cpp = CPU->cpu_physid;
863 #endif
864
865 sep->se_cpu_id = cpu_id;
866
867 #ifdef _USE_CPU_PHYSID
868 sep->se_core_id = cpp->cpu_coreid;
869 sep->se_cache_id = cpp->cpu_cacheid;
870 sep->se_chip_id = cpp->cpu_chipid;
871 #endif
872
873 /* sfxge_evq_t->se_lock held */
874 (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
875 DDI_NOSLEEP);
876 }
877
878 /* Synchronize the DMA memory for reading */
879 (void) ddi_dma_sync(sep->se_mem.esm_dma_handle,
880 0,
881 EFX_EVQ_SIZE(evq_size),
882 DDI_DMA_SYNC_FORKERNEL);
883
884 ASSERT3U(sep->se_rx, ==, 0);
885 ASSERT3U(sep->se_tx, ==, 0);
886 ASSERT3P(sep->se_stp, ==, NULL);
887 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
888
889 /* Poll the queue */
890 efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec),
891 sep);
892
893 sep->se_rx = 0;
894 sep->se_tx = 0;
895
896 /* Perform any pending completion processing */
897 sfxge_ev_qcomplete(sep, B_TRUE);
898
899 /* Re-prime the event queue for interrupts */
900 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
901 goto fail2;
902
903 mutex_exit(&(sep->se_lock));
904
905 return (0);
906
907 fail2:
908 DTRACE_PROBE(fail2);
909 fail1:
910 DTRACE_PROBE1(fail1, int, rc);
911
912 mutex_exit(&(sep->se_lock));
913
914 return (rc);
915 }
916
917 int
918 sfxge_ev_qprime(sfxge_t *sp, unsigned int index)
919 {
920 sfxge_evq_t *sep = sp->s_sep[index];
921 int rc;
922
923 mutex_enter(&(sep->se_lock));
924
925 if (sep->se_state != SFXGE_EVQ_STARTING &&
926 sep->se_state != SFXGE_EVQ_STARTED) {
927 rc = EINVAL;
928 goto fail1;
929 }
930
931 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
932 goto fail2;
933
934 mutex_exit(&(sep->se_lock));
935
936 return (0);
937
938 fail2:
939 DTRACE_PROBE(fail2);
940 fail1:
941 DTRACE_PROBE1(fail1, int, rc);
942
943 mutex_exit(&(sep->se_lock));
944
945 return (rc);
946 }
947
948
949 int
950 sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us)
951 {
952 sfxge_evq_t *sep = sp->s_sep[index];
953 efx_evq_t *eep = sep->se_eep;
954
955 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
956
957 return (efx_ev_qmoderate(eep, us));
958 }
959
960 static void
961 sfxge_ev_qstop(sfxge_t *sp, unsigned int index)
962 {
963 sfxge_evq_t *sep = sp->s_sep[index];
964 uint16_t evq_size;
965
966 mutex_enter(&(sep->se_lock));
967 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
968 sep->se_state = SFXGE_EVQ_INITIALIZED;
969 evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
970
971 /* Clear the CPU information */
972 #ifdef _USE_CPU_PHYSID
973 sep->se_cache_id = 0;
974 sep->se_core_id = 0;
975 sep->se_chip_id = 0;
976 #endif
977
978 sep->se_cpu_id = 0;
979
980 /* Clear the event count */
981 sep->se_count = 0;
982
983 /* Reset the exception flag */
984 sep->se_exception = B_FALSE;
985
986 /* Destroy the event queue */
987 efx_ev_qdestroy(sep->se_eep);
988 sep->se_eep = NULL;
989
990 mutex_exit(&(sep->se_lock));
991
992 /* Zero out the event handlers */
993 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
994
995 /* Clear entries from the buffer table */
996 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
997 }
998
999 static void
1000 sfxge_ev_qfini(sfxge_t *sp, unsigned int index)
1001 {
1002 sfxge_evq_t *sep = sp->s_sep[index];
1003
1004 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
1005
1006 sp->s_sep[index] = NULL;
1007 sep->se_state = SFXGE_EVQ_UNINITIALIZED;
1008
1009 /* Tear down the statistics */
1010 sfxge_ev_kstat_fini(sep);
1011
1012 cv_destroy(&(sep->se_init_kv));
1013 mutex_destroy(&(sep->se_lock));
1014
1015 sep->se_index = 0;
1016
1017 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
1018 }
1019
1020
1021 static kmem_cache_t *
1022 sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname,
1023 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *))
1024 {
1025 char name[MAXNAMELEN];
1026 kmem_cache_t *eqc;
1027
1028 (void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache",
1029 ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname);
1030
1031 eqc = kmem_cache_create(name, sizeof (sfxge_evq_t),
1032 SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0);
1033 ASSERT(eqc != NULL);
1034 return (eqc);
1035 }
1036
1037 int
1038 sfxge_ev_init(sfxge_t *sp)
1039 {
1040 sfxge_intr_t *sip = &(sp->s_intr);
1041 unsigned int evq0_size;
1042 unsigned int evqX_size;
1043 unsigned int ev_batch;
1044 int index;
1045 int rc;
1046
1047 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
1048
1049 /*
1050 * Must account for RXQ, TXQ(s); MCDI not event completed at present
1051 * Note that common code does not completely fill descriptor queues
1052 */
1053 evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS;
1054 evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */
1055 evq0_size += SFXGE_TX_NDESCS; /* no checksums */
1056
1057 ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS);
1058 ASSERT3U(evq0_size, >, evqX_size);
1059
1060 if (evq0_size > EFX_EVQ_MAXNEVS) {
1061 rc = EINVAL;
1062 goto fail1;
1063 }
1064
1065 sp->s_evq0_size = pow2_ge(evq0_size);
1066 sp->s_evqX_size = pow2_ge(evqX_size);
1067
1068 /* Read driver parameters */
1069 sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
1070 DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION);
1071
1072 ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
1073 DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH);
1074
1075 /*
1076 * It is slightly peverse to have a cache for one item. But it allows
1077 * for simple alignment control without increasing the allocation size
1078 */
1079 sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor,
1080 sfxge_ev_q0dtor);
1081 sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor,
1082 sfxge_ev_qXdtor);
1083
1084 /* Initialize the event queue(s) */
1085 for (index = 0; index < sip->si_nalloc; index++) {
1086 if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0)
1087 goto fail2;
1088 }
1089
1090 return (0);
1091
1092 fail2:
1093 DTRACE_PROBE(fail2);
1094
1095 while (--index >= 0)
1096 sfxge_ev_qfini(sp, index);
1097 sp->s_ev_moderation = 0;
1098
1099 fail1:
1100 DTRACE_PROBE1(fail1, int, rc);
1101
1102 kmem_cache_destroy(sp->s_eqXc);
1103 kmem_cache_destroy(sp->s_eq0c);
1104 sp->s_eqXc = NULL;
1105 sp->s_eq0c = NULL;
1106
1107 return (rc);
1108 }
1109
1110 int
1111 sfxge_ev_start(sfxge_t *sp)
1112 {
1113 sfxge_intr_t *sip = &(sp->s_intr);
1114 efx_nic_t *enp = sp->s_enp;
1115 int index;
1116 int rc;
1117
1118 ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
1119
1120 /* Initialize the event module */
1121 if ((rc = efx_ev_init(enp)) != 0)
1122 goto fail1;
1123
1124 /* Start the event queues */
1125 for (index = 0; index < sip->si_nalloc; index++) {
1126 if ((rc = sfxge_ev_qstart(sp, index)) != 0)
1127 goto fail2;
1128 }
1129
1130 return (0);
1131
1132 fail2:
1133 DTRACE_PROBE(fail2);
1134
1135 /* Stop the event queue(s) */
1136 while (--index >= 0)
1137 sfxge_ev_qstop(sp, index);
1138
1139 /* Tear down the event module */
1140 efx_ev_fini(enp);
1141
1142 fail1:
1143 DTRACE_PROBE1(fail1, int, rc);
1144
1145 return (rc);
1146 }
1147
1148 void
1149 sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp)
1150 {
1151 *usp = sp->s_ev_moderation;
1152 }
1153
1154 int
1155 sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us)
1156 {
1157 sfxge_intr_t *sip = &(sp->s_intr);
1158 int index;
1159 int rc;
1160
1161 if (sip->si_state != SFXGE_INTR_STARTED)
1162 return (ENODEV);
1163
1164 for (index = 0; index < sip->si_nalloc; index++) {
1165 if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0)
1166 goto fail1;
1167 }
1168
1169 sp->s_ev_moderation = us;
1170 return (0);
1171
1172 fail1:
1173 DTRACE_PROBE1(fail1, int, rc);
1174
1175 /* The only error path is if the value to set to is invalid. */
1176 ASSERT3U(index, ==, 0);
1177
1178 return (rc);
1179 }
1180
1181 void
1182 sfxge_ev_stop(sfxge_t *sp)
1183 {
1184 sfxge_intr_t *sip = &(sp->s_intr);
1185 efx_nic_t *enp = sp->s_enp;
1186 int index;
1187
1188 ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
1189
1190 /* Stop the event queue(s) */
1191 index = sip->si_nalloc;
1192 while (--index >= 0)
1193 sfxge_ev_qstop(sp, index);
1194
1195 /* Tear down the event module */
1196 efx_ev_fini(enp);
1197 }
1198
1199 void
1200 sfxge_ev_fini(sfxge_t *sp)
1201 {
1202 sfxge_intr_t *sip = &(sp->s_intr);
1203 int index;
1204
1205 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
1206
1207 sp->s_ev_moderation = 0;
1208
1209 /* Tear down the event queue(s) */
1210 index = sip->si_nalloc;
1211 while (--index >= 0)
1212 sfxge_ev_qfini(sp, index);
1213
1214 kmem_cache_destroy(sp->s_eqXc);
1215 kmem_cache_destroy(sp->s_eq0c);
1216 sp->s_eqXc = NULL;
1217 sp->s_eq0c = NULL;
1218 }