1 /*
2 * Copyright (c) 2008-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include <sys/types.h>
32 #include <sys/sysmacros.h>
33 #include <sys/ddi.h>
34 #include <sys/sunddi.h>
35 #include <sys/stream.h>
36 #include <sys/strsun.h>
37 #include <sys/strsubr.h>
38 #include <sys/cpu.h>
39 #include <sys/pghw.h>
40
41 #include "sfxge.h"
42
43 #include "efx.h"
44
45
46 /* Timeout to wait for DRIVER_EV_START event at EVQ startup */
47 #define SFXGE_EV_QSTART_TIMEOUT_USEC (2000000)
48
49
50 /* Event queue DMA attributes */
51 static ddi_device_acc_attr_t sfxge_evq_devacc = {
52
53 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
54 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
55 DDI_STRICTORDER_ACC /* devacc_attr_dataorder */
56 };
57
58 static ddi_dma_attr_t sfxge_evq_dma_attr = {
59 DMA_ATTR_V0, /* dma_attr_version */
60 0, /* dma_attr_addr_lo */
61 0xffffffffffffffffull, /* dma_attr_addr_hi */
62 0xffffffffffffffffull, /* dma_attr_count_max */
63 EFX_BUF_SIZE, /* dma_attr_align */
64 0xffffffff, /* dma_attr_burstsizes */
65 1, /* dma_attr_minxfer */
66 0xffffffffffffffffull, /* dma_attr_maxxfer */
67 0xffffffffffffffffull, /* dma_attr_seg */
68 1, /* dma_attr_sgllen */
69 1, /* dma_attr_granular */
70 0 /* dma_attr_flags */
71 };
72
73 static int
74 _sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size)
75 {
76 efsys_mem_t *esmp = &(sep->se_mem);
77 sfxge_dma_buffer_attr_t dma_attr;
78 int rc;
79
80 /* Compile-time structure layout checks */
81 EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <=
82 sizeof (sep->__se_u1.__se_pad));
83 EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <=
84 sizeof (sep->__se_u2.__se_pad));
85 EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <=
86 sizeof (sep->__se_u3.__se_pad));
87
88 bzero(sep, sizeof (sfxge_evq_t));
89
90 sep->se_sp = sp;
91
92 dma_attr.sdba_dip = sp->s_dip;
93 dma_attr.sdba_dattrp = &sfxge_evq_dma_attr;
94 dma_attr.sdba_callback = (kmflags == KM_SLEEP) ?
95 DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
96 dma_attr.sdba_length = EFX_EVQ_SIZE(evq_size);
97 dma_attr.sdba_memflags = DDI_DMA_CONSISTENT;
98 dma_attr.sdba_devaccp = &sfxge_evq_devacc;
99 dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT;
100 dma_attr.sdba_maxcookies = 1;
101 dma_attr.sdba_zeroinit = B_FALSE;
102
103 if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
104 goto fail1;
105
106 /* Allocate some buffer table entries */
107 if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size),
108 &(sep->se_id))) != 0)
109 goto fail2;
110
111 sep->se_stpp = &(sep->se_stp);
112
113 return (0);
114
115 fail2:
116 DTRACE_PROBE(fail2);
117
118 /* Tear down DMA setup */
119 esmp->esm_addr = 0;
120 sfxge_dma_buffer_destroy(esmp);
121
122 fail1:
123 DTRACE_PROBE1(fail1, int, rc);
124
125 sep->se_sp = NULL;
126
127 SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
128
129 return (-1);
130 }
131
132 static int
133 sfxge_ev_q0ctor(void *buf, void *arg, int kmflags)
134 {
135 sfxge_evq_t *sep = buf;
136 sfxge_t *sp = arg;
137 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size));
138 }
139
140 static int
141 sfxge_ev_qXctor(void *buf, void *arg, int kmflags)
142 {
143 sfxge_evq_t *sep = buf;
144 sfxge_t *sp = arg;
145 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size));
146 }
147 static void
148 _sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size)
149 {
150 efsys_mem_t *esmp = &(sep->se_mem);
151 ASSERT3P(sep->se_sp, ==, sp);
152 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
153 sep->se_stpp = NULL;
154
155 /* Free the buffer table entries */
156 sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
157 sep->se_id = 0;
158
159 /* Tear down DMA setup */
160 sfxge_dma_buffer_destroy(esmp);
161
162 sep->se_sp = NULL;
163
164 SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
165 }
166
167 static void
168 sfxge_ev_q0dtor(void *buf, void *arg)
169 {
170 sfxge_evq_t *sep = buf;
171 sfxge_t *sp = arg;
172 _sfxge_ev_qdtor(sp, sep, sp->s_evq0_size);
173 }
174
175 static void
176 sfxge_ev_qXdtor(void *buf, void *arg)
177 {
178 sfxge_evq_t *sep = buf;
179 sfxge_t *sp = arg;
180 _sfxge_ev_qdtor(sp, sep, sp->s_evqX_size);
181 }
182
183 static boolean_t
184 sfxge_ev_initialized(void *arg)
185 {
186 sfxge_evq_t *sep = arg;
187
188 ASSERT(mutex_owned(&(sep->se_lock)));
189
190 /* Init done events may be duplicated on 7xxx (see SFCbug31631) */
191 if (sep->se_state == SFXGE_EVQ_STARTED)
192 goto done;
193
194 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING);
195 sep->se_state = SFXGE_EVQ_STARTED;
196
197 cv_broadcast(&(sep->se_init_kv));
198
199 done:
200 return (B_FALSE);
201 }
202
203 static void
204 sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop)
205 {
206 sfxge_t *sp = sep->se_sp;
207 unsigned int index = sep->se_index;
208 sfxge_rxq_t *srp = sp->s_srp[index];
209 sfxge_txq_t *stp;
210
211 if ((stp = sep->se_stp) != NULL) {
212 sep->se_stp = NULL;
213 sep->se_stpp = &(sep->se_stp);
214
215 do {
216 sfxge_txq_t *next;
217
218 next = stp->st_next;
219 stp->st_next = NULL;
220
221 ASSERT3U(stp->st_evq, ==, index);
222
223 if (stp->st_pending != stp->st_completed)
224 sfxge_tx_qcomplete(stp);
225
226 stp = next;
227 } while (stp != NULL);
228 }
229
230 if (srp != NULL) {
231 if (srp->sr_pending != srp->sr_completed)
232 sfxge_rx_qcomplete(srp, eop);
233 }
234 }
235
236 static boolean_t
237 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
238 uint16_t flags)
239 {
240 sfxge_evq_t *sep = arg;
241 sfxge_t *sp = sep->se_sp;
242 sfxge_rxq_t *srp;
243 sfxge_rx_packet_t *srpp;
244 unsigned int prefetch;
245 unsigned int stop;
246 unsigned int delta;
247
248 ASSERT(mutex_owned(&(sep->se_lock)));
249
250 if (sep->se_exception)
251 goto done;
252
253 srp = sp->s_srp[label];
254 if (srp == NULL)
255 goto done;
256
257 ASSERT3U(sep->se_index, ==, srp->sr_index);
258 ASSERT3U(id, <, sp->s_rxq_size);
259
260 /*
261 * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset
262 * So the return missing srp->sr_pending increase is safe
263 */
264 if (srp->sr_state != SFXGE_RXQ_STARTED)
265 goto done;
266
267 stop = (id + 1) & (sp->s_rxq_size - 1);
268 id = srp->sr_pending & (sp->s_rxq_size - 1);
269
270 delta = (stop >= id) ? (stop - id) : (sp->s_rxq_size - id + stop);
271 srp->sr_pending += delta;
272
273 if (delta != 1) {
274 if ((!efx_nic_cfg_get(sp->s_enp)->enc_rx_batching_enabled) ||
275 (delta == 0) ||
276 (delta > efx_nic_cfg_get(sp->s_enp)->enc_rx_batch_max)) {
277 /*
278 * FIXME: This does not take into account scatter
279 * aborts. See Bug40811
280 */
281 sep->se_exception = B_TRUE;
282
283 DTRACE_PROBE(restart_ev_rx_id);
284 /* sfxge_evq_t->se_lock held */
285 (void) sfxge_restart_dispatch(sp, DDI_SLEEP,
286 SFXGE_HW_ERR, "Out of order RX event", delta);
287
288 goto done;
289 }
290 }
291
292 prefetch = (id + 4) & (sp->s_rxq_size - 1);
293 if ((srpp = srp->sr_srpp[prefetch]) != NULL)
294 prefetch_read_many(srpp);
295
296 srpp = srp->sr_srpp[id];
297 ASSERT(srpp != NULL);
298 prefetch_read_many(srpp->srp_mp);
299
300 for (; id != stop; id = (id + 1) & (sp->s_rxq_size - 1)) {
301 srpp = srp->sr_srpp[id];
302 ASSERT(srpp != NULL);
303
304 ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD);
305 srpp->srp_flags = flags;
306
307 ASSERT3U(size, <, (1 << 16));
308 srpp->srp_size = (uint16_t)size;
309 }
310
311 sep->se_rx++;
312
313 DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index,
314 unsigned int, srp->sr_added - srp->sr_pending);
315
316 if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH)
317 sfxge_ev_qcomplete(sep, B_FALSE);
318
319 done:
320 /* returning B_TRUE makes efx_ev_qpoll() stop processing events */
321 return (sep->se_rx >= sep->se_ev_batch);
322 }
323
324 static boolean_t
325 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
326 {
327 sfxge_evq_t *sep = arg;
328 sfxge_t *sp = sep->se_sp;
329
330 _NOTE(ARGUNUSED(code))
331 _NOTE(ARGUNUSED(data))
332
333 ASSERT(mutex_owned(&(sep->se_lock)));
334 sep->se_exception = B_TRUE;
335
336 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
337
338 DTRACE_PROBE(restart_ev_exception);
339
340 /* sfxge_evq_t->se_lock held */
341 (void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR,
342 "Unknown EV", code);
343 }
344
345 return (B_FALSE);
346 }
347
348 static boolean_t
349 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
350 {
351 sfxge_evq_t *sep_targetq, *sep = arg;
352 sfxge_t *sp = sep->se_sp;
353 sfxge_rxq_t *srp;
354 unsigned int index;
355 unsigned int label;
356 uint16_t magic;
357
358 ASSERT(mutex_owned(&(sep->se_lock)));
359
360 /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
361 srp = sp->s_srp[rxq_index];
362 if (srp == NULL)
363 goto done;
364
365 /* Process right now if it is the correct event queue */
366 index = srp->sr_index;
367 if (index == sep->se_index) {
368 sfxge_rx_qflush_done(srp);
369 goto done;
370 }
371
372 /* Resend a software event on the correct queue */
373 sep_targetq = sp->s_sep[index];
374
375 if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
376 goto done; /* TBD: state test not under the lock */
377
378 label = rxq_index;
379 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
380 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
381
382 efx_ev_qpost(sep_targetq->se_eep, magic);
383
384 done:
385 return (B_FALSE);
386 }
387
388 static boolean_t
389 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
390 {
391 sfxge_evq_t *sep_targetq, *sep = arg;
392 sfxge_t *sp = sep->se_sp;
393 sfxge_rxq_t *srp;
394 unsigned int index;
395 unsigned int label;
396 uint16_t magic;
397
398 ASSERT(mutex_owned(&(sep->se_lock)));
399
400 /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
401 srp = sp->s_srp[rxq_index];
402 if (srp == NULL)
403 goto done;
404
405 /* Process right now if it is the correct event queue */
406 index = srp->sr_index;
407 if (index == sep->se_index) {
408 sfxge_rx_qflush_failed(srp);
409 goto done;
410 }
411
412 /* Resend a software event on the correct queue */
413 sep_targetq = sp->s_sep[index];
414
415 label = rxq_index;
416 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
417 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
418
419 if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
420 goto done; /* TBD: state test not under the lock */
421
422 efx_ev_qpost(sep_targetq->se_eep, magic);
423
424 done:
425 return (B_FALSE);
426 }
427
428 static boolean_t
429 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
430 {
431 sfxge_evq_t *sep = arg;
432 sfxge_txq_t *stp;
433 unsigned int stop;
434 unsigned int delta;
435
436 ASSERT(mutex_owned(&(sep->se_lock)));
437
438 stp = sep->se_label_stp[label];
439 if (stp == NULL)
440 goto done;
441
442 if (stp->st_state != SFXGE_TXQ_STARTED)
443 goto done;
444
445 ASSERT3U(sep->se_index, ==, stp->st_evq);
446
447 stop = (id + 1) & (SFXGE_TX_NDESCS - 1);
448 id = stp->st_pending & (SFXGE_TX_NDESCS - 1);
449
450 delta = (stop >= id) ? (stop - id) : (SFXGE_TX_NDESCS - id + stop);
451 stp->st_pending += delta;
452
453 sep->se_tx++;
454
455 if (stp->st_next == NULL &&
456 sep->se_stpp != &(stp->st_next)) {
457 *(sep->se_stpp) = stp;
458 sep->se_stpp = &(stp->st_next);
459 }
460
461 DTRACE_PROBE2(qlevel, unsigned int, stp->st_index,
462 unsigned int, stp->st_added - stp->st_pending);
463
464 if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH)
465 sfxge_tx_qcomplete(stp);
466
467 done:
468 /* returning B_TRUE makes efx_ev_qpoll() stop processing events */
469 return (sep->se_tx >= sep->se_ev_batch);
470 }
471
472 static boolean_t
473 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
474 {
475 sfxge_evq_t *sep = arg;
476 sfxge_t *sp = sep->se_sp;
477 sfxge_txq_t *stp;
478 unsigned int evq;
479 unsigned int label;
480 uint16_t magic;
481
482 ASSERT(mutex_owned(&(sep->se_lock)));
483
484 /* Ensure TXQ exists, as events may arrive after TXQ was destroyed */
485 stp = sp->s_stp[txq_index];
486 if (stp == NULL)
487 goto done;
488
489 /* Process right now if it is the correct event queue */
490 evq = stp->st_evq;
491 if (evq == sep->se_index) {
492 sfxge_tx_qflush_done(stp);
493 goto done;
494 }
495
496 /* Resend a software event on the correct queue */
497 sep = sp->s_sep[evq];
498
499 label = stp->st_label;
500
501 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
502 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
503
504 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
505 efx_ev_qpost(sep->se_eep, magic);
506
507 done:
508 return (B_FALSE);
509 }
510
511 static boolean_t
512 sfxge_ev_software(void *arg, uint16_t magic)
513 {
514 sfxge_evq_t *sep = arg;
515 sfxge_t *sp = sep->se_sp;
516 dev_info_t *dip = sp->s_dip;
517 unsigned int label;
518
519 ASSERT(mutex_owned(&(sep->se_lock)));
520
521 EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
522 FSF_AZ_RX_EV_Q_LABEL_WIDTH);
523 EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
524 FSF_AZ_TX_EV_Q_LABEL_WIDTH);
525
526 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
527 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
528
529 switch (magic) {
530 case SFXGE_MAGIC_RX_QFLUSH_DONE: {
531 sfxge_rxq_t *srp = sp->s_srp[label];
532
533 if (srp != NULL) {
534 ASSERT3U(sep->se_index, ==, srp->sr_index);
535
536 sfxge_rx_qflush_done(srp);
537 }
538 break;
539 }
540 case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
541 sfxge_rxq_t *srp = sp->s_srp[label];
542
543 if (srp != NULL) {
544 ASSERT3U(sep->se_index, ==, srp->sr_index);
545
546 sfxge_rx_qflush_failed(srp);
547 }
548 break;
549 }
550 case SFXGE_MAGIC_RX_QFPP_TRIM: {
551 sfxge_rxq_t *srp = sp->s_srp[label];
552
553 if (srp != NULL) {
554 ASSERT3U(sep->se_index, ==, srp->sr_index);
555
556 sfxge_rx_qfpp_trim(srp);
557 }
558 break;
559 }
560 case SFXGE_MAGIC_TX_QFLUSH_DONE: {
561 sfxge_txq_t *stp = sep->se_label_stp[label];
562
563 if (stp != NULL) {
564 ASSERT3U(sep->se_index, ==, stp->st_evq);
565
566 sfxge_tx_qflush_done(stp);
567 }
568 break;
569 }
570 default:
571 dev_err(dip, CE_NOTE,
572 SFXGE_CMN_ERR "unknown software event 0x%x", magic);
573 break;
574 }
575
576 return (B_FALSE);
577 }
578
579 static boolean_t
580 sfxge_ev_sram(void *arg, uint32_t code)
581 {
582 _NOTE(ARGUNUSED(arg))
583
584 switch (code) {
585 case EFX_SRAM_UPDATE:
586 DTRACE_PROBE(sram_update);
587 break;
588
589 case EFX_SRAM_CLEAR:
590 DTRACE_PROBE(sram_clear);
591 break;
592
593 case EFX_SRAM_ILLEGAL_CLEAR:
594 DTRACE_PROBE(sram_illegal_clear);
595 break;
596
597 default:
598 ASSERT(B_FALSE);
599 break;
600 }
601
602 return (B_FALSE);
603 }
604
605 static boolean_t
606 sfxge_ev_timer(void *arg, uint32_t index)
607 {
608 _NOTE(ARGUNUSED(arg, index))
609
610 return (B_FALSE);
611 }
612
613 static boolean_t
614 sfxge_ev_wake_up(void *arg, uint32_t index)
615 {
616 _NOTE(ARGUNUSED(arg, index))
617
618 return (B_FALSE);
619 }
620
621 #if 0
622 static boolean_t
623 sfxge_ev_monitor(void)
624 {
625 _NOTE(ARGUNUSED(arg, id, value))
626
627 return (B_FALSE);
628 }
629 #endif
630
631 static boolean_t
632 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
633 {
634 sfxge_evq_t *sep = arg;
635 sfxge_t *sp = sep->se_sp;
636
637 sfxge_mac_link_update(sp, link_mode);
638
639 return (B_FALSE);
640 }
641
642 static int
643 sfxge_ev_kstat_update(kstat_t *ksp, int rw)
644 {
645 sfxge_evq_t *sep = ksp->ks_private;
646 kstat_named_t *knp;
647 int rc;
648
649 if (rw != KSTAT_READ) {
650 rc = EACCES;
651 goto fail1;
652 }
653
654 ASSERT(mutex_owned(&(sep->se_lock)));
655
656 if (sep->se_state != SFXGE_EVQ_STARTED)
657 goto done;
658
659 efx_ev_qstats_update(sep->se_eep, sep->se_stat);
660
661 knp = ksp->ks_data;
662 knp += EV_NQSTATS;
663
664 knp->value.ui64 = sep->se_cpu_id;
665
666 done:
667 return (0);
668
669 fail1:
670 DTRACE_PROBE1(fail1, int, rc);
671
672 return (rc);
673 }
674
675 static int
676 sfxge_ev_kstat_init(sfxge_evq_t *sep)
677 {
678 sfxge_t *sp = sep->se_sp;
679 unsigned int index = sep->se_index;
680 dev_info_t *dip = sp->s_dip;
681 kstat_t *ksp;
682 kstat_named_t *knp;
683 char name[MAXNAMELEN];
684 unsigned int id;
685 int rc;
686
687 /* Determine the name */
688 (void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d",
689 ddi_driver_name(dip), index);
690
691 /* Create the set */
692 if ((ksp = kstat_create((char *)ddi_driver_name(dip),
693 ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
694 EV_NQSTATS + 1, 0)) == NULL) {
695 rc = ENOMEM;
696 goto fail1;
697 }
698
699 sep->se_ksp = ksp;
700
701 ksp->ks_update = sfxge_ev_kstat_update;
702 ksp->ks_private = sep;
703 ksp->ks_lock = &(sep->se_lock);
704
705 /* Initialise the named stats */
706 sep->se_stat = knp = ksp->ks_data;
707 for (id = 0; id < EV_NQSTATS; id++) {
708 kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id),
709 KSTAT_DATA_UINT64);
710 knp++;
711 }
712
713 kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64);
714
715 kstat_install(ksp);
716 return (0);
717
718 fail1:
719 DTRACE_PROBE1(fail1, int, rc);
720
721 return (rc);
722 }
723
724 static void
725 sfxge_ev_kstat_fini(sfxge_evq_t *sep)
726 {
727 /* Destroy the set */
728 kstat_delete(sep->se_ksp);
729 sep->se_ksp = NULL;
730 sep->se_stat = NULL;
731 }
732
733 inline unsigned pow2_ge(unsigned int n) {
734 unsigned int order = 0;
735 ASSERT3U(n, >, 0);
736 while ((1ul << order) < n) ++order;
737 return (1ul << (order));
738 }
739
740 static int
741 sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch)
742 {
743 sfxge_evq_t *sep;
744 int rc;
745
746 ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
747
748 sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP);
749 if (sep == NULL) {
750 rc = ENOMEM;
751 goto fail1;
752 }
753 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED);
754
755 sep->se_index = index;
756
757 mutex_init(&(sep->se_lock), NULL,
758 MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri));
759
760 cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL);
761
762 /* Initialize the statistics */
763 if ((rc = sfxge_ev_kstat_init(sep)) != 0)
764 goto fail2;
765
766 sep->se_state = SFXGE_EVQ_INITIALIZED;
767 sep->se_ev_batch = (uint16_t)ev_batch;
768 sp->s_sep[index] = sep;
769
770 return (0);
771
772 fail2:
773 DTRACE_PROBE(fail2);
774
775 sep->se_index = 0;
776
777 cv_destroy(&(sep->se_init_kv));
778 mutex_destroy(&(sep->se_lock));
779
780 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
781
782 fail1:
783 DTRACE_PROBE1(fail1, int, rc);
784
785 return (rc);
786 }
787
788 static int
789 sfxge_ev_qstart(sfxge_t *sp, unsigned int index)
790 {
791 sfxge_evq_t *sep = sp->s_sep[index];
792 sfxge_intr_t *sip = &(sp->s_intr);
793 efx_nic_t *enp = sp->s_enp;
794 efx_ev_callbacks_t *eecp;
795 efsys_mem_t *esmp;
796 clock_t timeout;
797 int rc;
798 uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
799
800 mutex_enter(&(sep->se_lock));
801 esmp = &(sep->se_mem);
802
803 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
804
805 /* Set the memory to all ones */
806 (void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size));
807
808 /* Program the buffer table */
809 if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp,
810 EFX_EVQ_NBUFS(evq_size))) != 0)
811 goto fail1;
812
813 /* Set up the event callbacks */
814 eecp = &(sep->se_eec);
815 eecp->eec_initialized = sfxge_ev_initialized;
816 eecp->eec_rx = sfxge_ev_rx;
817 eecp->eec_tx = sfxge_ev_tx;
818 eecp->eec_exception = sfxge_ev_exception;
819 eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done;
820 eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed;
821 eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done;
822 eecp->eec_software = sfxge_ev_software;
823 eecp->eec_sram = sfxge_ev_sram;
824 eecp->eec_wake_up = sfxge_ev_wake_up;
825 eecp->eec_timer = sfxge_ev_timer;
826 eecp->eec_link_change = sfxge_ev_link_change;
827 #if 0
828 eecp->eec_monitor = (void *)sfxge_ev_monitor;
829 #endif
830
831 /* Create the event queue */
832 if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id,
833 &(sep->se_eep))) != 0)
834 goto fail2;
835
836 /* Set the default moderation */
837 if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0)
838 goto fail3;
839
840 /* Check that interrupts are enabled at the NIC */
841 if (sip->si_state != SFXGE_INTR_STARTED) {
842 rc = EINVAL;
843 goto fail4;
844 }
845
846 sep->se_state = SFXGE_EVQ_STARTING;
847
848 /* Prime the event queue for interrupts */
849 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
850 goto fail5;
851
852 /* Wait for the initialization event */
853 timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_EV_QSTART_TIMEOUT_USEC);
854 while (sep->se_state != SFXGE_EVQ_STARTED) {
855 if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock),
856 timeout) < 0) {
857 /* Timeout waiting for initialization */
858 dev_info_t *dip = sp->s_dip;
859
860 DTRACE_PROBE(timeout);
861 dev_err(dip, CE_NOTE,
862 SFXGE_CMN_ERR "evq[%d] qstart timeout", index);
863
864 rc = ETIMEDOUT;
865 goto fail6;
866 }
867 }
868
869 mutex_exit(&(sep->se_lock));
870 return (0);
871
872 fail6:
873 DTRACE_PROBE(fail6);
874
875 fail5:
876 DTRACE_PROBE(fail5);
877
878 sep->se_state = SFXGE_EVQ_INITIALIZED;
879
880 fail4:
881 DTRACE_PROBE(fail4);
882
883 fail3:
884 DTRACE_PROBE(fail3);
885
886 /* Destroy the event queue */
887 efx_ev_qdestroy(sep->se_eep);
888 sep->se_eep = NULL;
889
890 fail2:
891 DTRACE_PROBE(fail2);
892
893 /* Zero out the event handlers */
894 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
895
896 /* Clear entries from the buffer table */
897 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
898
899 fail1:
900 DTRACE_PROBE1(fail1, int, rc);
901
902 mutex_exit(&(sep->se_lock));
903
904 return (rc);
905 }
906
907 int
908 sfxge_ev_qpoll(sfxge_t *sp, unsigned int index)
909 {
910 sfxge_evq_t *sep = sp->s_sep[index];
911 processorid_t cpu_id;
912 int rc;
913 uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
914
915 mutex_enter(&(sep->se_lock));
916
917 if (sep->se_state != SFXGE_EVQ_STARTING &&
918 sep->se_state != SFXGE_EVQ_STARTED) {
919 rc = EINVAL;
920 goto fail1;
921 }
922
923 /* Make sure the CPU information is up to date */
924 cpu_id = CPU->cpu_id;
925
926 if (cpu_id != sep->se_cpu_id) {
927 sep->se_cpu_id = cpu_id;
928
929 /* sfxge_evq_t->se_lock held */
930 (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
931 DDI_NOSLEEP);
932 }
933
934 /* Synchronize the DMA memory for reading */
935 (void) ddi_dma_sync(sep->se_mem.esm_dma_handle,
936 0,
937 EFX_EVQ_SIZE(evq_size),
938 DDI_DMA_SYNC_FORKERNEL);
939
940 ASSERT3U(sep->se_rx, ==, 0);
941 ASSERT3U(sep->se_tx, ==, 0);
942 ASSERT3P(sep->se_stp, ==, NULL);
943 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
944
945 /* Poll the queue */
946 efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec),
947 sep);
948
949 sep->se_rx = 0;
950 sep->se_tx = 0;
951
952 /* Perform any pending completion processing */
953 sfxge_ev_qcomplete(sep, B_TRUE);
954
955 /* Re-prime the event queue for interrupts */
956 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
957 goto fail2;
958
959 mutex_exit(&(sep->se_lock));
960
961 return (0);
962
963 fail2:
964 DTRACE_PROBE(fail2);
965 fail1:
966 DTRACE_PROBE1(fail1, int, rc);
967
968 mutex_exit(&(sep->se_lock));
969
970 return (rc);
971 }
972
973 int
974 sfxge_ev_qprime(sfxge_t *sp, unsigned int index)
975 {
976 sfxge_evq_t *sep = sp->s_sep[index];
977 int rc;
978
979 mutex_enter(&(sep->se_lock));
980
981 if (sep->se_state != SFXGE_EVQ_STARTING &&
982 sep->se_state != SFXGE_EVQ_STARTED) {
983 rc = EINVAL;
984 goto fail1;
985 }
986
987 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
988 goto fail2;
989
990 mutex_exit(&(sep->se_lock));
991
992 return (0);
993
994 fail2:
995 DTRACE_PROBE(fail2);
996 fail1:
997 DTRACE_PROBE1(fail1, int, rc);
998
999 mutex_exit(&(sep->se_lock));
1000
1001 return (rc);
1002 }
1003
1004
1005 int
1006 sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us)
1007 {
1008 sfxge_evq_t *sep = sp->s_sep[index];
1009 efx_evq_t *eep = sep->se_eep;
1010
1011 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
1012
1013 return (efx_ev_qmoderate(eep, us));
1014 }
1015
1016 static void
1017 sfxge_ev_qstop(sfxge_t *sp, unsigned int index)
1018 {
1019 sfxge_evq_t *sep = sp->s_sep[index];
1020 uint16_t evq_size;
1021
1022 mutex_enter(&(sep->se_lock));
1023 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
1024 sep->se_state = SFXGE_EVQ_INITIALIZED;
1025 evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
1026
1027 /* Clear the CPU information */
1028 sep->se_cpu_id = 0;
1029
1030 /* Clear the event count */
1031 sep->se_count = 0;
1032
1033 /* Reset the exception flag */
1034 sep->se_exception = B_FALSE;
1035
1036 /* Destroy the event queue */
1037 efx_ev_qdestroy(sep->se_eep);
1038 sep->se_eep = NULL;
1039
1040 mutex_exit(&(sep->se_lock));
1041
1042 /* Zero out the event handlers */
1043 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
1044
1045 /* Clear entries from the buffer table */
1046 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
1047 }
1048
1049 static void
1050 sfxge_ev_qfini(sfxge_t *sp, unsigned int index)
1051 {
1052 sfxge_evq_t *sep = sp->s_sep[index];
1053
1054 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
1055
1056 sp->s_sep[index] = NULL;
1057 sep->se_state = SFXGE_EVQ_UNINITIALIZED;
1058
1059 /* Tear down the statistics */
1060 sfxge_ev_kstat_fini(sep);
1061
1062 cv_destroy(&(sep->se_init_kv));
1063 mutex_destroy(&(sep->se_lock));
1064
1065 sep->se_index = 0;
1066
1067 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
1068 }
1069
1070 int
1071 sfxge_ev_txlabel_alloc(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
1072 unsigned int *labelp)
1073 {
1074 sfxge_evq_t *sep = sp->s_sep[evq];
1075 sfxge_txq_t **stpp;
1076 unsigned int label;
1077 int rc;
1078
1079 mutex_enter(&(sep->se_lock));
1080
1081 if (stp == NULL || labelp == NULL) {
1082 rc = EINVAL;
1083 goto fail1;
1084 }
1085
1086 stpp = NULL;
1087 for (label = 0; label < SFXGE_TX_NLABELS; label++) {
1088 if (sep->se_label_stp[label] == stp) {
1089 rc = EEXIST;
1090 goto fail2;
1091 }
1092 if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) {
1093 stpp = &sep->se_label_stp[label];
1094 }
1095 }
1096 if (stpp == NULL) {
1097 rc = ENOSPC;
1098 goto fail3;
1099 }
1100 *stpp = stp;
1101 label = stpp - sep->se_label_stp;
1102
1103 ASSERT3U(label, <, SFXGE_TX_NLABELS);
1104 *labelp = label;
1105
1106 mutex_exit(&(sep->se_lock));
1107 return (0);
1108
1109 fail3:
1110 DTRACE_PROBE(fail3);
1111 fail2:
1112 DTRACE_PROBE(fail2);
1113 fail1:
1114 DTRACE_PROBE1(fail1, int, rc);
1115
1116 mutex_exit(&(sep->se_lock));
1117
1118 return (rc);
1119 }
1120
1121
1122 int
1123 sfxge_ev_txlabel_free(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
1124 unsigned int label)
1125 {
1126 sfxge_evq_t *sep = sp->s_sep[evq];
1127 int rc;
1128
1129 mutex_enter(&(sep->se_lock));
1130
1131 if (stp == NULL || label > SFXGE_TX_NLABELS) {
1132 rc = EINVAL;
1133 goto fail1;
1134 }
1135
1136 if (sep->se_label_stp[label] != stp) {
1137 rc = EINVAL;
1138 goto fail2;
1139 }
1140 sep->se_label_stp[label] = NULL;
1141
1142 mutex_exit(&(sep->se_lock));
1143
1144 return (0);
1145
1146 fail2:
1147 DTRACE_PROBE(fail2);
1148 fail1:
1149 DTRACE_PROBE1(fail1, int, rc);
1150
1151 mutex_exit(&(sep->se_lock));
1152
1153 return (rc);
1154 }
1155
1156
1157 static kmem_cache_t *
1158 sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname,
1159 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *))
1160 {
1161 char name[MAXNAMELEN];
1162 kmem_cache_t *eqc;
1163
1164 (void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache",
1165 ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname);
1166
1167 eqc = kmem_cache_create(name, sizeof (sfxge_evq_t),
1168 SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0);
1169 ASSERT(eqc != NULL);
1170 return (eqc);
1171 }
1172
1173 int
1174 sfxge_ev_init(sfxge_t *sp)
1175 {
1176 sfxge_intr_t *sip = &(sp->s_intr);
1177 unsigned int evq0_size;
1178 unsigned int evqX_size;
1179 unsigned int ev_batch;
1180 int index;
1181 int rc;
1182
1183 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
1184
1185 /*
1186 * Must account for RXQ, TXQ(s); MCDI not event completed at present
1187 * Note that common code does not completely fill descriptor queues
1188 */
1189 evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS;
1190 evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */
1191 evq0_size += SFXGE_TX_NDESCS; /* no checksums */
1192
1193 ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS);
1194 ASSERT3U(evq0_size, >, evqX_size);
1195
1196 if (evq0_size > EFX_EVQ_MAXNEVS) {
1197 rc = EINVAL;
1198 goto fail1;
1199 }
1200
1201 sp->s_evq0_size = pow2_ge(evq0_size);
1202 sp->s_evqX_size = pow2_ge(evqX_size);
1203
1204 /* Read driver parameters */
1205 sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
1206 DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION);
1207
1208 ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
1209 DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH);
1210
1211 /*
1212 * It is slightly peverse to have a cache for one item. But it allows
1213 * for simple alignment control without increasing the allocation size
1214 */
1215 sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor,
1216 sfxge_ev_q0dtor);
1217 sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor,
1218 sfxge_ev_qXdtor);
1219
1220 /* Initialize the event queue(s) */
1221 for (index = 0; index < sip->si_nalloc; index++) {
1222 if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0)
1223 goto fail2;
1224 }
1225
1226 return (0);
1227
1228 fail2:
1229 DTRACE_PROBE(fail2);
1230
1231 while (--index >= 0)
1232 sfxge_ev_qfini(sp, index);
1233 sp->s_ev_moderation = 0;
1234
1235 fail1:
1236 DTRACE_PROBE1(fail1, int, rc);
1237
1238 kmem_cache_destroy(sp->s_eqXc);
1239 kmem_cache_destroy(sp->s_eq0c);
1240 sp->s_eqXc = NULL;
1241 sp->s_eq0c = NULL;
1242
1243 return (rc);
1244 }
1245
1246 int
1247 sfxge_ev_start(sfxge_t *sp)
1248 {
1249 sfxge_intr_t *sip = &(sp->s_intr);
1250 efx_nic_t *enp = sp->s_enp;
1251 int index;
1252 int rc;
1253
1254 ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
1255
1256 /* Initialize the event module */
1257 if ((rc = efx_ev_init(enp)) != 0)
1258 goto fail1;
1259
1260 /* Start the event queues */
1261 for (index = 0; index < sip->si_nalloc; index++) {
1262 if ((rc = sfxge_ev_qstart(sp, index)) != 0)
1263 goto fail2;
1264 }
1265
1266 return (0);
1267
1268 fail2:
1269 DTRACE_PROBE(fail2);
1270
1271 /* Stop the event queue(s) */
1272 while (--index >= 0)
1273 sfxge_ev_qstop(sp, index);
1274
1275 /* Tear down the event module */
1276 efx_ev_fini(enp);
1277
1278 fail1:
1279 DTRACE_PROBE1(fail1, int, rc);
1280
1281 return (rc);
1282 }
1283
1284 void
1285 sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp)
1286 {
1287 *usp = sp->s_ev_moderation;
1288 }
1289
1290 int
1291 sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us)
1292 {
1293 sfxge_intr_t *sip = &(sp->s_intr);
1294 int index;
1295 int rc;
1296
1297 if (sip->si_state != SFXGE_INTR_STARTED)
1298 return (ENODEV);
1299
1300 for (index = 0; index < sip->si_nalloc; index++) {
1301 if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0)
1302 goto fail1;
1303 }
1304
1305 sp->s_ev_moderation = us;
1306 return (0);
1307
1308 fail1:
1309 DTRACE_PROBE1(fail1, int, rc);
1310
1311 /* The only error path is if the value to set to is invalid. */
1312 ASSERT3U(index, ==, 0);
1313
1314 return (rc);
1315 }
1316
1317 void
1318 sfxge_ev_stop(sfxge_t *sp)
1319 {
1320 sfxge_intr_t *sip = &(sp->s_intr);
1321 efx_nic_t *enp = sp->s_enp;
1322 int index;
1323
1324 ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
1325
1326 /* Stop the event queue(s) */
1327 index = sip->si_nalloc;
1328 while (--index >= 0)
1329 sfxge_ev_qstop(sp, index);
1330
1331 /* Tear down the event module */
1332 efx_ev_fini(enp);
1333 }
1334
1335 void
1336 sfxge_ev_fini(sfxge_t *sp)
1337 {
1338 sfxge_intr_t *sip = &(sp->s_intr);
1339 int index;
1340
1341 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
1342
1343 sp->s_ev_moderation = 0;
1344
1345 /* Tear down the event queue(s) */
1346 index = sip->si_nalloc;
1347 while (--index >= 0)
1348 sfxge_ev_qfini(sp, index);
1349
1350 kmem_cache_destroy(sp->s_eqXc);
1351 kmem_cache_destroy(sp->s_eq0c);
1352 sp->s_eqXc = NULL;
1353 sp->s_eq0c = NULL;
1354 }