1 /*
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include "efx.h"
32 #include "efx_impl.h"
33
34 #if EFSYS_OPT_QSTATS
35 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
36 do { \
37 (_etp)->et_stat[_stat]++; \
38 _NOTE(CONSTANTCONDITION) \
39 } while (B_FALSE)
40 #else
41 #define EFX_TX_QSTAT_INCR(_etp, _stat)
42 #endif
43
44 #if EFSYS_OPT_SIENA
45
46 static __checkReturn efx_rc_t
47 siena_tx_init(
48 __in efx_nic_t *enp);
49
50 static void
51 siena_tx_fini(
52 __in efx_nic_t *enp);
53
54 static __checkReturn efx_rc_t
55 siena_tx_qcreate(
56 __in efx_nic_t *enp,
57 __in unsigned int index,
58 __in unsigned int label,
59 __in efsys_mem_t *esmp,
60 __in size_t n,
61 __in uint32_t id,
62 __in uint16_t flags,
63 __in efx_evq_t *eep,
64 __in efx_txq_t *etp,
65 __out unsigned int *addedp);
66
67 static void
68 siena_tx_qdestroy(
69 __in efx_txq_t *etp);
70
71 static __checkReturn efx_rc_t
72 siena_tx_qpost(
73 __in efx_txq_t *etp,
74 __in_ecount(n) efx_buffer_t *eb,
75 __in unsigned int n,
76 __in unsigned int completed,
77 __inout unsigned int *addedp);
78
79 static void
80 siena_tx_qpush(
81 __in efx_txq_t *etp,
82 __in unsigned int added,
83 __in unsigned int pushed);
84
85 static __checkReturn efx_rc_t
86 siena_tx_qpace(
87 __in efx_txq_t *etp,
88 __in unsigned int ns);
89
90 static __checkReturn efx_rc_t
91 siena_tx_qflush(
92 __in efx_txq_t *etp);
93
94 static void
95 siena_tx_qenable(
96 __in efx_txq_t *etp);
97
98 __checkReturn efx_rc_t
99 siena_tx_qdesc_post(
100 __in efx_txq_t *etp,
101 __in_ecount(n) efx_desc_t *ed,
102 __in unsigned int n,
103 __in unsigned int completed,
104 __inout unsigned int *addedp);
105
106 void
107 siena_tx_qdesc_dma_create(
108 __in efx_txq_t *etp,
109 __in efsys_dma_addr_t addr,
110 __in size_t size,
111 __in boolean_t eop,
112 __out efx_desc_t *edp);
113
114 #if EFSYS_OPT_QSTATS
115 static void
116 siena_tx_qstats_update(
117 __in efx_txq_t *etp,
118 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
119 #endif
120
121 #endif /* EFSYS_OPT_SIENA */
122
123
124 #if EFSYS_OPT_SIENA
125 static const efx_tx_ops_t __efx_tx_siena_ops = {
126 siena_tx_init, /* etxo_init */
127 siena_tx_fini, /* etxo_fini */
128 siena_tx_qcreate, /* etxo_qcreate */
129 siena_tx_qdestroy, /* etxo_qdestroy */
130 siena_tx_qpost, /* etxo_qpost */
131 siena_tx_qpush, /* etxo_qpush */
132 siena_tx_qpace, /* etxo_qpace */
133 siena_tx_qflush, /* etxo_qflush */
134 siena_tx_qenable, /* etxo_qenable */
135 NULL, /* etxo_qpio_enable */
136 NULL, /* etxo_qpio_disable */
137 NULL, /* etxo_qpio_write */
138 NULL, /* etxo_qpio_post */
139 siena_tx_qdesc_post, /* etxo_qdesc_post */
140 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
141 NULL, /* etxo_qdesc_tso_create */
142 NULL, /* etxo_qdesc_tso2_create */
143 NULL, /* etxo_qdesc_vlantci_create */
144 #if EFSYS_OPT_QSTATS
145 siena_tx_qstats_update, /* etxo_qstats_update */
146 #endif
147 };
148 #endif /* EFSYS_OPT_SIENA */
149
150 #if EFSYS_OPT_HUNTINGTON
151 static const efx_tx_ops_t __efx_tx_hunt_ops = {
152 ef10_tx_init, /* etxo_init */
153 ef10_tx_fini, /* etxo_fini */
154 ef10_tx_qcreate, /* etxo_qcreate */
155 ef10_tx_qdestroy, /* etxo_qdestroy */
156 ef10_tx_qpost, /* etxo_qpost */
157 ef10_tx_qpush, /* etxo_qpush */
158 ef10_tx_qpace, /* etxo_qpace */
159 ef10_tx_qflush, /* etxo_qflush */
160 ef10_tx_qenable, /* etxo_qenable */
161 ef10_tx_qpio_enable, /* etxo_qpio_enable */
162 ef10_tx_qpio_disable, /* etxo_qpio_disable */
163 ef10_tx_qpio_write, /* etxo_qpio_write */
164 ef10_tx_qpio_post, /* etxo_qpio_post */
165 ef10_tx_qdesc_post, /* etxo_qdesc_post */
166 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
167 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
168 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
169 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
170 #if EFSYS_OPT_QSTATS
171 ef10_tx_qstats_update, /* etxo_qstats_update */
172 #endif
173 };
174 #endif /* EFSYS_OPT_HUNTINGTON */
175
176 #if EFSYS_OPT_MEDFORD
177 static const efx_tx_ops_t __efx_tx_medford_ops = {
178 ef10_tx_init, /* etxo_init */
179 ef10_tx_fini, /* etxo_fini */
180 ef10_tx_qcreate, /* etxo_qcreate */
181 ef10_tx_qdestroy, /* etxo_qdestroy */
182 ef10_tx_qpost, /* etxo_qpost */
183 ef10_tx_qpush, /* etxo_qpush */
184 ef10_tx_qpace, /* etxo_qpace */
185 ef10_tx_qflush, /* etxo_qflush */
186 ef10_tx_qenable, /* etxo_qenable */
187 ef10_tx_qpio_enable, /* etxo_qpio_enable */
188 ef10_tx_qpio_disable, /* etxo_qpio_disable */
189 ef10_tx_qpio_write, /* etxo_qpio_write */
190 ef10_tx_qpio_post, /* etxo_qpio_post */
191 ef10_tx_qdesc_post, /* etxo_qdesc_post */
192 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
193 NULL, /* etxo_qdesc_tso_create */
194 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
195 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
196 #if EFSYS_OPT_QSTATS
197 ef10_tx_qstats_update, /* etxo_qstats_update */
198 #endif
199 };
200 #endif /* EFSYS_OPT_MEDFORD */
201
202 __checkReturn efx_rc_t
203 efx_tx_init(
204 __in efx_nic_t *enp)
205 {
206 const efx_tx_ops_t *etxop;
207 efx_rc_t rc;
208
209 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
210 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
211
212 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
213 rc = EINVAL;
214 goto fail1;
215 }
216
217 if (enp->en_mod_flags & EFX_MOD_TX) {
218 rc = EINVAL;
219 goto fail2;
220 }
221
222 switch (enp->en_family) {
223 #if EFSYS_OPT_SIENA
224 case EFX_FAMILY_SIENA:
225 etxop = &__efx_tx_siena_ops;
226 break;
227 #endif /* EFSYS_OPT_SIENA */
228
229 #if EFSYS_OPT_HUNTINGTON
230 case EFX_FAMILY_HUNTINGTON:
231 etxop = &__efx_tx_hunt_ops;
232 break;
233 #endif /* EFSYS_OPT_HUNTINGTON */
234
235 #if EFSYS_OPT_MEDFORD
236 case EFX_FAMILY_MEDFORD:
237 etxop = &__efx_tx_medford_ops;
238 break;
239 #endif /* EFSYS_OPT_MEDFORD */
240
241 default:
242 EFSYS_ASSERT(0);
243 rc = ENOTSUP;
244 goto fail3;
245 }
246
247 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
248
249 if ((rc = etxop->etxo_init(enp)) != 0)
250 goto fail4;
251
252 enp->en_etxop = etxop;
253 enp->en_mod_flags |= EFX_MOD_TX;
254 return (0);
255
256 fail4:
257 EFSYS_PROBE(fail4);
258 fail3:
259 EFSYS_PROBE(fail3);
260 fail2:
261 EFSYS_PROBE(fail2);
262 fail1:
263 EFSYS_PROBE1(fail1, efx_rc_t, rc);
264
265 enp->en_etxop = NULL;
266 enp->en_mod_flags &= ~EFX_MOD_TX;
267 return (rc);
268 }
269
270 void
271 efx_tx_fini(
272 __in efx_nic_t *enp)
273 {
274 const efx_tx_ops_t *etxop = enp->en_etxop;
275
276 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
277 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
278 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
279 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
280
281 etxop->etxo_fini(enp);
282
283 enp->en_etxop = NULL;
284 enp->en_mod_flags &= ~EFX_MOD_TX;
285 }
286
287 __checkReturn efx_rc_t
288 efx_tx_qcreate(
289 __in efx_nic_t *enp,
290 __in unsigned int index,
291 __in unsigned int label,
292 __in efsys_mem_t *esmp,
293 __in size_t n,
294 __in uint32_t id,
295 __in uint16_t flags,
296 __in efx_evq_t *eep,
297 __deref_out efx_txq_t **etpp,
298 __out unsigned int *addedp)
299 {
300 const efx_tx_ops_t *etxop = enp->en_etxop;
301 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
302 efx_txq_t *etp;
303 efx_rc_t rc;
304
305 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
306 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
307
308 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
309
310 /* Allocate an TXQ object */
311 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
312
313 if (etp == NULL) {
314 rc = ENOMEM;
315 goto fail1;
316 }
317
318 etp->et_magic = EFX_TXQ_MAGIC;
319 etp->et_enp = enp;
320 etp->et_index = index;
321 etp->et_mask = n - 1;
322 etp->et_esmp = esmp;
323
324 /* Initial descriptor index may be modified by etxo_qcreate */
325 *addedp = 0;
326
327 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
328 n, id, flags, eep, etp, addedp)) != 0)
329 goto fail2;
330
331 enp->en_tx_qcount++;
332 *etpp = etp;
333
334 return (0);
335
336 fail2:
337 EFSYS_PROBE(fail2);
338 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
339 fail1:
340 EFSYS_PROBE1(fail1, efx_rc_t, rc);
341 return (rc);
342 }
343
344 void
345 efx_tx_qdestroy(
346 __in efx_txq_t *etp)
347 {
348 efx_nic_t *enp = etp->et_enp;
349 const efx_tx_ops_t *etxop = enp->en_etxop;
350
351 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
352
353 EFSYS_ASSERT(enp->en_tx_qcount != 0);
354 --enp->en_tx_qcount;
355
356 etxop->etxo_qdestroy(etp);
357
358 /* Free the TXQ object */
359 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
360 }
361
362 __checkReturn efx_rc_t
363 efx_tx_qpost(
364 __in efx_txq_t *etp,
365 __in_ecount(n) efx_buffer_t *eb,
366 __in unsigned int n,
367 __in unsigned int completed,
368 __inout unsigned int *addedp)
369 {
370 efx_nic_t *enp = etp->et_enp;
371 const efx_tx_ops_t *etxop = enp->en_etxop;
372 efx_rc_t rc;
373
374 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
375
376 if ((rc = etxop->etxo_qpost(etp, eb,
377 n, completed, addedp)) != 0)
378 goto fail1;
379
380 return (0);
381
382 fail1:
383 EFSYS_PROBE1(fail1, efx_rc_t, rc);
384 return (rc);
385 }
386
387 void
388 efx_tx_qpush(
389 __in efx_txq_t *etp,
390 __in unsigned int added,
391 __in unsigned int pushed)
392 {
393 efx_nic_t *enp = etp->et_enp;
394 const efx_tx_ops_t *etxop = enp->en_etxop;
395
396 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
397
398 etxop->etxo_qpush(etp, added, pushed);
399 }
400
401 __checkReturn efx_rc_t
402 efx_tx_qpace(
403 __in efx_txq_t *etp,
404 __in unsigned int ns)
405 {
406 efx_nic_t *enp = etp->et_enp;
407 const efx_tx_ops_t *etxop = enp->en_etxop;
408 efx_rc_t rc;
409
410 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
411
412 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
413 goto fail1;
414
415 return (0);
416
417 fail1:
418 EFSYS_PROBE1(fail1, efx_rc_t, rc);
419 return (rc);
420 }
421
422 __checkReturn efx_rc_t
423 efx_tx_qflush(
424 __in efx_txq_t *etp)
425 {
426 efx_nic_t *enp = etp->et_enp;
427 const efx_tx_ops_t *etxop = enp->en_etxop;
428 efx_rc_t rc;
429
430 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
431
432 if ((rc = etxop->etxo_qflush(etp)) != 0)
433 goto fail1;
434
435 return (0);
436
437 fail1:
438 EFSYS_PROBE1(fail1, efx_rc_t, rc);
439 return (rc);
440 }
441
442 void
443 efx_tx_qenable(
444 __in efx_txq_t *etp)
445 {
446 efx_nic_t *enp = etp->et_enp;
447 const efx_tx_ops_t *etxop = enp->en_etxop;
448
449 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
450
451 etxop->etxo_qenable(etp);
452 }
453
454 __checkReturn efx_rc_t
455 efx_tx_qpio_enable(
456 __in efx_txq_t *etp)
457 {
458 efx_nic_t *enp = etp->et_enp;
459 const efx_tx_ops_t *etxop = enp->en_etxop;
460 efx_rc_t rc;
461
462 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
463
464 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
465 rc = ENOTSUP;
466 goto fail1;
467 }
468 if (etxop->etxo_qpio_enable == NULL) {
469 rc = ENOTSUP;
470 goto fail2;
471 }
472 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
473 goto fail3;
474
475 return (0);
476
477 fail3:
478 EFSYS_PROBE(fail3);
479 fail2:
480 EFSYS_PROBE(fail2);
481 fail1:
482 EFSYS_PROBE1(fail1, efx_rc_t, rc);
483 return (rc);
484 }
485
486 void
487 efx_tx_qpio_disable(
488 __in efx_txq_t *etp)
489 {
490 efx_nic_t *enp = etp->et_enp;
491 const efx_tx_ops_t *etxop = enp->en_etxop;
492
493 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
494
495 if (etxop->etxo_qpio_disable != NULL)
496 etxop->etxo_qpio_disable(etp);
497 }
498
499 __checkReturn efx_rc_t
500 efx_tx_qpio_write(
501 __in efx_txq_t *etp,
502 __in_ecount(buf_length) uint8_t *buffer,
503 __in size_t buf_length,
504 __in size_t pio_buf_offset)
505 {
506 efx_nic_t *enp = etp->et_enp;
507 const efx_tx_ops_t *etxop = enp->en_etxop;
508 efx_rc_t rc;
509
510 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
511
512 if (etxop->etxo_qpio_write != NULL) {
513 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
514 pio_buf_offset)) != 0)
515 goto fail1;
516 return (0);
517 }
518
519 return (ENOTSUP);
520
521 fail1:
522 EFSYS_PROBE1(fail1, efx_rc_t, rc);
523 return (rc);
524 }
525
526 __checkReturn efx_rc_t
527 efx_tx_qpio_post(
528 __in efx_txq_t *etp,
529 __in size_t pkt_length,
530 __in unsigned int completed,
531 __inout unsigned int *addedp)
532 {
533 efx_nic_t *enp = etp->et_enp;
534 const efx_tx_ops_t *etxop = enp->en_etxop;
535 efx_rc_t rc;
536
537 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
538
539 if (etxop->etxo_qpio_post != NULL) {
540 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
541 addedp)) != 0)
542 goto fail1;
543 return (0);
544 }
545
546 return (ENOTSUP);
547
548 fail1:
549 EFSYS_PROBE1(fail1, efx_rc_t, rc);
550 return (rc);
551 }
552
553 __checkReturn efx_rc_t
554 efx_tx_qdesc_post(
555 __in efx_txq_t *etp,
556 __in_ecount(n) efx_desc_t *ed,
557 __in unsigned int n,
558 __in unsigned int completed,
559 __inout unsigned int *addedp)
560 {
561 efx_nic_t *enp = etp->et_enp;
562 const efx_tx_ops_t *etxop = enp->en_etxop;
563 efx_rc_t rc;
564
565 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
566
567 if ((rc = etxop->etxo_qdesc_post(etp, ed,
568 n, completed, addedp)) != 0)
569 goto fail1;
570
571 return (0);
572
573 fail1:
574 EFSYS_PROBE1(fail1, efx_rc_t, rc);
575 return (rc);
576 }
577
578 void
579 efx_tx_qdesc_dma_create(
580 __in efx_txq_t *etp,
581 __in efsys_dma_addr_t addr,
582 __in size_t size,
583 __in boolean_t eop,
584 __out efx_desc_t *edp)
585 {
586 efx_nic_t *enp = etp->et_enp;
587 const efx_tx_ops_t *etxop = enp->en_etxop;
588
589 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
590 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
591
592 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
593 }
594
595 void
596 efx_tx_qdesc_tso_create(
597 __in efx_txq_t *etp,
598 __in uint16_t ipv4_id,
599 __in uint32_t tcp_seq,
600 __in uint8_t tcp_flags,
601 __out efx_desc_t *edp)
602 {
603 efx_nic_t *enp = etp->et_enp;
604 const efx_tx_ops_t *etxop = enp->en_etxop;
605
606 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
607 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
608
609 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
610 }
611
612 void
613 efx_tx_qdesc_tso2_create(
614 __in efx_txq_t *etp,
615 __in uint16_t ipv4_id,
616 __in uint32_t tcp_seq,
617 __in uint16_t mss,
618 __out_ecount(count) efx_desc_t *edp,
619 __in int count)
620 {
621 efx_nic_t *enp = etp->et_enp;
622 const efx_tx_ops_t *etxop = enp->en_etxop;
623
624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
625 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
626
627 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
628 }
629
630 void
631 efx_tx_qdesc_vlantci_create(
632 __in efx_txq_t *etp,
633 __in uint16_t tci,
634 __out efx_desc_t *edp)
635 {
636 efx_nic_t *enp = etp->et_enp;
637 const efx_tx_ops_t *etxop = enp->en_etxop;
638
639 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
640 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
641
642 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
643 }
644
645
646 #if EFSYS_OPT_QSTATS
647 void
648 efx_tx_qstats_update(
649 __in efx_txq_t *etp,
650 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
651 {
652 efx_nic_t *enp = etp->et_enp;
653 const efx_tx_ops_t *etxop = enp->en_etxop;
654
655 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
656
657 etxop->etxo_qstats_update(etp, stat);
658 }
659 #endif
660
661
662 #if EFSYS_OPT_SIENA
663
664 static __checkReturn efx_rc_t
665 siena_tx_init(
666 __in efx_nic_t *enp)
667 {
668 efx_oword_t oword;
669
670 /*
671 * Disable the timer-based TX DMA backoff and allow TX DMA to be
672 * controlled by the RX FIFO fill level (although always allow a
673 * minimal trickle).
674 */
675 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
676 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
677 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
678 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
683
684 /*
685 * Filter all packets less than 14 bytes to avoid parsing
686 * errors.
687 */
688 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
689 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
690
691 /*
692 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
693 * descriptors (which is bad).
694 */
695 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
696 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
697 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
698
699 return (0);
700 }
701
702 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
703 do { \
704 unsigned int id; \
705 size_t offset; \
706 efx_qword_t qword; \
707 \
708 id = (_added)++ & (_etp)->et_mask; \
709 offset = id * sizeof (efx_qword_t); \
710 \
711 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
712 unsigned int, id, efsys_dma_addr_t, (_addr), \
713 size_t, (_size), boolean_t, (_eop)); \
714 \
715 EFX_POPULATE_QWORD_4(qword, \
716 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
717 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
718 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
719 (uint32_t)((_addr) & 0xffffffff), \
720 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
721 (uint32_t)((_addr) >> 32)); \
722 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
723 \
724 _NOTE(CONSTANTCONDITION) \
725 } while (B_FALSE)
726
727 static __checkReturn efx_rc_t
728 siena_tx_qpost(
729 __in efx_txq_t *etp,
730 __in_ecount(n) efx_buffer_t *eb,
731 __in unsigned int n,
732 __in unsigned int completed,
733 __inout unsigned int *addedp)
734 {
735 unsigned int added = *addedp;
736 unsigned int i;
737 int rc = ENOSPC;
738
739 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
740 goto fail1;
741
742 for (i = 0; i < n; i++) {
743 efx_buffer_t *ebp = &eb[i];
744 efsys_dma_addr_t start = ebp->eb_addr;
745 size_t size = ebp->eb_size;
746 efsys_dma_addr_t end = start + size;
747
748 /* Fragments must not span 4k boundaries. */
749 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
750
751 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
752 }
753
754 EFX_TX_QSTAT_INCR(etp, TX_POST);
755
756 *addedp = added;
757 return (0);
758
759 fail1:
760 EFSYS_PROBE1(fail1, efx_rc_t, rc);
761
762 return (rc);
763 }
764
765 static void
766 siena_tx_qpush(
767 __in efx_txq_t *etp,
768 __in unsigned int added,
769 __in unsigned int pushed)
770 {
771 efx_nic_t *enp = etp->et_enp;
772 uint32_t wptr;
773 efx_dword_t dword;
774 efx_oword_t oword;
775
776 /* Push the populated descriptors out */
777 wptr = added & etp->et_mask;
778
779 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
780
781 /* Only write the third DWORD */
782 EFX_POPULATE_DWORD_1(dword,
783 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
784
785 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
786 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
787 wptr, pushed & etp->et_mask);
788 EFSYS_PIO_WRITE_BARRIER();
789 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
790 etp->et_index, &dword, B_FALSE);
791 }
792
793 #define EFX_MAX_PACE_VALUE 20
794
795 static __checkReturn efx_rc_t
796 siena_tx_qpace(
797 __in efx_txq_t *etp,
798 __in unsigned int ns)
799 {
800 efx_nic_t *enp = etp->et_enp;
801 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
802 efx_oword_t oword;
803 unsigned int pace_val;
804 unsigned int timer_period;
805 efx_rc_t rc;
806
807 if (ns == 0) {
808 pace_val = 0;
809 } else {
810 /*
811 * The pace_val to write into the table is s.t
812 * ns <= timer_period * (2 ^ pace_val)
813 */
814 timer_period = 104 / encp->enc_clk_mult;
815 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
816 if ((timer_period << pace_val) >= ns)
817 break;
818 }
819 }
820 if (pace_val > EFX_MAX_PACE_VALUE) {
821 rc = EINVAL;
822 goto fail1;
823 }
824
825 /* Update the pacing table */
826 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
827 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
828 &oword, B_TRUE);
829
830 return (0);
831
832 fail1:
833 EFSYS_PROBE1(fail1, efx_rc_t, rc);
834
835 return (rc);
836 }
837
838 static __checkReturn efx_rc_t
839 siena_tx_qflush(
840 __in efx_txq_t *etp)
841 {
842 efx_nic_t *enp = etp->et_enp;
843 efx_oword_t oword;
844 uint32_t label;
845
846 (void) efx_tx_qpace(etp, 0);
847
848 label = etp->et_index;
849
850 /* Flush the queue */
851 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
852 FRF_AZ_TX_FLUSH_DESCQ, label);
853 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
854
855 return (0);
856 }
857
858 static void
859 siena_tx_qenable(
860 __in efx_txq_t *etp)
861 {
862 efx_nic_t *enp = etp->et_enp;
863 efx_oword_t oword;
864
865 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
866 etp->et_index, &oword, B_TRUE);
867
868 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
869 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
870 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
871 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
872 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
873
874 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
875 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
876 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
877
878 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
879 etp->et_index, &oword, B_TRUE);
880 }
881
882 static __checkReturn efx_rc_t
883 siena_tx_qcreate(
884 __in efx_nic_t *enp,
885 __in unsigned int index,
886 __in unsigned int label,
887 __in efsys_mem_t *esmp,
888 __in size_t n,
889 __in uint32_t id,
890 __in uint16_t flags,
891 __in efx_evq_t *eep,
892 __in efx_txq_t *etp,
893 __out unsigned int *addedp)
894 {
895 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
896 efx_oword_t oword;
897 uint32_t size;
898 efx_rc_t rc;
899
900 _NOTE(ARGUNUSED(esmp));
901
902 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
903 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
904 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
905
906 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
907 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
908
909 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
910 rc = EINVAL;
911 goto fail1;
912 }
913 if (index >= encp->enc_txq_limit) {
914 rc = EINVAL;
915 goto fail2;
916 }
917 for (size = 0;
918 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
919 size++)
920 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
921 break;
922 if (id + (1 << size) >= encp->enc_buftbl_limit) {
923 rc = EINVAL;
924 goto fail3;
925 }
926
927 /* Set up the new descriptor queue */
928 *addedp = 0;
929
930 EFX_POPULATE_OWORD_6(oword,
931 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
932 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
933 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
934 FRF_AZ_TX_DESCQ_LABEL, label,
935 FRF_AZ_TX_DESCQ_SIZE, size,
936 FRF_AZ_TX_DESCQ_TYPE, 0);
937
938 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
939 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
940 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
941 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
942 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
943
944 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
945 etp->et_index, &oword, B_TRUE);
946
947 return (0);
948
949 fail3:
950 EFSYS_PROBE(fail3);
951 fail2:
952 EFSYS_PROBE(fail2);
953 fail1:
954 EFSYS_PROBE1(fail1, efx_rc_t, rc);
955
956 return (rc);
957 }
958
959 __checkReturn efx_rc_t
960 siena_tx_qdesc_post(
961 __in efx_txq_t *etp,
962 __in_ecount(n) efx_desc_t *ed,
963 __in unsigned int n,
964 __in unsigned int completed,
965 __inout unsigned int *addedp)
966 {
967 unsigned int added = *addedp;
968 unsigned int i;
969 efx_rc_t rc;
970
971 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
972 rc = ENOSPC;
973 goto fail1;
974 }
975
976 for (i = 0; i < n; i++) {
977 efx_desc_t *edp = &ed[i];
978 unsigned int id;
979 size_t offset;
980
981 id = added++ & etp->et_mask;
982 offset = id * sizeof (efx_desc_t);
983
984 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
985 }
986
987 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
988 unsigned int, added, unsigned int, n);
989
990 EFX_TX_QSTAT_INCR(etp, TX_POST);
991
992 *addedp = added;
993 return (0);
994
995 fail1:
996 EFSYS_PROBE1(fail1, efx_rc_t, rc);
997 return (rc);
998 }
999
1000 void
1001 siena_tx_qdesc_dma_create(
1002 __in efx_txq_t *etp,
1003 __in efsys_dma_addr_t addr,
1004 __in size_t size,
1005 __in boolean_t eop,
1006 __out efx_desc_t *edp)
1007 {
1008 /* Fragments must not span 4k boundaries. */
1009 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
1010
1011 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1012 efsys_dma_addr_t, addr,
1013 size_t, size, boolean_t, eop);
1014
1015 EFX_POPULATE_QWORD_4(edp->ed_eq,
1016 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1017 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1018 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1019 (uint32_t)(addr & 0xffffffff),
1020 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1021 (uint32_t)(addr >> 32));
1022 }
1023
1024 #endif /* EFSYS_OPT_SIENA */
1025
1026 #if EFSYS_OPT_QSTATS
1027 #if EFSYS_OPT_NAMES
1028 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
1029 static const char *__efx_tx_qstat_name[] = {
1030 "post",
1031 "post_pio",
1032 };
1033 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1034
1035 const char *
1036 efx_tx_qstat_name(
1037 __in efx_nic_t *enp,
1038 __in unsigned int id)
1039 {
1040 _NOTE(ARGUNUSED(enp))
1041 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1042 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1043
1044 return (__efx_tx_qstat_name[id]);
1045 }
1046 #endif /* EFSYS_OPT_NAMES */
1047 #endif /* EFSYS_OPT_QSTATS */
1048
1049 #if EFSYS_OPT_SIENA
1050
1051 #if EFSYS_OPT_QSTATS
1052 static void
1053 siena_tx_qstats_update(
1054 __in efx_txq_t *etp,
1055 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1056 {
1057 unsigned int id;
1058
1059 for (id = 0; id < TX_NQSTATS; id++) {
1060 efsys_stat_t *essp = &stat[id];
1061
1062 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1063 etp->et_stat[id] = 0;
1064 }
1065 }
1066 #endif /* EFSYS_OPT_QSTATS */
1067
1068 static void
1069 siena_tx_qdestroy(
1070 __in efx_txq_t *etp)
1071 {
1072 efx_nic_t *enp = etp->et_enp;
1073 efx_oword_t oword;
1074
1075 /* Purge descriptor queue */
1076 EFX_ZERO_OWORD(oword);
1077
1078 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1079 etp->et_index, &oword, B_TRUE);
1080 }
1081
1082 static void
1083 siena_tx_fini(
1084 __in efx_nic_t *enp)
1085 {
1086 _NOTE(ARGUNUSED(enp))
1087 }
1088
1089 #endif /* EFSYS_OPT_SIENA */