1 /*
2 * Copyright 2007-2013 Solarflare Communications Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "efsys.h"
27 #include "efx.h"
28 #include "efx_types.h"
29 #include "efx_regs.h"
30 #include "efx_impl.h"
31
32 #if EFSYS_OPT_QSTATS
33 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
34 do { \
35 (_etp)->et_stat[_stat]++; \
36 _NOTE(CONSTANTCONDITION) \
37 } while (B_FALSE)
38 #else
39 #define EFX_TX_QSTAT_INCR(_etp, _stat)
40 #endif
41
42
43
44 __checkReturn int
45 efx_tx_init(
46 __in efx_nic_t *enp)
47 {
48 efx_oword_t oword;
49 int rc;
50
51 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
52 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
53
54 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
55 rc = EINVAL;
56 goto fail1;
57 }
58
59 if (enp->en_mod_flags & EFX_MOD_TX) {
60 rc = EINVAL;
61 goto fail2;
62 }
63
64 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
65
66 /*
67 * Disable the timer-based TX DMA backoff and allow TX DMA to be
68 * controlled by the RX FIFO fill level (although always allow a
69 * minimal trickle).
70 */
71 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
72 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
79
80 /*
81 * Filter all packets less than 14 bytes to avoid parsing
82 * errors.
83 */
84 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
85 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
86
87 /*
88 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
89 * descriptors (which is bad).
90 */
91 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
92 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
93 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
94
95 enp->en_mod_flags |= EFX_MOD_TX;
96 return (0);
97
98 fail2:
99 EFSYS_PROBE(fail2);
100 fail1:
101 EFSYS_PROBE1(fail1, int, rc);
102
103 return (rc);
104 }
105
106 #if EFSYS_OPT_FILTER
107 extern __checkReturn int
108 efx_tx_filter_insert(
109 __in efx_txq_t *etp,
110 __inout efx_filter_spec_t *spec)
111 {
112 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
113 EFSYS_ASSERT3P(spec, !=, NULL);
114
115 spec->efs_dmaq_id = (uint16_t)etp->et_index;
116 return (efx_filter_insert_filter(etp->et_enp, spec, B_FALSE));
117 }
118 #endif
119
120 #if EFSYS_OPT_FILTER
121 extern __checkReturn int
122 efx_tx_filter_remove(
123 __in efx_txq_t *etp,
124 __inout efx_filter_spec_t *spec)
125 {
126 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
127 EFSYS_ASSERT3P(spec, !=, NULL);
128
129 spec->efs_dmaq_id = (uint16_t)etp->et_index;
130 return (efx_filter_remove_filter(etp->et_enp, spec));
131 }
132 #endif
133
134 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
135 do { \
136 unsigned int id; \
137 size_t offset; \
138 efx_qword_t qword; \
139 \
140 id = (_added)++ & (_etp)->et_mask; \
141 offset = id * sizeof (efx_qword_t); \
142 \
143 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
144 unsigned int, id, efsys_dma_addr_t, (_addr), \
145 size_t, (_size), boolean_t, (_eop)); \
146 \
147 EFX_POPULATE_QWORD_4(qword, \
148 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
149 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
150 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
151 (uint32_t)((_addr) & 0xffffffff), \
152 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
153 (uint32_t)((_addr) >> 32)); \
154 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
155 \
156 _NOTE(CONSTANTCONDITION) \
157 } while (B_FALSE)
158
159 __checkReturn int
160 efx_tx_qpost(
161 __in efx_txq_t *etp,
162 __in_ecount(n) efx_buffer_t *eb,
163 __in unsigned int n,
164 __in unsigned int completed,
165 __inout unsigned int *addedp)
166 {
167 unsigned int added = *addedp;
168 unsigned int i;
169 int rc = ENOSPC;
170
171 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
172
173 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
174 goto fail1;
175
176 for (i = 0; i < n; i++) {
177 efx_buffer_t *ebp = &eb[i];
178 efsys_dma_addr_t start = ebp->eb_addr;
179 size_t size = ebp->eb_size;
180 efsys_dma_addr_t end = start + size;
181
182 /* Fragments must not span 4k boundaries. */
183 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
184
185 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
186 }
187
188 EFX_TX_QSTAT_INCR(etp, TX_POST);
189
190 *addedp = added;
191 return (0);
192
193 fail1:
194 EFSYS_PROBE1(fail1, int, rc);
195
196 return (rc);
197 }
198
199 void
200 efx_tx_qpush(
201 __in efx_txq_t *etp,
202 __in unsigned int added)
203 {
204 efx_nic_t *enp = etp->et_enp;
205 uint32_t wptr;
206 efx_dword_t dword;
207 efx_oword_t oword;
208
209 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
210
211 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
212 EFSYS_PIO_WRITE_BARRIER();
213
214 /* Push the populated descriptors out */
215 wptr = added & etp->et_mask;
216
217 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
218
219 /* Only write the third DWORD */
220 EFX_POPULATE_DWORD_1(dword,
221 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
222 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
223 etp->et_index, &dword, B_FALSE);
224 }
225
226 #define EFX_MAX_PACE_VALUE 20
227
228 __checkReturn int
229 efx_tx_qpace(
230 __in efx_txq_t *etp,
231 __in unsigned int ns)
232 {
233 efx_nic_t *enp = etp->et_enp;
234 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
235 efx_oword_t oword;
236 unsigned int pace_val;
237 unsigned int timer_period;
238 int rc;
239
240 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
241
242 if (ns == 0) {
243 pace_val = 0;
244 } else {
245 /*
246 * The pace_val to write into the table is s.t
247 * ns <= timer_period * (2 ^ pace_val)
248 */
249 timer_period = 104 / encp->enc_clk_mult;
250 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
251 if ((timer_period << pace_val) >= ns)
252 break;
253 }
254 }
255 if (pace_val > EFX_MAX_PACE_VALUE) {
256 rc = EINVAL;
257 goto fail1;
258 }
259
260 /* Update the pacing table */
261 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
262 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, &oword);
263
264 return (0);
265
266 fail1:
267 EFSYS_PROBE1(fail1, int, rc);
268
269 return (rc);
270 }
271
272 void
273 efx_tx_qflush(
274 __in efx_txq_t *etp)
275 {
276 efx_nic_t *enp = etp->et_enp;
277 efx_oword_t oword;
278 uint32_t label;
279
280 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
281
282 efx_tx_qpace(etp, 0);
283
284 label = etp->et_index;
285
286 /* Flush the queue */
287 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
288 FRF_AZ_TX_FLUSH_DESCQ, label);
289 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
290 }
291
292 void
293 efx_tx_qenable(
294 __in efx_txq_t *etp)
295 {
296 efx_nic_t *enp = etp->et_enp;
297 efx_oword_t oword;
298
299 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
300
301 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
302 etp->et_index, &oword);
303
304 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
305 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
306 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
307 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
308 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
309
310 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
311 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
312 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
313
314 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
315 etp->et_index, &oword);
316 }
317
318 __checkReturn int
319 efx_tx_qcreate(
320 __in efx_nic_t *enp,
321 __in unsigned int index,
322 __in unsigned int label,
323 __in efsys_mem_t *esmp,
324 __in size_t n,
325 __in uint32_t id,
326 __in uint16_t flags,
327 __in efx_evq_t *eep,
328 __deref_out efx_txq_t **etpp)
329 {
330 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
331 efx_txq_t *etp;
332 efx_oword_t oword;
333 uint32_t size;
334 int rc;
335
336 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
337 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
338
339 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
340 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
341 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
342 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
343
344 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
345 rc = EINVAL;
346 goto fail1;
347 }
348 if (index >= encp->enc_txq_limit) {
349 rc = EINVAL;
350 goto fail2;
351 }
352 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
353 size++)
354 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
355 break;
356 if (id + (1 << size) >= encp->enc_buftbl_limit) {
357 rc = EINVAL;
358 goto fail3;
359 }
360
361 /* Allocate an TXQ object */
362 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
363
364 if (etp == NULL) {
365 rc = ENOMEM;
366 goto fail4;
367 }
368
369 etp->et_magic = EFX_TXQ_MAGIC;
370 etp->et_enp = enp;
371 etp->et_index = index;
372 etp->et_mask = n - 1;
373 etp->et_esmp = esmp;
374
375 /* Set up the new descriptor queue */
376 EFX_POPULATE_OWORD_6(oword,
377 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
378 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
379 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
380 FRF_AZ_TX_DESCQ_LABEL, label,
381 FRF_AZ_TX_DESCQ_SIZE, size,
382 FRF_AZ_TX_DESCQ_TYPE, 0);
383
384 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
385 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
386 (flags & EFX_CKSUM_IPV4) ? 0 : 1);
387 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
388 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
389
390 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
391 etp->et_index, &oword);
392
393 enp->en_tx_qcount++;
394 *etpp = etp;
395 return (0);
396
397 fail4:
398 EFSYS_PROBE(fail4);
399 fail3:
400 EFSYS_PROBE(fail3);
401 fail2:
402 EFSYS_PROBE(fail2);
403 fail1:
404 EFSYS_PROBE1(fail1, int, rc);
405
406 return (rc);
407 }
408
409 #if EFSYS_OPT_NAMES
410 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
411 static const char __cs * __cs __efx_tx_qstat_name[] = {
412 "post",
413 "unaligned_split",
414 };
415 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
416
417 const char __cs *
418 efx_tx_qstat_name(
419 __in efx_nic_t *enp,
420 __in unsigned int id)
421 {
422 _NOTE(ARGUNUSED(enp))
423 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
424 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
425
426 return (__efx_tx_qstat_name[id]);
427 }
428 #endif /* EFSYS_OPT_NAMES */
429
430 #if EFSYS_OPT_QSTATS
431 void
432 efx_tx_qstats_update(
433 __in efx_txq_t *etp,
434 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
435 {
436 unsigned int id;
437
438 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
439
440 for (id = 0; id < TX_NQSTATS; id++) {
441 efsys_stat_t *essp = &stat[id];
442
443 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
444 etp->et_stat[id] = 0;
445 }
446 }
447 #endif /* EFSYS_OPT_QSTATS */
448
449 void
450 efx_tx_qdestroy(
451 __in efx_txq_t *etp)
452 {
453 efx_nic_t *enp = etp->et_enp;
454 efx_oword_t oword;
455
456 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
457
458 EFSYS_ASSERT(enp->en_tx_qcount != 0);
459 --enp->en_tx_qcount;
460
461 /* Purge descriptor queue */
462 EFX_ZERO_OWORD(oword);
463
464 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
465 etp->et_index, &oword);
466
467 /* Free the TXQ object */
468 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
469 }
470
471 void
472 efx_tx_fini(
473 __in efx_nic_t *enp)
474 {
475 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
476 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
478 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
479
480 enp->en_mod_flags &= ~EFX_MOD_TX;
481 }