1 /*
2 * Copyright 2007-2013 Solarflare Communications Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "efsys.h"
27 #include "efx.h"
28 #include "efx_types.h"
29 #include "efx_regs.h"
30 #include "efx_impl.h"
31
32 __checkReturn int
33 efx_rx_init(
34 __in efx_nic_t *enp)
35 {
36 efx_oword_t oword;
37 unsigned int index;
38 int rc;
39
40 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
41 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
42
43 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
44 rc = EINVAL;
45 goto fail1;
46 }
47
48 if (enp->en_mod_flags & EFX_MOD_RX) {
49 rc = EINVAL;
50 goto fail2;
51 }
52
53 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
54
55 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
56 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
57 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
58 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
59 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
60 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
61 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
62
63 /* Zero the RSS table */
64 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
65 index++) {
66 EFX_ZERO_OWORD(oword);
67 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
68 index, &oword);
69 }
70
71 enp->en_mod_flags |= EFX_MOD_RX;
72 return (0);
73
74 fail2:
75 EFSYS_PROBE(fail2);
76 fail1:
77 EFSYS_PROBE1(fail1, int, rc);
78
79 return (rc);
80 }
81
82 #if EFSYS_OPT_RX_HDR_SPLIT
83 __checkReturn int
84 efx_rx_hdr_split_enable(
85 __in efx_nic_t *enp,
86 __in unsigned int hdr_buf_size,
87 __in unsigned int pld_buf_size)
88 {
89 unsigned int nhdr32;
90 unsigned int npld32;
91 efx_oword_t oword;
92 int rc;
93
94 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
95 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
96 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
97
98 nhdr32 = hdr_buf_size / 32;
99 if ((nhdr32 == 0) ||
100 (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
101 ((hdr_buf_size % 32) != 0)) {
102 rc = EINVAL;
103 goto fail1;
104 }
105
106 npld32 = pld_buf_size / 32;
107 if ((npld32 == 0) ||
108 (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
109 ((pld_buf_size % 32) != 0)) {
110 rc = EINVAL;
111 goto fail2;
112 }
113
114 if (enp->en_rx_qcount > 0) {
115 rc = EBUSY;
116 goto fail3;
117 }
118
119 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
120
121 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
122 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
123 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
124
125 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
126
127 return (0);
128
129 fail3:
130 EFSYS_PROBE(fail3);
131 fail2:
132 EFSYS_PROBE(fail2);
133 fail1:
134 EFSYS_PROBE1(fail1, int, rc);
135
136 return (rc);
137 }
138 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
139
140
141 #if EFSYS_OPT_RX_SCATTER
142 __checkReturn int
143 efx_rx_scatter_enable(
144 __in efx_nic_t *enp,
145 __in unsigned int buf_size)
146 {
147 unsigned int nbuf32;
148 efx_oword_t oword;
149 int rc;
150
151 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
152 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
153 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
154
155 nbuf32 = buf_size / 32;
156 if ((nbuf32 == 0) ||
157 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
158 ((buf_size % 32) != 0)) {
159 rc = EINVAL;
160 goto fail1;
161 }
162
163 if (enp->en_rx_qcount > 0) {
164 rc = EBUSY;
165 goto fail2;
166 }
167
168 /* Set scatter buffer size */
169 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
170 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
171 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
172
173 /* Enable scatter for packets not matching a filter */
174 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
175 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
176 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
177
178 return (0);
179
180 fail2:
181 EFSYS_PROBE(fail2);
182 fail1:
183 EFSYS_PROBE1(fail1, int, rc);
184
185 return (rc);
186 }
187 #endif /* EFSYS_OPT_RX_SCATTER */
188
189
190 #define EFX_RX_LFSR_HASH(_enp, _insert) \
191 do { \
192 efx_oword_t oword; \
193 \
194 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
195 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
196 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
197 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
198 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
199 (_insert) ? 1 : 0); \
200 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
201 \
202 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
203 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
204 &oword); \
205 EFX_SET_OWORD_FIELD(oword, \
206 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
207 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
208 &oword); \
209 } \
210 \
211 _NOTE(CONSTANTCONDITION) \
212 } while (B_FALSE)
213
214 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
215 do { \
216 efx_oword_t oword; \
217 \
218 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
219 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
220 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
221 (_ip) ? 1 : 0); \
222 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
223 (_tcp) ? 0 : 1); \
224 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
225 (_insert) ? 1 : 0); \
226 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
227 \
228 _NOTE(CONSTANTCONDITION) \
229 } while (B_FALSE)
230
231 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
232 do { \
233 efx_oword_t oword; \
234 \
235 if ((_enp)->en_family == EFX_FAMILY_FALCON) { \
236 (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \
237 break; \
238 } \
239 \
240 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
241 EFX_SET_OWORD_FIELD(oword, \
242 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
243 EFX_SET_OWORD_FIELD(oword, \
244 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
245 EFX_SET_OWORD_FIELD(oword, \
246 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
247 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
248 \
249 (_rc) = 0; \
250 \
251 _NOTE(CONSTANTCONDITION) \
252 } while (B_FALSE)
253
254
255 #if EFSYS_OPT_RX_SCALE
256 __checkReturn int
257 efx_rx_scale_mode_set(
258 __in efx_nic_t *enp,
259 __in efx_rx_hash_alg_t alg,
260 __in efx_rx_hash_type_t type,
261 __in boolean_t insert)
262 {
263 int rc;
264
265 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
266 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
267 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
268
269 switch (alg) {
270 case EFX_RX_HASHALG_LFSR:
271 EFX_RX_LFSR_HASH(enp, insert);
272 break;
273
274 case EFX_RX_HASHALG_TOEPLITZ:
275 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
276 type & (1 << EFX_RX_HASH_IPV4),
277 type & (1 << EFX_RX_HASH_TCPIPV4));
278
279 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
280 type & (1 << EFX_RX_HASH_IPV6),
281 type & (1 << EFX_RX_HASH_TCPIPV6),
282 rc);
283 if (rc != 0)
284 goto fail1;
285
286 break;
287
288 default:
289 rc = EINVAL;
290 goto fail2;
291 }
292
293 return (0);
294
295 fail2:
296 EFSYS_PROBE(fail2);
297 fail1:
298 EFSYS_PROBE1(fail1, int, rc);
299
300 EFX_RX_LFSR_HASH(enp, B_FALSE);
301
302 return (rc);
303 }
304 #endif
305
306 #if EFSYS_OPT_RX_SCALE
307 __checkReturn int
308 efx_rx_scale_toeplitz_ipv4_key_set(
309 __in efx_nic_t *enp,
310 __in_ecount(n) uint8_t *key,
311 __in size_t n)
312 {
313 efx_oword_t oword;
314 unsigned int byte;
315 unsigned int offset;
316 int rc;
317
318 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
319 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
320
321 byte = 0;
322
323 /* Write toeplitz hash key */
324 EFX_ZERO_OWORD(oword);
325 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
326 offset > 0 && byte < n;
327 --offset)
328 oword.eo_u8[offset - 1] = key[byte++];
329
330 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
331
332 byte = 0;
333
334 /* Verify toeplitz hash key */
335 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
336 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
337 offset > 0 && byte < n;
338 --offset) {
339 if (oword.eo_u8[offset - 1] != key[byte++]) {
340 rc = EFAULT;
341 goto fail1;
342 }
343 }
344
345 return (0);
346
347 fail1:
348 EFSYS_PROBE1(fail1, int, rc);
349
350 return (rc);
351 }
352 #endif
353
354 #if EFSYS_OPT_RX_SCALE
355 __checkReturn int
356 efx_rx_scale_toeplitz_ipv6_key_set(
357 __in efx_nic_t *enp,
358 __in_ecount(n) uint8_t *key,
359 __in size_t n)
360 {
361 efx_oword_t oword;
362 unsigned int byte;
363 int offset;
364 int rc;
365
366 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
367 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
368
369 byte = 0;
370
371 /* Write toeplitz hash key 3 */
372 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
373 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
374 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
375 offset > 0 && byte < n;
376 --offset)
377 oword.eo_u8[offset - 1] = key[byte++];
378
379 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
380
381 /* Write toeplitz hash key 2 */
382 EFX_ZERO_OWORD(oword);
383 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
384 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
385 offset > 0 && byte < n;
386 --offset)
387 oword.eo_u8[offset - 1] = key[byte++];
388
389 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
390
391 /* Write toeplitz hash key 1 */
392 EFX_ZERO_OWORD(oword);
393 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
394 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
395 offset > 0 && byte < n;
396 --offset)
397 oword.eo_u8[offset - 1] = key[byte++];
398
399 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
400
401 byte = 0;
402
403 /* Verify toeplitz hash key 3 */
404 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
405 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
406 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
407 offset > 0 && byte < n;
408 --offset) {
409 if (oword.eo_u8[offset - 1] != key[byte++]) {
410 rc = EFAULT;
411 goto fail1;
412 }
413 }
414
415 /* Verify toeplitz hash key 2 */
416 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
417 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
418 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
419 offset > 0 && byte < n;
420 --offset) {
421 if (oword.eo_u8[offset - 1] != key[byte++]) {
422 rc = EFAULT;
423 goto fail2;
424 }
425 }
426
427 /* Verify toeplitz hash key 1 */
428 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
429 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
430 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
431 offset > 0 && byte < n;
432 --offset) {
433 if (oword.eo_u8[offset - 1] != key[byte++]) {
434 rc = EFAULT;
435 goto fail3;
436 }
437 }
438
439 return (0);
440
441 fail3:
442 EFSYS_PROBE(fail3);
443 fail2:
444 EFSYS_PROBE(fail2);
445 fail1:
446 EFSYS_PROBE1(fail1, int, rc);
447
448 return (rc);
449 }
450 #endif
451
452 #if EFSYS_OPT_RX_SCALE
453 __checkReturn int
454 efx_rx_scale_tbl_set(
455 __in efx_nic_t *enp,
456 __in_ecount(n) unsigned int *table,
457 __in size_t n)
458 {
459 efx_oword_t oword;
460 int index;
461 int rc;
462
463 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
464 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
465
466 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
467 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
468
469 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
470 rc = EINVAL;
471 goto fail1;
472 }
473
474 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
475 uint32_t byte;
476
477 /* Calculate the entry to place in the table */
478 byte = (uint32_t)table[index % n];
479
480 EFSYS_PROBE2(table, int, index, uint32_t, byte);
481
482 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
483
484 /* Write the table */
485 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
486 index, &oword);
487 }
488
489 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
490 uint32_t byte;
491
492 /* Determine if we're starting a new batch */
493 byte = (uint32_t)table[index % n];
494
495 /* Read the table */
496 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
497 index, &oword);
498
499 /* Verify the entry */
500 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
501 rc = EFAULT;
502 goto fail2;
503 }
504 }
505
506 return (0);
507
508 fail2:
509 EFSYS_PROBE(fail2);
510 fail1:
511 EFSYS_PROBE1(fail1, int, rc);
512
513 return (rc);
514 }
515 #endif
516
517 #if EFSYS_OPT_FILTER
518 extern __checkReturn int
519 efx_rx_filter_insert(
520 __in efx_rxq_t *erp,
521 __inout efx_filter_spec_t *spec)
522 {
523 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
524 EFSYS_ASSERT3P(spec, !=, NULL);
525
526 spec->efs_dmaq_id = (uint16_t)erp->er_index;
527 return (efx_filter_insert_filter(erp->er_enp, spec, B_FALSE));
528 }
529 #endif
530
531 #if EFSYS_OPT_FILTER
532 extern __checkReturn int
533 efx_rx_filter_remove(
534 __in efx_rxq_t *erp,
535 __inout efx_filter_spec_t *spec)
536 {
537 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
538 EFSYS_ASSERT3P(spec, !=, NULL);
539
540 spec->efs_dmaq_id = (uint16_t)erp->er_index;
541 return (efx_filter_remove_filter(erp->er_enp, spec));
542 }
543 #endif
544
545 extern void
546 efx_rx_qpost(
547 __in efx_rxq_t *erp,
548 __in_ecount(n) efsys_dma_addr_t *addrp,
549 __in size_t size,
550 __in unsigned int n,
551 __in unsigned int completed,
552 __in unsigned int added)
553 {
554 efx_qword_t qword;
555 unsigned int i;
556 unsigned int offset;
557 unsigned int id;
558
559 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
560
561 /* The client driver must not overfill the queue */
562 EFSYS_ASSERT3U(added - completed + n, <=,
563 EFX_RXQ_LIMIT(erp->er_mask + 1));
564
565 id = added & (erp->er_mask);
566 for (i = 0; i < n; i++) {
567 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
568 unsigned int, id, efsys_dma_addr_t, addrp[i],
569 size_t, size);
570
571 EFX_POPULATE_QWORD_3(qword,
572 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
573 FSF_AZ_RX_KER_BUF_ADDR_DW0,
574 (uint32_t)(addrp[i] & 0xffffffff),
575 FSF_AZ_RX_KER_BUF_ADDR_DW1,
576 (uint32_t)(addrp[i] >> 32));
577
578 offset = id * sizeof (efx_qword_t);
579 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
580
581 id = (id + 1) & (erp->er_mask);
582 }
583 }
584
585 void
586 efx_rx_qpush(
587 __in efx_rxq_t *erp,
588 __in unsigned int added)
589 {
590 efx_nic_t *enp = erp->er_enp;
591 uint32_t wptr;
592 efx_oword_t oword;
593 efx_dword_t dword;
594
595 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
596
597 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
598 EFSYS_PIO_WRITE_BARRIER();
599
600 /* Push the populated descriptors out */
601 wptr = added & erp->er_mask;
602
603 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
604
605 /* Only write the third DWORD */
606 EFX_POPULATE_DWORD_1(dword,
607 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
608 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
609 erp->er_index, &dword, B_FALSE);
610 }
611
612 void
613 efx_rx_qflush(
614 __in efx_rxq_t *erp)
615 {
616 efx_nic_t *enp = erp->er_enp;
617 efx_oword_t oword;
618 uint32_t label;
619
620 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
621
622 label = erp->er_index;
623
624 /* Flush the queue */
625 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
626 FRF_AZ_RX_FLUSH_DESCQ, label);
627 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
628 }
629
630 void
631 efx_rx_qenable(
632 __in efx_rxq_t *erp)
633 {
634 efx_nic_t *enp = erp->er_enp;
635 efx_oword_t oword;
636
637 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
638
639 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
640 erp->er_index, &oword);
641
642 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
643 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
644 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
645
646 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
647 erp->er_index, &oword);
648 }
649
650 __checkReturn int
651 efx_rx_qcreate(
652 __in efx_nic_t *enp,
653 __in unsigned int index,
654 __in unsigned int label,
655 __in efx_rxq_type_t type,
656 __in efsys_mem_t *esmp,
657 __in size_t n,
658 __in uint32_t id,
659 __in efx_evq_t *eep,
660 __deref_out efx_rxq_t **erpp)
661 {
662 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
663 efx_rxq_t *erp;
664 efx_oword_t oword;
665 uint32_t size;
666 boolean_t split;
667 boolean_t jumbo;
668 int rc;
669
670 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
671 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
672
673 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
674 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
675 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
676 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
677
678 if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
679 rc = EINVAL;
680 goto fail1;
681 }
682 if (index >= encp->enc_rxq_limit) {
683 rc = EINVAL;
684 goto fail2;
685 }
686 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
687 size++)
688 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
689 break;
690 if (id + (1 << size) >= encp->enc_buftbl_limit) {
691 rc = EINVAL;
692 goto fail3;
693 }
694
695 switch (type) {
696 case EFX_RXQ_TYPE_DEFAULT:
697 split = B_FALSE;
698 jumbo = B_FALSE;
699 break;
700
701 #if EFSYS_OPT_RX_HDR_SPLIT
702 case EFX_RXQ_TYPE_SPLIT_HEADER:
703 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
704 rc = EINVAL;
705 goto fail4;
706 }
707 split = B_TRUE;
708 jumbo = B_TRUE;
709 break;
710
711 case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
712 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
713 rc = EINVAL;
714 goto fail4;
715 }
716 split = B_FALSE;
717 jumbo = B_TRUE;
718 break;
719 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
720
721 #if EFSYS_OPT_RX_SCATTER
722 case EFX_RXQ_TYPE_SCATTER:
723 if (enp->en_family < EFX_FAMILY_SIENA) {
724 rc = EINVAL;
725 goto fail4;
726 }
727 split = B_FALSE;
728 jumbo = B_TRUE;
729 break;
730 #endif /* EFSYS_OPT_RX_SCATTER */
731
732 default:
733 rc = EINVAL;
734 goto fail4;
735 }
736
737 /* Allocate an RXQ object */
738 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
739
740 if (erp == NULL) {
741 rc = ENOMEM;
742 goto fail5;
743 }
744
745 erp->er_magic = EFX_RXQ_MAGIC;
746 erp->er_enp = enp;
747 erp->er_index = index;
748 erp->er_mask = n - 1;
749 erp->er_esmp = esmp;
750
751 /* Set up the new descriptor queue */
752 EFX_POPULATE_OWORD_10(oword,
753 FRF_CZ_RX_HDR_SPLIT, split,
754 FRF_AZ_RX_ISCSI_DDIG_EN, 0,
755 FRF_AZ_RX_ISCSI_HDIG_EN, 0,
756 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
757 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
758 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
759 FRF_AZ_RX_DESCQ_LABEL, label,
760 FRF_AZ_RX_DESCQ_SIZE, size,
761 FRF_AZ_RX_DESCQ_TYPE, 0,
762 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
763
764 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
765 erp->er_index, &oword);
766
767 enp->en_rx_qcount++;
768 *erpp = erp;
769 return (0);
770
771 fail5:
772 EFSYS_PROBE(fail5);
773 fail4:
774 EFSYS_PROBE(fail4);
775 fail3:
776 EFSYS_PROBE(fail3);
777 fail2:
778 EFSYS_PROBE(fail2);
779 fail1:
780 EFSYS_PROBE1(fail1, int, rc);
781
782 return (rc);
783 }
784
785 void
786 efx_rx_qdestroy(
787 __in efx_rxq_t *erp)
788 {
789 efx_nic_t *enp = erp->er_enp;
790 efx_oword_t oword;
791
792 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
793
794 EFSYS_ASSERT(enp->en_rx_qcount != 0);
795 --enp->en_rx_qcount;
796
797 /* Purge descriptor queue */
798 EFX_ZERO_OWORD(oword);
799
800 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
801 erp->er_index, &oword);
802
803 /* Free the RXQ object */
804 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
805 }
806
807 void
808 efx_rx_fini(
809 __in efx_nic_t *enp)
810 {
811 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
812 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
813 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
814 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
815
816 enp->en_mod_flags &= ~EFX_MOD_RX;
817 }