1 /*
   2  * Copyright 2007-2013 Solarflare Communications Inc.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25 
  26 #include "efsys.h"
  27 #include "efx.h"
  28 #include "efx_types.h"
  29 #include "efx_regs.h"
  30 #include "efx_impl.h"
  31 
  32         __checkReturn   int
  33 efx_rx_init(
  34         __in            efx_nic_t *enp)
  35 {
  36         efx_oword_t oword;
  37         unsigned int index;
  38         int rc;
  39 
  40         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
  41         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
  42 
  43         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
  44                 rc = EINVAL;
  45                 goto fail1;
  46         }
  47 
  48         if (enp->en_mod_flags & EFX_MOD_RX) {
  49                 rc = EINVAL;
  50                 goto fail2;
  51         }
  52 
  53         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
  54 
  55         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
  56         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
  57         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
  58         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
  59         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
  60         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
  61         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
  62 
  63         /* Zero the RSS table */
  64         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
  65             index++) {
  66                 EFX_ZERO_OWORD(oword);
  67                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
  68                                     index, &oword);
  69         }
  70 
  71         enp->en_mod_flags |= EFX_MOD_RX;
  72         return (0);
  73 
  74 fail2:
  75         EFSYS_PROBE(fail2);
  76 fail1:
  77         EFSYS_PROBE1(fail1, int, rc);
  78 
  79         return (rc);
  80 }
  81 
  82 #if EFSYS_OPT_RX_HDR_SPLIT
  83         __checkReturn   int
  84 efx_rx_hdr_split_enable(
  85         __in            efx_nic_t *enp,
  86         __in            unsigned int hdr_buf_size,
  87         __in            unsigned int pld_buf_size)
  88 {
  89         unsigned int nhdr32;
  90         unsigned int npld32;
  91         efx_oword_t oword;
  92         int rc;
  93 
  94         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
  95         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
  96         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
  97 
  98         nhdr32 = hdr_buf_size / 32;
  99         if ((nhdr32 == 0) ||
 100             (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
 101             ((hdr_buf_size % 32) != 0)) {
 102                 rc = EINVAL;
 103                 goto fail1;
 104         }
 105 
 106         npld32 = pld_buf_size / 32;
 107         if ((npld32 == 0) ||
 108             (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
 109             ((pld_buf_size % 32) != 0)) {
 110                 rc = EINVAL;
 111                 goto fail2;
 112         }
 113 
 114         if (enp->en_rx_qcount > 0) {
 115                 rc = EBUSY;
 116                 goto fail3;
 117         }
 118 
 119         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 120 
 121         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
 122         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
 123         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
 124 
 125         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 126 
 127         return (0);
 128 
 129 fail3:
 130         EFSYS_PROBE(fail3);
 131 fail2:
 132         EFSYS_PROBE(fail2);
 133 fail1:
 134         EFSYS_PROBE1(fail1, int, rc);
 135 
 136         return (rc);
 137 }
 138 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
 139 
 140 
 141 #if EFSYS_OPT_RX_SCATTER
 142         __checkReturn   int
 143 efx_rx_scatter_enable(
 144         __in            efx_nic_t *enp,
 145         __in            unsigned int buf_size)
 146 {
 147         unsigned int nbuf32;
 148         efx_oword_t oword;
 149         int rc;
 150 
 151         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 152         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 153         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
 154 
 155         nbuf32 = buf_size / 32;
 156         if ((nbuf32 == 0) ||
 157             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 158             ((buf_size % 32) != 0)) {
 159                 rc = EINVAL;
 160                 goto fail1;
 161         }
 162 
 163         if (enp->en_rx_qcount > 0) {
 164                 rc = EBUSY;
 165                 goto fail2;
 166         }
 167 
 168         /* Set scatter buffer size */
 169         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 170         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 171         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 172 
 173         /* Enable scatter for packets not matching a filter */
 174         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 175         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 176         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 177 
 178         return (0);
 179 
 180 fail2:
 181         EFSYS_PROBE(fail2);
 182 fail1:
 183         EFSYS_PROBE1(fail1, int, rc);
 184 
 185         return (rc);
 186 }
 187 #endif  /* EFSYS_OPT_RX_SCATTER */
 188 
 189 
 190 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 191         do {                                                            \
 192                 efx_oword_t oword;                                      \
 193                                                                         \
 194                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);    \
 195                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 196                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 197                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 198                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 199                     (_insert) ? 1 : 0);                                 \
 200                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);   \
 201                                                                         \
 202                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {         \
 203                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 204                             &oword);                                        \
 205                         EFX_SET_OWORD_FIELD(oword,                      \
 206                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 207                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 208                             &oword);                                        \
 209                 }                                                       \
 210                                                                         \
 211                 _NOTE(CONSTANTCONDITION)                                \
 212         } while (B_FALSE)
 213 
 214 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 215         do {                                                            \
 216                 efx_oword_t oword;                                      \
 217                                                                         \
 218                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);    \
 219                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 220                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 221                     (_ip) ? 1 : 0);                                     \
 222                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 223                     (_tcp) ? 0 : 1);                                    \
 224                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 225                     (_insert) ? 1 : 0);                                 \
 226                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);   \
 227                                                                         \
 228                 _NOTE(CONSTANTCONDITION)                                \
 229         } while (B_FALSE)
 230 
 231 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 232         do {                                                            \
 233                 efx_oword_t oword;                                      \
 234                                                                         \
 235                 if ((_enp)->en_family == EFX_FAMILY_FALCON) {                \
 236                         (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;        \
 237                         break;                                          \
 238                 }                                                       \
 239                                                                         \
 240                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);      \
 241                 EFX_SET_OWORD_FIELD(oword,                              \
 242                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 243                 EFX_SET_OWORD_FIELD(oword,                              \
 244                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 245                 EFX_SET_OWORD_FIELD(oword,                              \
 246                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 247                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);     \
 248                                                                         \
 249                 (_rc) = 0;                                              \
 250                                                                         \
 251                 _NOTE(CONSTANTCONDITION)                                \
 252         } while (B_FALSE)
 253 
 254 
 255 #if EFSYS_OPT_RX_SCALE
 256         __checkReturn   int
 257 efx_rx_scale_mode_set(
 258         __in            efx_nic_t *enp,
 259         __in            efx_rx_hash_alg_t alg,
 260         __in            efx_rx_hash_type_t type,
 261         __in            boolean_t insert)
 262 {
 263         int rc;
 264 
 265         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 266         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 267         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
 268 
 269         switch (alg) {
 270         case EFX_RX_HASHALG_LFSR:
 271                 EFX_RX_LFSR_HASH(enp, insert);
 272                 break;
 273 
 274         case EFX_RX_HASHALG_TOEPLITZ:
 275                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 276                     type & (1 << EFX_RX_HASH_IPV4),
 277                     type & (1 << EFX_RX_HASH_TCPIPV4));
 278 
 279                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 280                     type & (1 << EFX_RX_HASH_IPV6),
 281                     type & (1 << EFX_RX_HASH_TCPIPV6),
 282                     rc);
 283                 if (rc != 0)
 284                         goto fail1;
 285 
 286                 break;
 287 
 288         default:
 289                 rc = EINVAL;
 290                 goto fail2;
 291         }
 292 
 293         return (0);
 294 
 295 fail2:
 296         EFSYS_PROBE(fail2);
 297 fail1:
 298         EFSYS_PROBE1(fail1, int, rc);
 299 
 300         EFX_RX_LFSR_HASH(enp, B_FALSE);
 301 
 302         return (rc);
 303 }
 304 #endif
 305 
 306 #if EFSYS_OPT_RX_SCALE
 307         __checkReturn   int
 308 efx_rx_scale_toeplitz_ipv4_key_set(
 309         __in            efx_nic_t *enp,
 310         __in_ecount(n)  uint8_t *key,
 311         __in            size_t n)
 312 {
 313         efx_oword_t oword;
 314         unsigned int byte;
 315         unsigned int offset;
 316         int rc;
 317 
 318         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 319         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 320 
 321         byte = 0;
 322 
 323         /* Write toeplitz hash key */
 324         EFX_ZERO_OWORD(oword);
 325         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 326             offset > 0 && byte < n;
 327             --offset)
 328                 oword.eo_u8[offset - 1] = key[byte++];
 329 
 330         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 331 
 332         byte = 0;
 333 
 334         /* Verify toeplitz hash key */
 335         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 336         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 337             offset > 0 && byte < n;
 338             --offset) {
 339                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 340                         rc = EFAULT;
 341                         goto fail1;
 342                 }
 343         }
 344 
 345         return (0);
 346 
 347 fail1:
 348         EFSYS_PROBE1(fail1, int, rc);
 349 
 350         return (rc);
 351 }
 352 #endif
 353 
 354 #if EFSYS_OPT_RX_SCALE
 355         __checkReturn   int
 356 efx_rx_scale_toeplitz_ipv6_key_set(
 357         __in            efx_nic_t *enp,
 358         __in_ecount(n)  uint8_t *key,
 359         __in            size_t n)
 360 {
 361         efx_oword_t oword;
 362         unsigned int byte;
 363         int offset;
 364         int rc;
 365 
 366         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 367         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 368 
 369         byte = 0;
 370 
 371         /* Write toeplitz hash key 3 */
 372         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 373         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 374             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 375             offset > 0 && byte < n;
 376             --offset)
 377                 oword.eo_u8[offset - 1] = key[byte++];
 378 
 379         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 380 
 381         /* Write toeplitz hash key 2 */
 382         EFX_ZERO_OWORD(oword);
 383         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 384             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 385             offset > 0 && byte < n;
 386             --offset)
 387                 oword.eo_u8[offset - 1] = key[byte++];
 388 
 389         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 390 
 391         /* Write toeplitz hash key 1 */
 392         EFX_ZERO_OWORD(oword);
 393         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 394             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 395             offset > 0 && byte < n;
 396             --offset)
 397                 oword.eo_u8[offset - 1] = key[byte++];
 398 
 399         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 400 
 401         byte = 0;
 402 
 403         /* Verify toeplitz hash key 3 */
 404         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 405         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 406             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 407             offset > 0 && byte < n;
 408             --offset) {
 409                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 410                         rc = EFAULT;
 411                         goto fail1;
 412                 }
 413         }
 414 
 415         /* Verify toeplitz hash key 2 */
 416         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 417         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 418             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 419             offset > 0 && byte < n;
 420             --offset) {
 421                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 422                         rc = EFAULT;
 423                         goto fail2;
 424                 }
 425         }
 426 
 427         /* Verify toeplitz hash key 1 */
 428         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 429         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 430             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 431             offset > 0 && byte < n;
 432             --offset) {
 433                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 434                         rc = EFAULT;
 435                         goto fail3;
 436                 }
 437         }
 438 
 439         return (0);
 440 
 441 fail3:
 442         EFSYS_PROBE(fail3);
 443 fail2:
 444         EFSYS_PROBE(fail2);
 445 fail1:
 446         EFSYS_PROBE1(fail1, int, rc);
 447 
 448         return (rc);
 449 }
 450 #endif
 451 
 452 #if EFSYS_OPT_RX_SCALE
 453         __checkReturn   int
 454 efx_rx_scale_tbl_set(
 455         __in            efx_nic_t *enp,
 456         __in_ecount(n)  unsigned int *table,
 457         __in            size_t n)
 458 {
 459         efx_oword_t oword;
 460         int index;
 461         int rc;
 462 
 463         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 464         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 465 
 466         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 467         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 468 
 469         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 470                 rc = EINVAL;
 471                 goto fail1;
 472         }
 473 
 474         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 475                 uint32_t byte;
 476 
 477                 /* Calculate the entry to place in the table */
 478                 byte = (uint32_t)table[index % n];
 479 
 480                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 481 
 482                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 483 
 484                 /* Write the table */
 485                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 486                                     index, &oword);
 487         }
 488 
 489         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
 490                 uint32_t byte;
 491 
 492                 /* Determine if we're starting a new batch */
 493                 byte = (uint32_t)table[index % n];
 494 
 495                 /* Read the table */
 496                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
 497                                     index, &oword);
 498 
 499                 /* Verify the entry */
 500                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
 501                         rc = EFAULT;
 502                         goto fail2;
 503                 }
 504         }
 505 
 506         return (0);
 507 
 508 fail2:
 509         EFSYS_PROBE(fail2);
 510 fail1:
 511         EFSYS_PROBE1(fail1, int, rc);
 512 
 513         return (rc);
 514 }
 515 #endif
 516 
 517 #if EFSYS_OPT_FILTER
 518 extern  __checkReturn   int
 519 efx_rx_filter_insert(
 520         __in            efx_rxq_t *erp,
 521         __inout         efx_filter_spec_t *spec)
 522 {
 523         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 524         EFSYS_ASSERT3P(spec, !=, NULL);
 525 
 526         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 527         return (efx_filter_insert_filter(erp->er_enp, spec, B_FALSE));
 528 }
 529 #endif
 530 
 531 #if EFSYS_OPT_FILTER
 532 extern  __checkReturn   int
 533 efx_rx_filter_remove(
 534         __in            efx_rxq_t *erp,
 535         __inout         efx_filter_spec_t *spec)
 536 {
 537         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 538         EFSYS_ASSERT3P(spec, !=, NULL);
 539 
 540         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 541         return (efx_filter_remove_filter(erp->er_enp, spec));
 542 }
 543 #endif
 544 
 545 extern                  void
 546 efx_rx_qpost(
 547         __in            efx_rxq_t *erp,
 548         __in_ecount(n)  efsys_dma_addr_t *addrp,
 549         __in            size_t size,
 550         __in            unsigned int n,
 551         __in            unsigned int completed,
 552         __in            unsigned int added)
 553 {
 554         efx_qword_t qword;
 555         unsigned int i;
 556         unsigned int offset;
 557         unsigned int id;
 558 
 559         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 560 
 561         /* The client driver must not overfill the queue */
 562         EFSYS_ASSERT3U(added - completed + n, <=,
 563             EFX_RXQ_LIMIT(erp->er_mask + 1));
 564 
 565         id = added & (erp->er_mask);
 566         for (i = 0; i < n; i++) {
 567                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 568                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 569                     size_t, size);
 570 
 571                 EFX_POPULATE_QWORD_3(qword,
 572                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
 573                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
 574                     (uint32_t)(addrp[i] & 0xffffffff),
 575                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
 576                     (uint32_t)(addrp[i] >> 32));
 577 
 578                 offset = id * sizeof (efx_qword_t);
 579                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 580 
 581                 id = (id + 1) & (erp->er_mask);
 582         }
 583 }
 584 
 585                 void
 586 efx_rx_qpush(
 587         __in    efx_rxq_t *erp,
 588         __in    unsigned int added)
 589 {
 590         efx_nic_t *enp = erp->er_enp;
 591         uint32_t wptr;
 592         efx_oword_t oword;
 593         efx_dword_t dword;
 594 
 595         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 596 
 597         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 598         EFSYS_PIO_WRITE_BARRIER();
 599 
 600         /* Push the populated descriptors out */
 601         wptr = added & erp->er_mask;
 602 
 603         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
 604 
 605         /* Only write the third DWORD */
 606         EFX_POPULATE_DWORD_1(dword,
 607             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
 608         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
 609                             erp->er_index, &dword, B_FALSE);
 610 }
 611 
 612                 void
 613 efx_rx_qflush(
 614         __in    efx_rxq_t *erp)
 615 {
 616         efx_nic_t *enp = erp->er_enp;
 617         efx_oword_t oword;
 618         uint32_t label;
 619 
 620         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 621 
 622         label = erp->er_index;
 623 
 624         /* Flush the queue */
 625         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 626             FRF_AZ_RX_FLUSH_DESCQ, label);
 627         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
 628 }
 629 
 630                 void
 631 efx_rx_qenable(
 632         __in    efx_rxq_t *erp)
 633 {
 634         efx_nic_t *enp = erp->er_enp;
 635         efx_oword_t oword;
 636 
 637         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 638 
 639         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
 640                             erp->er_index, &oword);
 641 
 642         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
 643         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
 644         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
 645 
 646         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 647                             erp->er_index, &oword);
 648 }
 649 
 650         __checkReturn   int
 651 efx_rx_qcreate(
 652         __in            efx_nic_t *enp,
 653         __in            unsigned int index,
 654         __in            unsigned int label,
 655         __in            efx_rxq_type_t type,
 656         __in            efsys_mem_t *esmp,
 657         __in            size_t n,
 658         __in            uint32_t id,
 659         __in            efx_evq_t *eep,
 660         __deref_out     efx_rxq_t **erpp)
 661 {
 662         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 663         efx_rxq_t *erp;
 664         efx_oword_t oword;
 665         uint32_t size;
 666         boolean_t split;
 667         boolean_t jumbo;
 668         int rc;
 669 
 670         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 671         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 672 
 673         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
 674             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
 675         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 676         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 677 
 678         if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
 679                 rc = EINVAL;
 680                 goto fail1;
 681         }
 682         if (index >= encp->enc_rxq_limit) {
 683                 rc = EINVAL;
 684                 goto fail2;
 685         }
 686         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
 687             size++)
 688                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
 689                         break;
 690         if (id + (1 << size) >= encp->enc_buftbl_limit) {
 691                 rc = EINVAL;
 692                 goto fail3;
 693         }
 694 
 695         switch (type) {
 696         case EFX_RXQ_TYPE_DEFAULT:
 697                 split = B_FALSE;
 698                 jumbo = B_FALSE;
 699                 break;
 700 
 701 #if EFSYS_OPT_RX_HDR_SPLIT
 702         case EFX_RXQ_TYPE_SPLIT_HEADER:
 703                 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
 704                         rc = EINVAL;
 705                         goto fail4;
 706                 }
 707                 split = B_TRUE;
 708                 jumbo = B_TRUE;
 709                 break;
 710 
 711         case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
 712                 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
 713                         rc = EINVAL;
 714                         goto fail4;
 715                 }
 716                 split = B_FALSE;
 717                 jumbo = B_TRUE;
 718                 break;
 719 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
 720 
 721 #if EFSYS_OPT_RX_SCATTER
 722         case EFX_RXQ_TYPE_SCATTER:
 723                 if (enp->en_family < EFX_FAMILY_SIENA) {
 724                         rc = EINVAL;
 725                         goto fail4;
 726                 }
 727                 split = B_FALSE;
 728                 jumbo = B_TRUE;
 729                 break;
 730 #endif  /* EFSYS_OPT_RX_SCATTER */
 731 
 732         default:
 733                 rc = EINVAL;
 734                 goto fail4;
 735         }
 736 
 737         /* Allocate an RXQ object */
 738         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 739 
 740         if (erp == NULL) {
 741                 rc = ENOMEM;
 742                 goto fail5;
 743         }
 744 
 745         erp->er_magic = EFX_RXQ_MAGIC;
 746         erp->er_enp = enp;
 747         erp->er_index = index;
 748         erp->er_mask = n - 1;
 749         erp->er_esmp = esmp;
 750 
 751         /* Set up the new descriptor queue */
 752         EFX_POPULATE_OWORD_10(oword,
 753             FRF_CZ_RX_HDR_SPLIT, split,
 754             FRF_AZ_RX_ISCSI_DDIG_EN, 0,
 755             FRF_AZ_RX_ISCSI_HDIG_EN, 0,
 756             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
 757             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
 758             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 759             FRF_AZ_RX_DESCQ_LABEL, label,
 760             FRF_AZ_RX_DESCQ_SIZE, size,
 761             FRF_AZ_RX_DESCQ_TYPE, 0,
 762             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
 763 
 764         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 765                             erp->er_index, &oword);
 766 
 767         enp->en_rx_qcount++;
 768         *erpp = erp;
 769         return (0);
 770 
 771 fail5:
 772         EFSYS_PROBE(fail5);
 773 fail4:
 774         EFSYS_PROBE(fail4);
 775 fail3:
 776         EFSYS_PROBE(fail3);
 777 fail2:
 778         EFSYS_PROBE(fail2);
 779 fail1:
 780         EFSYS_PROBE1(fail1, int, rc);
 781 
 782         return (rc);
 783 }
 784 
 785                 void
 786 efx_rx_qdestroy(
 787         __in    efx_rxq_t *erp)
 788 {
 789         efx_nic_t *enp = erp->er_enp;
 790         efx_oword_t oword;
 791 
 792         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 793 
 794         EFSYS_ASSERT(enp->en_rx_qcount != 0);
 795         --enp->en_rx_qcount;
 796 
 797         /* Purge descriptor queue */
 798         EFX_ZERO_OWORD(oword);
 799 
 800         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 801                             erp->er_index, &oword);
 802 
 803         /* Free the RXQ object */
 804         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 805 }
 806 
 807                 void
 808 efx_rx_fini(
 809         __in    efx_nic_t *enp)
 810 {
 811         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 812         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 813         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 814         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 815 
 816         enp->en_mod_flags &= ~EFX_MOD_RX;
 817 }