1 /*
   2  * Copyright (c) 2012-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30 
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33 
  34 
  35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  36 
  37 
  38 static  __checkReturn   efx_rc_t
  39 efx_mcdi_init_rxq(
  40         __in            efx_nic_t *enp,
  41         __in            uint32_t size,
  42         __in            uint32_t target_evq,
  43         __in            uint32_t label,
  44         __in            uint32_t instance,
  45         __in            efsys_mem_t *esmp,
  46         __in            boolean_t disable_scatter)
  47 {
  48         efx_mcdi_req_t req;
  49 
  50         uint8_t payload[
  51                 MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS))];
  52         int npages = EFX_RXQ_NBUFS(size);
  53         int i;
  54         efx_qword_t *dma_addr;
  55         uint64_t addr;
  56         efx_rc_t rc;
  57 
  58         /* If this changes, then the payload size might need to change. */
  59         EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
  60         EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
  61 
  62         (void) memset(payload, 0, sizeof (payload));
  63         req.emr_cmd = MC_CMD_INIT_RXQ;
  64         req.emr_in_buf = payload;
  65         req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
  66         req.emr_out_buf = payload;
  67         req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
  68 
  69         MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
  70         MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
  71         MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
  72         MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
  73         MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
  74                             INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
  75                             INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
  76                             INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
  77                             INIT_RXQ_IN_CRC_MODE, 0,
  78                             INIT_RXQ_IN_FLAG_PREFIX, 1,
  79                             INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
  80         MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
  81         MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  82 
  83         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
  84         addr = EFSYS_MEM_ADDR(esmp);
  85 
  86         for (i = 0; i < npages; i++) {
  87                 EFX_POPULATE_QWORD_2(*dma_addr,
  88                     EFX_DWORD_1, (uint32_t)(addr >> 32),
  89                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
  90 
  91                 dma_addr++;
  92                 addr += EFX_BUF_SIZE;
  93         }
  94 
  95         efx_mcdi_execute(enp, &req);
  96 
  97         if (req.emr_rc != 0) {
  98                 rc = req.emr_rc;
  99                 goto fail1;
 100         }
 101 
 102         return (0);
 103 
 104 fail1:
 105         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 106 
 107         return (rc);
 108 }
 109 
 110 static  __checkReturn   efx_rc_t
 111 efx_mcdi_fini_rxq(
 112         __in            efx_nic_t *enp,
 113         __in            uint32_t instance)
 114 {
 115         efx_mcdi_req_t req;
 116         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
 117                             MC_CMD_FINI_RXQ_OUT_LEN)];
 118         efx_rc_t rc;
 119 
 120         (void) memset(payload, 0, sizeof (payload));
 121         req.emr_cmd = MC_CMD_FINI_RXQ;
 122         req.emr_in_buf = payload;
 123         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 124         req.emr_out_buf = payload;
 125         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 126 
 127         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 128 
 129         efx_mcdi_execute(enp, &req);
 130 
 131         if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) {
 132                 rc = req.emr_rc;
 133                 goto fail1;
 134         }
 135 
 136         return (0);
 137 
 138 fail1:
 139         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 140 
 141         return (rc);
 142 }
 143 
 144 #if EFSYS_OPT_RX_SCALE
 145 static  __checkReturn   efx_rc_t
 146 efx_mcdi_rss_context_alloc(
 147         __in            efx_nic_t *enp,
 148         __in            efx_rx_scale_support_t scale_support,
 149         __in            uint32_t num_queues,
 150         __out           uint32_t *rss_contextp)
 151 {
 152         efx_mcdi_req_t req;
 153         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 154                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
 155         uint32_t rss_context;
 156         uint32_t context_type;
 157         efx_rc_t rc;
 158 
 159         if (num_queues > EFX_MAXRSS) {
 160                 rc = EINVAL;
 161                 goto fail1;
 162         }
 163 
 164         switch (scale_support) {
 165         case EFX_RX_SCALE_EXCLUSIVE:
 166                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 167                 break;
 168         case EFX_RX_SCALE_SHARED:
 169                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 170                 break;
 171         default:
 172                 rc = EINVAL;
 173                 goto fail2;
 174         }
 175 
 176         (void) memset(payload, 0, sizeof (payload));
 177         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 178         req.emr_in_buf = payload;
 179         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 180         req.emr_out_buf = payload;
 181         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 182 
 183         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 184             EVB_PORT_ID_ASSIGNED);
 185         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 186         /* NUM_QUEUES is only used to validate indirection table offsets */
 187         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 188 
 189         efx_mcdi_execute(enp, &req);
 190 
 191         if (req.emr_rc != 0) {
 192                 rc = req.emr_rc;
 193                 goto fail3;
 194         }
 195 
 196         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 197                 rc = EMSGSIZE;
 198                 goto fail4;
 199         }
 200 
 201         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 202         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 203                 rc = ENOENT;
 204                 goto fail5;
 205         }
 206 
 207         *rss_contextp = rss_context;
 208 
 209         return (0);
 210 
 211 fail5:
 212         EFSYS_PROBE(fail5);
 213 fail4:
 214         EFSYS_PROBE(fail4);
 215 fail3:
 216         EFSYS_PROBE(fail3);
 217 fail2:
 218         EFSYS_PROBE(fail2);
 219 fail1:
 220         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 221 
 222         return (rc);
 223 }
 224 #endif /* EFSYS_OPT_RX_SCALE */
 225 
 226 #if EFSYS_OPT_RX_SCALE
 227 static                  efx_rc_t
 228 efx_mcdi_rss_context_free(
 229         __in            efx_nic_t *enp,
 230         __in            uint32_t rss_context)
 231 {
 232         efx_mcdi_req_t req;
 233         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 234                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
 235         efx_rc_t rc;
 236 
 237         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 238                 rc = EINVAL;
 239                 goto fail1;
 240         }
 241 
 242         (void) memset(payload, 0, sizeof (payload));
 243         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 244         req.emr_in_buf = payload;
 245         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 246         req.emr_out_buf = payload;
 247         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 248 
 249         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 250 
 251         efx_mcdi_execute(enp, &req);
 252 
 253         if (req.emr_rc != 0) {
 254                 rc = req.emr_rc;
 255                 goto fail2;
 256         }
 257 
 258         return (0);
 259 
 260 fail2:
 261         EFSYS_PROBE(fail2);
 262 fail1:
 263         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 264 
 265         return (rc);
 266 }
 267 #endif /* EFSYS_OPT_RX_SCALE */
 268 
 269 #if EFSYS_OPT_RX_SCALE
 270 static                  efx_rc_t
 271 efx_mcdi_rss_context_set_flags(
 272         __in            efx_nic_t *enp,
 273         __in            uint32_t rss_context,
 274         __in            efx_rx_hash_type_t type)
 275 {
 276         efx_mcdi_req_t req;
 277         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 278                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
 279         efx_rc_t rc;
 280 
 281         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 282                 rc = EINVAL;
 283                 goto fail1;
 284         }
 285 
 286         (void) memset(payload, 0, sizeof (payload));
 287         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 288         req.emr_in_buf = payload;
 289         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 290         req.emr_out_buf = payload;
 291         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 292 
 293         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 294             rss_context);
 295 
 296         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 297             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 298             (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0,
 299             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 300             (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0,
 301             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 302             (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0,
 303             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 304             (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0);
 305 
 306         efx_mcdi_execute(enp, &req);
 307 
 308         if (req.emr_rc != 0) {
 309                 rc = req.emr_rc;
 310                 goto fail2;
 311         }
 312 
 313         return (0);
 314 
 315 fail2:
 316         EFSYS_PROBE(fail2);
 317 fail1:
 318         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 319 
 320         return (rc);
 321 }
 322 #endif /* EFSYS_OPT_RX_SCALE */
 323 
 324 #if EFSYS_OPT_RX_SCALE
 325 static                  efx_rc_t
 326 efx_mcdi_rss_context_set_key(
 327         __in            efx_nic_t *enp,
 328         __in            uint32_t rss_context,
 329         __in_ecount(n)  uint8_t *key,
 330         __in            size_t n)
 331 {
 332         efx_mcdi_req_t req;
 333         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 334                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
 335         efx_rc_t rc;
 336 
 337         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 338                 rc = EINVAL;
 339                 goto fail1;
 340         }
 341 
 342         (void) memset(payload, 0, sizeof (payload));
 343         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 344         req.emr_in_buf = payload;
 345         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 346         req.emr_out_buf = payload;
 347         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 348 
 349         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 350             rss_context);
 351 
 352         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 353         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 354                 rc = EINVAL;
 355                 goto fail2;
 356         }
 357 
 358         (void) memcpy(MCDI_IN2(req, uint8_t,
 359             RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), key, n);
 360 
 361         efx_mcdi_execute(enp, &req);
 362 
 363         if (req.emr_rc != 0) {
 364                 rc = req.emr_rc;
 365                 goto fail3;
 366         }
 367 
 368         return (0);
 369 
 370 fail3:
 371         EFSYS_PROBE(fail3);
 372 fail2:
 373         EFSYS_PROBE(fail2);
 374 fail1:
 375         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 376 
 377         return (rc);
 378 }
 379 #endif /* EFSYS_OPT_RX_SCALE */
 380 
 381 #if EFSYS_OPT_RX_SCALE
 382 static                  efx_rc_t
 383 efx_mcdi_rss_context_set_table(
 384         __in            efx_nic_t *enp,
 385         __in            uint32_t rss_context,
 386         __in_ecount(n)  unsigned int *table,
 387         __in            size_t n)
 388 {
 389         efx_mcdi_req_t req;
 390         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 391                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
 392         uint8_t *req_table;
 393         int i, rc;
 394 
 395         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 396                 rc = EINVAL;
 397                 goto fail1;
 398         }
 399 
 400         (void) memset(payload, 0, sizeof (payload));
 401         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 402         req.emr_in_buf = payload;
 403         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 404         req.emr_out_buf = payload;
 405         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 406 
 407         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 408             rss_context);
 409 
 410         req_table =
 411             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 412 
 413         for (i = 0;
 414             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 415             i++) {
 416                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 417         }
 418 
 419         efx_mcdi_execute(enp, &req);
 420 
 421         if (req.emr_rc != 0) {
 422                 rc = req.emr_rc;
 423                 goto fail2;
 424         }
 425 
 426         return (0);
 427 
 428 fail2:
 429         EFSYS_PROBE(fail2);
 430 fail1:
 431         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 432 
 433         return (rc);
 434 }
 435 #endif /* EFSYS_OPT_RX_SCALE */
 436 
 437 
 438         __checkReturn   efx_rc_t
 439 ef10_rx_init(
 440         __in            efx_nic_t *enp)
 441 {
 442 #if EFSYS_OPT_RX_SCALE
 443 
 444         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 445                 &enp->en_rss_context) == 0) {
 446                 /*
 447                  * Allocated an exclusive RSS context, which allows both the
 448                  * indirection table and key to be modified.
 449                  */
 450                 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 451                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 452         } else {
 453                 /*
 454                  * Failed to allocate an exclusive RSS context. Continue
 455                  * operation without support for RSS. The pseudo-header in
 456                  * received packets will not contain a Toeplitz hash value.
 457                  */
 458                 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
 459                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 460         }
 461 
 462 #endif /* EFSYS_OPT_RX_SCALE */
 463 
 464         return (0);
 465 }
 466 
 467 #if EFSYS_OPT_RX_SCATTER
 468         __checkReturn   efx_rc_t
 469 ef10_rx_scatter_enable(
 470         __in            efx_nic_t *enp,
 471         __in            unsigned int buf_size)
 472 {
 473         _NOTE(ARGUNUSED(enp, buf_size))
 474         return (0);
 475 }
 476 #endif  /* EFSYS_OPT_RX_SCATTER */
 477 
 478 #if EFSYS_OPT_RX_SCALE
 479         __checkReturn   efx_rc_t
 480 ef10_rx_scale_mode_set(
 481         __in            efx_nic_t *enp,
 482         __in            efx_rx_hash_alg_t alg,
 483         __in            efx_rx_hash_type_t type,
 484         __in            boolean_t insert)
 485 {
 486         efx_rc_t rc;
 487 
 488         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
 489         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 490 
 491         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
 492                 rc = EINVAL;
 493                 goto fail1;
 494         }
 495 
 496         if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
 497                 rc = ENOTSUP;
 498                 goto fail2;
 499         }
 500 
 501         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 502                     enp->en_rss_context, type)) != 0)
 503                 goto fail3;
 504 
 505         return (0);
 506 
 507 fail3:
 508         EFSYS_PROBE(fail3);
 509 fail2:
 510         EFSYS_PROBE(fail2);
 511 fail1:
 512         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 513 
 514         return (rc);
 515 }
 516 #endif /* EFSYS_OPT_RX_SCALE */
 517 
 518 #if EFSYS_OPT_RX_SCALE
 519         __checkReturn   efx_rc_t
 520 ef10_rx_scale_key_set(
 521         __in            efx_nic_t *enp,
 522         __in_ecount(n)  uint8_t *key,
 523         __in            size_t n)
 524 {
 525         efx_rc_t rc;
 526 
 527         if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
 528                 rc = ENOTSUP;
 529                 goto fail1;
 530         }
 531 
 532         if ((rc = efx_mcdi_rss_context_set_key(enp,
 533             enp->en_rss_context, key, n)) != 0)
 534                 goto fail2;
 535 
 536         return (0);
 537 
 538 fail2:
 539         EFSYS_PROBE(fail2);
 540 fail1:
 541         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 542 
 543         return (rc);
 544 }
 545 #endif /* EFSYS_OPT_RX_SCALE */
 546 
 547 #if EFSYS_OPT_RX_SCALE
 548         __checkReturn   efx_rc_t
 549 ef10_rx_scale_tbl_set(
 550         __in            efx_nic_t *enp,
 551         __in_ecount(n)  unsigned int *table,
 552         __in            size_t n)
 553 {
 554         efx_rc_t rc;
 555 
 556         if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
 557                 rc = ENOTSUP;
 558                 goto fail1;
 559         }
 560 
 561         if ((rc = efx_mcdi_rss_context_set_table(enp,
 562             enp->en_rss_context, table, n)) != 0)
 563                 goto fail2;
 564 
 565         return (0);
 566 
 567 fail2:
 568         EFSYS_PROBE(fail2);
 569 fail1:
 570         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 571 
 572         return (rc);
 573 }
 574 #endif /* EFSYS_OPT_RX_SCALE */
 575 
 576 
 577 /*
 578  * EF10 RX pseudo-header
 579  * ---------------------
 580  *
 581  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 582  *
 583  *  +00: Toeplitz hash value.
 584  *       (32bit little-endian)
 585  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 586  *       (16bit big-endian)
 587  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 588  *       (16bit big-endian)
 589  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 590  *       (16bit little-endian)
 591  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 592  *       (32bit little-endian)
 593  *
 594  * See "The RX Pseudo-header" in SF-109306-TC.
 595  */
 596 
 597         __checkReturn   efx_rc_t
 598 ef10_rx_prefix_pktlen(
 599         __in            efx_nic_t *enp,
 600         __in            uint8_t *buffer,
 601         __out           uint16_t *lengthp)
 602 {
 603         _NOTE(ARGUNUSED(enp))
 604         /*
 605          * The RX pseudo-header contains the packet length, excluding the
 606          * pseudo-header. If the hardware receive datapath was operating in
 607          * cut-through mode then the length in the RX pseudo-header will be
 608          * zero, and the packet length must be obtained from the DMA length
 609          * reported in the RX event.
 610          */
 611         *lengthp = buffer[8] | (buffer[9] << 8);
 612         return (0);
 613 }
 614 
 615 #if EFSYS_OPT_RX_SCALE
 616         __checkReturn   uint32_t
 617 ef10_rx_prefix_hash(
 618         __in            efx_nic_t *enp,
 619         __in            efx_rx_hash_alg_t func,
 620         __in            uint8_t *buffer)
 621 {
 622         _NOTE(ARGUNUSED(enp))
 623         switch (func) {
 624         case EFX_RX_HASHALG_TOEPLITZ:
 625                 return (buffer[0] |
 626                     (buffer[1] << 8) |
 627                     (buffer[2] << 16) |
 628                     (buffer[3] << 24));
 629 
 630         default:
 631                 EFSYS_ASSERT(0);
 632                 return (0);
 633         }
 634 }
 635 #endif /* EFSYS_OPT_RX_SCALE */
 636 
 637                         void
 638 ef10_rx_qpost(
 639         __in            efx_rxq_t *erp,
 640         __in_ecount(n)  efsys_dma_addr_t *addrp,
 641         __in            size_t size,
 642         __in            unsigned int n,
 643         __in            unsigned int completed,
 644         __in            unsigned int added)
 645 {
 646         efx_qword_t qword;
 647         unsigned int i;
 648         unsigned int offset;
 649         unsigned int id;
 650 
 651         /* The client driver must not overfill the queue */
 652         EFSYS_ASSERT3U(added - completed + n, <=,
 653             EFX_RXQ_LIMIT(erp->er_mask + 1));
 654 
 655         id = added & (erp->er_mask);
 656         for (i = 0; i < n; i++) {
 657                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 658                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 659                     size_t, size);
 660 
 661                 EFX_POPULATE_QWORD_3(qword,
 662                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 663                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 664                     (uint32_t)(addrp[i] & 0xffffffff),
 665                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 666                     (uint32_t)(addrp[i] >> 32));
 667 
 668                 offset = id * sizeof (efx_qword_t);
 669                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 670 
 671                 id = (id + 1) & (erp->er_mask);
 672         }
 673 }
 674 
 675                         void
 676 ef10_rx_qpush(
 677         __in    efx_rxq_t *erp,
 678         __in    unsigned int added,
 679         __inout unsigned int *pushedp)
 680 {
 681         efx_nic_t *enp = erp->er_enp;
 682         unsigned int pushed = *pushedp;
 683         uint32_t wptr;
 684         efx_dword_t dword;
 685 
 686         /* Hardware has alignment restriction for WPTR */
 687         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 688         if (pushed == wptr)
 689                 return;
 690 
 691         *pushedp = wptr;
 692 
 693         /* Push the populated descriptors out */
 694         wptr &= erp->er_mask;
 695 
 696         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 697 
 698         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 699         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 700             wptr, pushed & erp->er_mask);
 701         EFSYS_PIO_WRITE_BARRIER();
 702         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 703                             erp->er_index, &dword, B_FALSE);
 704 }
 705 
 706         __checkReturn   efx_rc_t
 707 ef10_rx_qflush(
 708         __in    efx_rxq_t *erp)
 709 {
 710         efx_nic_t *enp = erp->er_enp;
 711         efx_rc_t rc;
 712 
 713         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 714                 goto fail1;
 715 
 716         return (0);
 717 
 718 fail1:
 719         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 720 
 721         return (rc);
 722 }
 723 
 724                 void
 725 ef10_rx_qenable(
 726         __in    efx_rxq_t *erp)
 727 {
 728         /* FIXME */
 729         _NOTE(ARGUNUSED(erp))
 730         /* FIXME */
 731 }
 732 
 733         __checkReturn   efx_rc_t
 734 ef10_rx_qcreate(
 735         __in            efx_nic_t *enp,
 736         __in            unsigned int index,
 737         __in            unsigned int label,
 738         __in            efx_rxq_type_t type,
 739         __in            efsys_mem_t *esmp,
 740         __in            size_t n,
 741         __in            uint32_t id,
 742         __in            efx_evq_t *eep,
 743         __in            efx_rxq_t *erp)
 744 {
 745         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 746         efx_rc_t rc;
 747         boolean_t disable_scatter;
 748 
 749         _NOTE(ARGUNUSED(erp, id))
 750 
 751         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 752         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 753         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 754 
 755         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 756         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 757 
 758         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
 759                 rc = EINVAL;
 760                 goto fail1;
 761         }
 762         if (index >= encp->enc_rxq_limit) {
 763                 rc = EINVAL;
 764                 goto fail2;
 765         }
 766 
 767         /* Scatter can only be disabled if the firmware supports doing so */
 768         if ((type != EFX_RXQ_TYPE_SCATTER) &&
 769             enp->en_nic_cfg.enc_rx_disable_scatter_supported) {
 770                 disable_scatter = B_TRUE;
 771         } else {
 772                 disable_scatter = B_FALSE;
 773         }
 774 
 775         if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
 776             esmp, disable_scatter)) != 0)
 777                 goto fail3;
 778 
 779         erp->er_eep = eep;
 780         erp->er_label = label;
 781 
 782         ef10_ev_rxlabel_init(eep, erp, label);
 783 
 784         return (0);
 785 
 786 fail3:
 787         EFSYS_PROBE(fail3);
 788 fail2:
 789         EFSYS_PROBE(fail2);
 790 fail1:
 791         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 792 
 793         return (rc);
 794 }
 795 
 796                 void
 797 ef10_rx_qdestroy(
 798         __in    efx_rxq_t *erp)
 799 {
 800         efx_nic_t *enp = erp->er_enp;
 801         efx_evq_t *eep = erp->er_eep;
 802         unsigned int label = erp->er_label;
 803 
 804         ef10_ev_rxlabel_fini(eep, label);
 805 
 806         EFSYS_ASSERT(enp->en_rx_qcount != 0);
 807         --enp->en_rx_qcount;
 808 
 809         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 810 }
 811 
 812                 void
 813 ef10_rx_fini(
 814         __in    efx_nic_t *enp)
 815 {
 816 #if EFSYS_OPT_RX_SCALE
 817         if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
 818                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
 819         }
 820         enp->en_rss_context = 0;
 821         enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
 822 #else
 823         _NOTE(ARGUNUSED(enp))
 824 #endif /* EFSYS_OPT_RX_SCALE */
 825 }
 826 
 827 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */