1 /* 2 * Copyright (c) 2008-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/sysmacros.h> 33 #include <sys/ddi.h> 34 #include <sys/sunddi.h> 35 #include <sys/stream.h> 36 #include <sys/strsun.h> 37 #include <sys/strsubr.h> 38 #include <sys/cpu.h> 39 #include <sys/pghw.h> 40 41 #include "sfxge.h" 42 43 #include "efx.h" 44 45 46 /* Timeout to wait for DRIVER_EV_START event at EVQ startup */ 47 #define SFXGE_EV_QSTART_TIMEOUT_USEC (2000000) 48 49 50 /* Event queue DMA attributes */ 51 static ddi_device_acc_attr_t sfxge_evq_devacc = { 52 53 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */ 54 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */ 55 DDI_STRICTORDER_ACC /* devacc_attr_dataorder */ 56 }; 57 58 static ddi_dma_attr_t sfxge_evq_dma_attr = { 59 DMA_ATTR_V0, /* dma_attr_version */ 60 0, /* dma_attr_addr_lo */ 61 0xffffffffffffffffull, /* dma_attr_addr_hi */ 62 0xffffffffffffffffull, /* dma_attr_count_max */ 63 EFX_BUF_SIZE, /* dma_attr_align */ 64 0xffffffff, /* dma_attr_burstsizes */ 65 1, /* dma_attr_minxfer */ 66 0xffffffffffffffffull, /* dma_attr_maxxfer */ 67 0xffffffffffffffffull, /* dma_attr_seg */ 68 1, /* dma_attr_sgllen */ 69 1, /* dma_attr_granular */ 70 0 /* dma_attr_flags */ 71 }; 72 73 static int 74 _sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size) 75 { 76 efsys_mem_t *esmp = &(sep->se_mem); 77 sfxge_dma_buffer_attr_t dma_attr; 78 int rc; 79 80 /* Compile-time structure layout checks */ 81 EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <= 82 sizeof (sep->__se_u1.__se_pad)); 83 EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <= 84 sizeof (sep->__se_u2.__se_pad)); 85 EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <= 86 sizeof (sep->__se_u3.__se_pad)); 87 88 bzero(sep, sizeof (sfxge_evq_t)); 89 90 sep->se_sp = sp; 91 92 dma_attr.sdba_dip = sp->s_dip; 93 dma_attr.sdba_dattrp = &sfxge_evq_dma_attr; 94 dma_attr.sdba_callback = (kmflags == KM_SLEEP) ? 95 DDI_DMA_SLEEP : DDI_DMA_DONTWAIT; 96 dma_attr.sdba_length = EFX_EVQ_SIZE(evq_size); 97 dma_attr.sdba_memflags = DDI_DMA_CONSISTENT; 98 dma_attr.sdba_devaccp = &sfxge_evq_devacc; 99 dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT; 100 dma_attr.sdba_maxcookies = 1; 101 dma_attr.sdba_zeroinit = B_FALSE; 102 103 if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0) 104 goto fail1; 105 106 /* Allocate some buffer table entries */ 107 if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size), 108 &(sep->se_id))) != 0) 109 goto fail2; 110 111 sep->se_stpp = &(sep->se_stp); 112 113 return (0); 114 115 fail2: 116 DTRACE_PROBE(fail2); 117 118 /* Tear down DMA setup */ 119 esmp->esm_addr = 0; 120 sfxge_dma_buffer_destroy(esmp); 121 122 fail1: 123 DTRACE_PROBE1(fail1, int, rc); 124 125 sep->se_sp = NULL; 126 127 SFXGE_OBJ_CHECK(sep, sfxge_evq_t); 128 129 return (-1); 130 } 131 132 static int 133 sfxge_ev_q0ctor(void *buf, void *arg, int kmflags) 134 { 135 sfxge_evq_t *sep = buf; 136 sfxge_t *sp = arg; 137 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size)); 138 } 139 140 static int 141 sfxge_ev_qXctor(void *buf, void *arg, int kmflags) 142 { 143 sfxge_evq_t *sep = buf; 144 sfxge_t *sp = arg; 145 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size)); 146 } 147 static void 148 _sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size) 149 { 150 efsys_mem_t *esmp = &(sep->se_mem); 151 ASSERT3P(sep->se_sp, ==, sp); 152 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp)); 153 sep->se_stpp = NULL; 154 155 /* Free the buffer table entries */ 156 sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); 157 sep->se_id = 0; 158 159 /* Tear down DMA setup */ 160 sfxge_dma_buffer_destroy(esmp); 161 162 sep->se_sp = NULL; 163 164 SFXGE_OBJ_CHECK(sep, sfxge_evq_t); 165 } 166 167 static void 168 sfxge_ev_q0dtor(void *buf, void *arg) 169 { 170 sfxge_evq_t *sep = buf; 171 sfxge_t *sp = arg; 172 _sfxge_ev_qdtor(sp, sep, sp->s_evq0_size); 173 } 174 175 static void 176 sfxge_ev_qXdtor(void *buf, void *arg) 177 { 178 sfxge_evq_t *sep = buf; 179 sfxge_t *sp = arg; 180 _sfxge_ev_qdtor(sp, sep, sp->s_evqX_size); 181 } 182 183 static boolean_t 184 sfxge_ev_initialized(void *arg) 185 { 186 sfxge_evq_t *sep = arg; 187 188 ASSERT(mutex_owned(&(sep->se_lock))); 189 190 /* Init done events may be duplicated on 7xxx (see SFCbug31631) */ 191 if (sep->se_state == SFXGE_EVQ_STARTED) 192 goto done; 193 194 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING); 195 sep->se_state = SFXGE_EVQ_STARTED; 196 197 cv_broadcast(&(sep->se_init_kv)); 198 199 done: 200 return (B_FALSE); 201 } 202 203 static void 204 sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop) 205 { 206 sfxge_t *sp = sep->se_sp; 207 unsigned int index = sep->se_index; 208 sfxge_rxq_t *srp = sp->s_srp[index]; 209 sfxge_txq_t *stp; 210 211 if ((stp = sep->se_stp) != NULL) { 212 sep->se_stp = NULL; 213 sep->se_stpp = &(sep->se_stp); 214 215 do { 216 sfxge_txq_t *next; 217 218 next = stp->st_next; 219 stp->st_next = NULL; 220 221 ASSERT3U(stp->st_evq, ==, index); 222 223 if (stp->st_pending != stp->st_completed) 224 sfxge_tx_qcomplete(stp); 225 226 stp = next; 227 } while (stp != NULL); 228 } 229 230 if (srp != NULL) { 231 if (srp->sr_pending != srp->sr_completed) 232 sfxge_rx_qcomplete(srp, eop); 233 } 234 } 235 236 static boolean_t 237 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 238 uint16_t flags) 239 { 240 sfxge_evq_t *sep = arg; 241 sfxge_t *sp = sep->se_sp; 242 sfxge_rxq_t *srp; 243 sfxge_rx_packet_t *srpp; 244 unsigned int prefetch; 245 unsigned int stop; 246 unsigned int delta; 247 248 ASSERT(mutex_owned(&(sep->se_lock))); 249 250 if (sep->se_exception) 251 goto done; 252 253 srp = sp->s_srp[label]; 254 if (srp == NULL) 255 goto done; 256 257 ASSERT3U(sep->se_index, ==, srp->sr_index); 258 ASSERT3U(id, <, sp->s_rxq_size); 259 260 /* 261 * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset 262 * So the return missing srp->sr_pending increase is safe 263 */ 264 if (srp->sr_state != SFXGE_RXQ_STARTED) 265 goto done; 266 267 stop = (id + 1) & (sp->s_rxq_size - 1); 268 id = srp->sr_pending & (sp->s_rxq_size - 1); 269 270 delta = (stop >= id) ? (stop - id) : (sp->s_rxq_size - id + stop); 271 srp->sr_pending += delta; 272 273 if (delta != 1) { 274 if ((!efx_nic_cfg_get(sp->s_enp)->enc_rx_batching_enabled) || 275 (delta == 0) || 276 (delta > efx_nic_cfg_get(sp->s_enp)->enc_rx_batch_max)) { 277 /* 278 * FIXME: This does not take into account scatter 279 * aborts. See Bug40811 280 */ 281 sep->se_exception = B_TRUE; 282 283 DTRACE_PROBE(restart_ev_rx_id); 284 /* sfxge_evq_t->se_lock held */ 285 (void) sfxge_restart_dispatch(sp, DDI_SLEEP, 286 SFXGE_HW_ERR, "Out of order RX event", delta); 287 288 goto done; 289 } 290 } 291 292 prefetch = (id + 4) & (sp->s_rxq_size - 1); 293 if ((srpp = srp->sr_srpp[prefetch]) != NULL) 294 prefetch_read_many(srpp); 295 296 srpp = srp->sr_srpp[id]; 297 ASSERT(srpp != NULL); 298 prefetch_read_many(srpp->srp_mp); 299 300 for (; id != stop; id = (id + 1) & (sp->s_rxq_size - 1)) { 301 srpp = srp->sr_srpp[id]; 302 ASSERT(srpp != NULL); 303 304 ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD); 305 srpp->srp_flags = flags; 306 307 ASSERT3U(size, <, (1 << 16)); 308 srpp->srp_size = (uint16_t)size; 309 } 310 311 sep->se_rx++; 312 313 DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index, 314 unsigned int, srp->sr_added - srp->sr_pending); 315 316 if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH) 317 sfxge_ev_qcomplete(sep, B_FALSE); 318 319 done: 320 /* returning B_TRUE makes efx_ev_qpoll() stop processing events */ 321 return (sep->se_rx >= sep->se_ev_batch); 322 } 323 324 static boolean_t 325 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 326 { 327 sfxge_evq_t *sep = arg; 328 sfxge_t *sp = sep->se_sp; 329 330 _NOTE(ARGUNUSED(code)) 331 _NOTE(ARGUNUSED(data)) 332 333 ASSERT(mutex_owned(&(sep->se_lock))); 334 sep->se_exception = B_TRUE; 335 336 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 337 338 DTRACE_PROBE(restart_ev_exception); 339 340 /* sfxge_evq_t->se_lock held */ 341 (void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR, 342 "Unknown EV", code); 343 } 344 345 return (B_FALSE); 346 } 347 348 static boolean_t 349 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 350 { 351 sfxge_evq_t *sep_targetq, *sep = arg; 352 sfxge_t *sp = sep->se_sp; 353 sfxge_rxq_t *srp; 354 unsigned int index; 355 unsigned int label; 356 uint16_t magic; 357 358 ASSERT(mutex_owned(&(sep->se_lock))); 359 360 /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */ 361 srp = sp->s_srp[rxq_index]; 362 if (srp == NULL) 363 goto done; 364 365 /* Process right now if it is the correct event queue */ 366 index = srp->sr_index; 367 if (index == sep->se_index) { 368 sfxge_rx_qflush_done(srp); 369 goto done; 370 } 371 372 /* Resend a software event on the correct queue */ 373 sep_targetq = sp->s_sep[index]; 374 375 if (sep_targetq->se_state != SFXGE_EVQ_STARTED) 376 goto done; /* TBD: state test not under the lock */ 377 378 label = rxq_index; 379 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label); 380 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 381 382 efx_ev_qpost(sep_targetq->se_eep, magic); 383 384 done: 385 return (B_FALSE); 386 } 387 388 static boolean_t 389 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 390 { 391 sfxge_evq_t *sep_targetq, *sep = arg; 392 sfxge_t *sp = sep->se_sp; 393 sfxge_rxq_t *srp; 394 unsigned int index; 395 unsigned int label; 396 uint16_t magic; 397 398 ASSERT(mutex_owned(&(sep->se_lock))); 399 400 /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */ 401 srp = sp->s_srp[rxq_index]; 402 if (srp == NULL) 403 goto done; 404 405 /* Process right now if it is the correct event queue */ 406 index = srp->sr_index; 407 if (index == sep->se_index) { 408 sfxge_rx_qflush_failed(srp); 409 goto done; 410 } 411 412 /* Resend a software event on the correct queue */ 413 sep_targetq = sp->s_sep[index]; 414 415 label = rxq_index; 416 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label); 417 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 418 419 if (sep_targetq->se_state != SFXGE_EVQ_STARTED) 420 goto done; /* TBD: state test not under the lock */ 421 422 efx_ev_qpost(sep_targetq->se_eep, magic); 423 424 done: 425 return (B_FALSE); 426 } 427 428 static boolean_t 429 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 430 { 431 sfxge_evq_t *sep = arg; 432 sfxge_txq_t *stp; 433 unsigned int stop; 434 unsigned int delta; 435 436 ASSERT(mutex_owned(&(sep->se_lock))); 437 438 stp = sep->se_label_stp[label]; 439 if (stp == NULL) 440 goto done; 441 442 if (stp->st_state != SFXGE_TXQ_STARTED) 443 goto done; 444 445 ASSERT3U(sep->se_index, ==, stp->st_evq); 446 447 stop = (id + 1) & (SFXGE_TX_NDESCS - 1); 448 id = stp->st_pending & (SFXGE_TX_NDESCS - 1); 449 450 delta = (stop >= id) ? (stop - id) : (SFXGE_TX_NDESCS - id + stop); 451 stp->st_pending += delta; 452 453 sep->se_tx++; 454 455 if (stp->st_next == NULL && 456 sep->se_stpp != &(stp->st_next)) { 457 *(sep->se_stpp) = stp; 458 sep->se_stpp = &(stp->st_next); 459 } 460 461 DTRACE_PROBE2(qlevel, unsigned int, stp->st_index, 462 unsigned int, stp->st_added - stp->st_pending); 463 464 if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH) 465 sfxge_tx_qcomplete(stp); 466 467 done: 468 /* returning B_TRUE makes efx_ev_qpoll() stop processing events */ 469 return (sep->se_tx >= sep->se_ev_batch); 470 } 471 472 static boolean_t 473 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 474 { 475 sfxge_evq_t *sep = arg; 476 sfxge_t *sp = sep->se_sp; 477 sfxge_txq_t *stp; 478 unsigned int evq; 479 unsigned int label; 480 uint16_t magic; 481 482 ASSERT(mutex_owned(&(sep->se_lock))); 483 484 /* Ensure TXQ exists, as events may arrive after TXQ was destroyed */ 485 stp = sp->s_stp[txq_index]; 486 if (stp == NULL) 487 goto done; 488 489 /* Process right now if it is the correct event queue */ 490 evq = stp->st_evq; 491 if (evq == sep->se_index) { 492 sfxge_tx_qflush_done(stp); 493 goto done; 494 } 495 496 /* Resend a software event on the correct queue */ 497 sep = sp->s_sep[evq]; 498 499 label = stp->st_label; 500 501 ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label); 502 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 503 504 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); 505 efx_ev_qpost(sep->se_eep, magic); 506 507 done: 508 return (B_FALSE); 509 } 510 511 static boolean_t 512 sfxge_ev_software(void *arg, uint16_t magic) 513 { 514 sfxge_evq_t *sep = arg; 515 sfxge_t *sp = sep->se_sp; 516 dev_info_t *dip = sp->s_dip; 517 unsigned int label; 518 519 ASSERT(mutex_owned(&(sep->se_lock))); 520 521 EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH == 522 FSF_AZ_RX_EV_Q_LABEL_WIDTH); 523 EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH == 524 FSF_AZ_TX_EV_Q_LABEL_WIDTH); 525 526 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 527 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 528 529 switch (magic) { 530 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 531 sfxge_rxq_t *srp = sp->s_srp[label]; 532 533 if (srp != NULL) { 534 ASSERT3U(sep->se_index, ==, srp->sr_index); 535 536 sfxge_rx_qflush_done(srp); 537 } 538 break; 539 } 540 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 541 sfxge_rxq_t *srp = sp->s_srp[label]; 542 543 if (srp != NULL) { 544 ASSERT3U(sep->se_index, ==, srp->sr_index); 545 546 sfxge_rx_qflush_failed(srp); 547 } 548 break; 549 } 550 case SFXGE_MAGIC_RX_QFPP_TRIM: { 551 sfxge_rxq_t *srp = sp->s_srp[label]; 552 553 if (srp != NULL) { 554 ASSERT3U(sep->se_index, ==, srp->sr_index); 555 556 sfxge_rx_qfpp_trim(srp); 557 } 558 break; 559 } 560 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 561 sfxge_txq_t *stp = sep->se_label_stp[label]; 562 563 if (stp != NULL) { 564 ASSERT3U(sep->se_index, ==, stp->st_evq); 565 566 sfxge_tx_qflush_done(stp); 567 } 568 break; 569 } 570 default: 571 dev_err(dip, CE_NOTE, 572 SFXGE_CMN_ERR "unknown software event 0x%x", magic); 573 break; 574 } 575 576 return (B_FALSE); 577 } 578 579 static boolean_t 580 sfxge_ev_sram(void *arg, uint32_t code) 581 { 582 _NOTE(ARGUNUSED(arg)) 583 584 switch (code) { 585 case EFX_SRAM_UPDATE: 586 DTRACE_PROBE(sram_update); 587 break; 588 589 case EFX_SRAM_CLEAR: 590 DTRACE_PROBE(sram_clear); 591 break; 592 593 case EFX_SRAM_ILLEGAL_CLEAR: 594 DTRACE_PROBE(sram_illegal_clear); 595 break; 596 597 default: 598 ASSERT(B_FALSE); 599 break; 600 } 601 602 return (B_FALSE); 603 } 604 605 static boolean_t 606 sfxge_ev_timer(void *arg, uint32_t index) 607 { 608 _NOTE(ARGUNUSED(arg, index)) 609 610 return (B_FALSE); 611 } 612 613 static boolean_t 614 sfxge_ev_wake_up(void *arg, uint32_t index) 615 { 616 _NOTE(ARGUNUSED(arg, index)) 617 618 return (B_FALSE); 619 } 620 621 static boolean_t 622 sfxge_ev_monitor(void *arg, efx_mon_stat_t id, efx_mon_stat_value_t value) 623 { 624 _NOTE(ARGUNUSED(arg, id, value)) 625 626 return (B_FALSE); 627 } 628 629 static boolean_t 630 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 631 { 632 sfxge_evq_t *sep = arg; 633 sfxge_t *sp = sep->se_sp; 634 635 sfxge_mac_link_update(sp, link_mode); 636 637 return (B_FALSE); 638 } 639 640 static int 641 sfxge_ev_kstat_update(kstat_t *ksp, int rw) 642 { 643 sfxge_evq_t *sep = ksp->ks_private; 644 kstat_named_t *knp; 645 int rc; 646 647 if (rw != KSTAT_READ) { 648 rc = EACCES; 649 goto fail1; 650 } 651 652 ASSERT(mutex_owned(&(sep->se_lock))); 653 654 if (sep->se_state != SFXGE_EVQ_STARTED) 655 goto done; 656 657 efx_ev_qstats_update(sep->se_eep, sep->se_stat); 658 659 knp = ksp->ks_data; 660 knp += EV_NQSTATS; 661 662 knp->value.ui64 = sep->se_cpu_id; 663 664 done: 665 return (0); 666 667 fail1: 668 DTRACE_PROBE1(fail1, int, rc); 669 670 return (rc); 671 } 672 673 static int 674 sfxge_ev_kstat_init(sfxge_evq_t *sep) 675 { 676 sfxge_t *sp = sep->se_sp; 677 unsigned int index = sep->se_index; 678 dev_info_t *dip = sp->s_dip; 679 kstat_t *ksp; 680 kstat_named_t *knp; 681 char name[MAXNAMELEN]; 682 unsigned int id; 683 int rc; 684 685 /* Determine the name */ 686 (void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d", 687 ddi_driver_name(dip), index); 688 689 /* Create the set */ 690 if ((ksp = kstat_create((char *)ddi_driver_name(dip), 691 ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED, 692 EV_NQSTATS + 1, 0)) == NULL) { 693 rc = ENOMEM; 694 goto fail1; 695 } 696 697 sep->se_ksp = ksp; 698 699 ksp->ks_update = sfxge_ev_kstat_update; 700 ksp->ks_private = sep; 701 ksp->ks_lock = &(sep->se_lock); 702 703 /* Initialise the named stats */ 704 sep->se_stat = knp = ksp->ks_data; 705 for (id = 0; id < EV_NQSTATS; id++) { 706 kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id), 707 KSTAT_DATA_UINT64); 708 knp++; 709 } 710 711 kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64); 712 713 kstat_install(ksp); 714 return (0); 715 716 fail1: 717 DTRACE_PROBE1(fail1, int, rc); 718 719 return (rc); 720 } 721 722 static void 723 sfxge_ev_kstat_fini(sfxge_evq_t *sep) 724 { 725 /* Destroy the set */ 726 kstat_delete(sep->se_ksp); 727 sep->se_ksp = NULL; 728 sep->se_stat = NULL; 729 } 730 731 inline unsigned pow2_ge(unsigned int n) { 732 unsigned int order = 0; 733 ASSERT3U(n, >, 0); 734 while ((1ul << order) < n) ++order; 735 return (1ul << (order)); 736 } 737 738 static int 739 sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch) 740 { 741 sfxge_evq_t *sep; 742 int rc; 743 744 ASSERT3U(index, <, SFXGE_RX_SCALE_MAX); 745 746 sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP); 747 if (sep == NULL) { 748 rc = ENOMEM; 749 goto fail1; 750 } 751 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED); 752 753 sep->se_index = index; 754 755 mutex_init(&(sep->se_lock), NULL, 756 MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri)); 757 758 cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL); 759 760 /* Initialize the statistics */ 761 if ((rc = sfxge_ev_kstat_init(sep)) != 0) 762 goto fail2; 763 764 sep->se_state = SFXGE_EVQ_INITIALIZED; 765 sep->se_ev_batch = (uint16_t)ev_batch; 766 sp->s_sep[index] = sep; 767 768 return (0); 769 770 fail2: 771 DTRACE_PROBE(fail2); 772 773 sep->se_index = 0; 774 775 cv_destroy(&(sep->se_init_kv)); 776 mutex_destroy(&(sep->se_lock)); 777 778 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep); 779 780 fail1: 781 DTRACE_PROBE1(fail1, int, rc); 782 783 return (rc); 784 } 785 786 static int 787 sfxge_ev_qstart(sfxge_t *sp, unsigned int index) 788 { 789 sfxge_evq_t *sep = sp->s_sep[index]; 790 sfxge_intr_t *sip = &(sp->s_intr); 791 efx_nic_t *enp = sp->s_enp; 792 efx_ev_callbacks_t *eecp; 793 efsys_mem_t *esmp; 794 clock_t timeout; 795 int rc; 796 uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size; 797 798 mutex_enter(&(sep->se_lock)); 799 esmp = &(sep->se_mem); 800 801 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED); 802 803 /* Set the memory to all ones */ 804 (void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size)); 805 806 /* Program the buffer table */ 807 if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp, 808 EFX_EVQ_NBUFS(evq_size))) != 0) 809 goto fail1; 810 811 /* Set up the event callbacks */ 812 eecp = &(sep->se_eec); 813 eecp->eec_initialized = sfxge_ev_initialized; 814 eecp->eec_rx = sfxge_ev_rx; 815 eecp->eec_tx = sfxge_ev_tx; 816 eecp->eec_exception = sfxge_ev_exception; 817 eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done; 818 eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed; 819 eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done; 820 eecp->eec_software = sfxge_ev_software; 821 eecp->eec_sram = sfxge_ev_sram; 822 eecp->eec_wake_up = sfxge_ev_wake_up; 823 eecp->eec_timer = sfxge_ev_timer; 824 eecp->eec_link_change = sfxge_ev_link_change; 825 eecp->eec_monitor = sfxge_ev_monitor; 826 827 /* Create the event queue */ 828 if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id, 829 &(sep->se_eep))) != 0) 830 goto fail2; 831 832 /* Set the default moderation */ 833 if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0) 834 goto fail3; 835 836 /* Check that interrupts are enabled at the NIC */ 837 if (sip->si_state != SFXGE_INTR_STARTED) { 838 rc = EINVAL; 839 goto fail4; 840 } 841 842 sep->se_state = SFXGE_EVQ_STARTING; 843 844 /* Prime the event queue for interrupts */ 845 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) 846 goto fail5; 847 848 /* Wait for the initialization event */ 849 timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_EV_QSTART_TIMEOUT_USEC); 850 while (sep->se_state != SFXGE_EVQ_STARTED) { 851 if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock), 852 timeout) < 0) { 853 /* Timeout waiting for initialization */ 854 dev_info_t *dip = sp->s_dip; 855 856 DTRACE_PROBE(timeout); 857 dev_err(dip, CE_NOTE, 858 SFXGE_CMN_ERR "evq[%d] qstart timeout", index); 859 860 rc = ETIMEDOUT; 861 goto fail6; 862 } 863 } 864 865 mutex_exit(&(sep->se_lock)); 866 return (0); 867 868 fail6: 869 DTRACE_PROBE(fail6); 870 871 fail5: 872 DTRACE_PROBE(fail5); 873 874 sep->se_state = SFXGE_EVQ_INITIALIZED; 875 876 fail4: 877 DTRACE_PROBE(fail4); 878 879 fail3: 880 DTRACE_PROBE(fail3); 881 882 /* Destroy the event queue */ 883 efx_ev_qdestroy(sep->se_eep); 884 sep->se_eep = NULL; 885 886 fail2: 887 DTRACE_PROBE(fail2); 888 889 /* Zero out the event handlers */ 890 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t)); 891 892 /* Clear entries from the buffer table */ 893 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); 894 895 fail1: 896 DTRACE_PROBE1(fail1, int, rc); 897 898 mutex_exit(&(sep->se_lock)); 899 900 return (rc); 901 } 902 903 int 904 sfxge_ev_qpoll(sfxge_t *sp, unsigned int index) 905 { 906 sfxge_evq_t *sep = sp->s_sep[index]; 907 processorid_t cpu_id; 908 int rc; 909 uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size; 910 911 mutex_enter(&(sep->se_lock)); 912 913 if (sep->se_state != SFXGE_EVQ_STARTING && 914 sep->se_state != SFXGE_EVQ_STARTED) { 915 rc = EINVAL; 916 goto fail1; 917 } 918 919 /* Make sure the CPU information is up to date */ 920 cpu_id = CPU->cpu_id; 921 922 if (cpu_id != sep->se_cpu_id) { 923 sep->se_cpu_id = cpu_id; 924 925 /* sfxge_evq_t->se_lock held */ 926 (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp, 927 DDI_NOSLEEP); 928 } 929 930 /* Synchronize the DMA memory for reading */ 931 (void) ddi_dma_sync(sep->se_mem.esm_dma_handle, 932 0, 933 EFX_EVQ_SIZE(evq_size), 934 DDI_DMA_SYNC_FORKERNEL); 935 936 ASSERT3U(sep->se_rx, ==, 0); 937 ASSERT3U(sep->se_tx, ==, 0); 938 ASSERT3P(sep->se_stp, ==, NULL); 939 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp)); 940 941 /* Poll the queue */ 942 efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec), 943 sep); 944 945 sep->se_rx = 0; 946 sep->se_tx = 0; 947 948 /* Perform any pending completion processing */ 949 sfxge_ev_qcomplete(sep, B_TRUE); 950 951 /* Re-prime the event queue for interrupts */ 952 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) 953 goto fail2; 954 955 mutex_exit(&(sep->se_lock)); 956 957 return (0); 958 959 fail2: 960 DTRACE_PROBE(fail2); 961 fail1: 962 DTRACE_PROBE1(fail1, int, rc); 963 964 mutex_exit(&(sep->se_lock)); 965 966 return (rc); 967 } 968 969 int 970 sfxge_ev_qprime(sfxge_t *sp, unsigned int index) 971 { 972 sfxge_evq_t *sep = sp->s_sep[index]; 973 int rc; 974 975 mutex_enter(&(sep->se_lock)); 976 977 if (sep->se_state != SFXGE_EVQ_STARTING && 978 sep->se_state != SFXGE_EVQ_STARTED) { 979 rc = EINVAL; 980 goto fail1; 981 } 982 983 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) 984 goto fail2; 985 986 mutex_exit(&(sep->se_lock)); 987 988 return (0); 989 990 fail2: 991 DTRACE_PROBE(fail2); 992 fail1: 993 DTRACE_PROBE1(fail1, int, rc); 994 995 mutex_exit(&(sep->se_lock)); 996 997 return (rc); 998 } 999 1000 1001 int 1002 sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us) 1003 { 1004 sfxge_evq_t *sep = sp->s_sep[index]; 1005 efx_evq_t *eep = sep->se_eep; 1006 1007 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); 1008 1009 return (efx_ev_qmoderate(eep, us)); 1010 } 1011 1012 static void 1013 sfxge_ev_qstop(sfxge_t *sp, unsigned int index) 1014 { 1015 sfxge_evq_t *sep = sp->s_sep[index]; 1016 uint16_t evq_size; 1017 1018 mutex_enter(&(sep->se_lock)); 1019 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); 1020 sep->se_state = SFXGE_EVQ_INITIALIZED; 1021 evq_size = index ? sp->s_evqX_size : sp->s_evq0_size; 1022 1023 /* Clear the CPU information */ 1024 sep->se_cpu_id = 0; 1025 1026 /* Clear the event count */ 1027 sep->se_count = 0; 1028 1029 /* Reset the exception flag */ 1030 sep->se_exception = B_FALSE; 1031 1032 /* Destroy the event queue */ 1033 efx_ev_qdestroy(sep->se_eep); 1034 sep->se_eep = NULL; 1035 1036 mutex_exit(&(sep->se_lock)); 1037 1038 /* Zero out the event handlers */ 1039 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t)); 1040 1041 /* Clear entries from the buffer table */ 1042 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); 1043 } 1044 1045 static void 1046 sfxge_ev_qfini(sfxge_t *sp, unsigned int index) 1047 { 1048 sfxge_evq_t *sep = sp->s_sep[index]; 1049 1050 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED); 1051 1052 sp->s_sep[index] = NULL; 1053 sep->se_state = SFXGE_EVQ_UNINITIALIZED; 1054 1055 /* Tear down the statistics */ 1056 sfxge_ev_kstat_fini(sep); 1057 1058 cv_destroy(&(sep->se_init_kv)); 1059 mutex_destroy(&(sep->se_lock)); 1060 1061 sep->se_index = 0; 1062 1063 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep); 1064 } 1065 1066 int 1067 sfxge_ev_txlabel_alloc(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp, 1068 unsigned int *labelp) 1069 { 1070 sfxge_evq_t *sep = sp->s_sep[evq]; 1071 sfxge_txq_t **stpp; 1072 unsigned int label; 1073 int rc; 1074 1075 mutex_enter(&(sep->se_lock)); 1076 1077 if (stp == NULL || labelp == NULL) { 1078 rc = EINVAL; 1079 goto fail1; 1080 } 1081 1082 stpp = NULL; 1083 for (label = 0; label < SFXGE_TX_NLABELS; label++) { 1084 if (sep->se_label_stp[label] == stp) { 1085 rc = EEXIST; 1086 goto fail2; 1087 } 1088 if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) { 1089 stpp = &sep->se_label_stp[label]; 1090 } 1091 } 1092 if (stpp == NULL) { 1093 rc = ENOSPC; 1094 goto fail3; 1095 } 1096 *stpp = stp; 1097 label = stpp - sep->se_label_stp; 1098 1099 ASSERT3U(label, <, SFXGE_TX_NLABELS); 1100 *labelp = label; 1101 1102 mutex_exit(&(sep->se_lock)); 1103 return (0); 1104 1105 fail3: 1106 DTRACE_PROBE(fail3); 1107 fail2: 1108 DTRACE_PROBE(fail2); 1109 fail1: 1110 DTRACE_PROBE1(fail1, int, rc); 1111 1112 mutex_exit(&(sep->se_lock)); 1113 1114 return (rc); 1115 } 1116 1117 1118 int 1119 sfxge_ev_txlabel_free(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp, 1120 unsigned int label) 1121 { 1122 sfxge_evq_t *sep = sp->s_sep[evq]; 1123 int rc; 1124 1125 mutex_enter(&(sep->se_lock)); 1126 1127 if (stp == NULL || label > SFXGE_TX_NLABELS) { 1128 rc = EINVAL; 1129 goto fail1; 1130 } 1131 1132 if (sep->se_label_stp[label] != stp) { 1133 rc = EINVAL; 1134 goto fail2; 1135 } 1136 sep->se_label_stp[label] = NULL; 1137 1138 mutex_exit(&(sep->se_lock)); 1139 1140 return (0); 1141 1142 fail2: 1143 DTRACE_PROBE(fail2); 1144 fail1: 1145 DTRACE_PROBE1(fail1, int, rc); 1146 1147 mutex_exit(&(sep->se_lock)); 1148 1149 return (rc); 1150 } 1151 1152 1153 static kmem_cache_t * 1154 sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname, 1155 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *)) 1156 { 1157 char name[MAXNAMELEN]; 1158 kmem_cache_t *eqc; 1159 1160 (void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache", 1161 ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname); 1162 1163 eqc = kmem_cache_create(name, sizeof (sfxge_evq_t), 1164 SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0); 1165 ASSERT(eqc != NULL); 1166 return (eqc); 1167 } 1168 1169 int 1170 sfxge_ev_init(sfxge_t *sp) 1171 { 1172 sfxge_intr_t *sip = &(sp->s_intr); 1173 unsigned int evq0_size; 1174 unsigned int evqX_size; 1175 unsigned int ev_batch; 1176 int index; 1177 int rc; 1178 1179 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED); 1180 1181 /* 1182 * Must account for RXQ, TXQ(s); MCDI not event completed at present 1183 * Note that common code does not completely fill descriptor queues 1184 */ 1185 evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS; 1186 evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */ 1187 evq0_size += SFXGE_TX_NDESCS; /* no checksums */ 1188 1189 ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS); 1190 ASSERT3U(evq0_size, >, evqX_size); 1191 1192 if (evq0_size > EFX_EVQ_MAXNEVS) { 1193 rc = EINVAL; 1194 goto fail1; 1195 } 1196 1197 sp->s_evq0_size = pow2_ge(evq0_size); 1198 sp->s_evqX_size = pow2_ge(evqX_size); 1199 1200 /* Read driver parameters */ 1201 sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, 1202 DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION); 1203 1204 ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, 1205 DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH); 1206 1207 /* 1208 * It is slightly peverse to have a cache for one item. But it allows 1209 * for simple alignment control without increasing the allocation size 1210 */ 1211 sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor, 1212 sfxge_ev_q0dtor); 1213 sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor, 1214 sfxge_ev_qXdtor); 1215 1216 /* Initialize the event queue(s) */ 1217 for (index = 0; index < sip->si_nalloc; index++) { 1218 if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0) 1219 goto fail2; 1220 } 1221 1222 return (0); 1223 1224 fail2: 1225 DTRACE_PROBE(fail2); 1226 1227 while (--index >= 0) 1228 sfxge_ev_qfini(sp, index); 1229 sp->s_ev_moderation = 0; 1230 1231 fail1: 1232 DTRACE_PROBE1(fail1, int, rc); 1233 1234 kmem_cache_destroy(sp->s_eqXc); 1235 kmem_cache_destroy(sp->s_eq0c); 1236 sp->s_eqXc = NULL; 1237 sp->s_eq0c = NULL; 1238 1239 return (rc); 1240 } 1241 1242 int 1243 sfxge_ev_start(sfxge_t *sp) 1244 { 1245 sfxge_intr_t *sip = &(sp->s_intr); 1246 efx_nic_t *enp = sp->s_enp; 1247 int index; 1248 int rc; 1249 1250 ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED); 1251 1252 /* Initialize the event module */ 1253 if ((rc = efx_ev_init(enp)) != 0) 1254 goto fail1; 1255 1256 /* Start the event queues */ 1257 for (index = 0; index < sip->si_nalloc; index++) { 1258 if ((rc = sfxge_ev_qstart(sp, index)) != 0) 1259 goto fail2; 1260 } 1261 1262 return (0); 1263 1264 fail2: 1265 DTRACE_PROBE(fail2); 1266 1267 /* Stop the event queue(s) */ 1268 while (--index >= 0) 1269 sfxge_ev_qstop(sp, index); 1270 1271 /* Tear down the event module */ 1272 efx_ev_fini(enp); 1273 1274 fail1: 1275 DTRACE_PROBE1(fail1, int, rc); 1276 1277 return (rc); 1278 } 1279 1280 void 1281 sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp) 1282 { 1283 *usp = sp->s_ev_moderation; 1284 } 1285 1286 int 1287 sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us) 1288 { 1289 sfxge_intr_t *sip = &(sp->s_intr); 1290 int index; 1291 int rc; 1292 1293 if (sip->si_state != SFXGE_INTR_STARTED) 1294 return (ENODEV); 1295 1296 for (index = 0; index < sip->si_nalloc; index++) { 1297 if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0) 1298 goto fail1; 1299 } 1300 1301 sp->s_ev_moderation = us; 1302 return (0); 1303 1304 fail1: 1305 DTRACE_PROBE1(fail1, int, rc); 1306 1307 /* The only error path is if the value to set to is invalid. */ 1308 ASSERT3U(index, ==, 0); 1309 1310 return (rc); 1311 } 1312 1313 void 1314 sfxge_ev_stop(sfxge_t *sp) 1315 { 1316 sfxge_intr_t *sip = &(sp->s_intr); 1317 efx_nic_t *enp = sp->s_enp; 1318 int index; 1319 1320 ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED); 1321 1322 /* Stop the event queue(s) */ 1323 index = sip->si_nalloc; 1324 while (--index >= 0) 1325 sfxge_ev_qstop(sp, index); 1326 1327 /* Tear down the event module */ 1328 efx_ev_fini(enp); 1329 } 1330 1331 void 1332 sfxge_ev_fini(sfxge_t *sp) 1333 { 1334 sfxge_intr_t *sip = &(sp->s_intr); 1335 int index; 1336 1337 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED); 1338 1339 sp->s_ev_moderation = 0; 1340 1341 /* Tear down the event queue(s) */ 1342 index = sip->si_nalloc; 1343 while (--index >= 0) 1344 sfxge_ev_qfini(sp, index); 1345 1346 kmem_cache_destroy(sp->s_eqXc); 1347 kmem_cache_destroy(sp->s_eq0c); 1348 sp->s_eqXc = NULL; 1349 sp->s_eq0c = NULL; 1350 }