1 /*
   2  * Copyright (c) 2008-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30 
  31 #include <sys/types.h>
  32 #include <sys/ddi.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/atomic.h>
  35 #include <sys/modctl.h>
  36 #include <sys/conf.h>
  37 #include <sys/ethernet.h>
  38 #include <sys/pci.h>
  39 #include <sys/pcie.h>
  40 
  41 #include "sfxge.h"
  42 
  43 #include "efx.h"
  44 
  45 
  46 /* Interrupt table DMA attributes */
  47 static ddi_device_acc_attr_t sfxge_intr_devacc = {
  48 
  49         DDI_DEVICE_ATTR_V0,     /* devacc_attr_version */
  50         DDI_NEVERSWAP_ACC,      /* devacc_attr_endian_flags */
  51         DDI_STRICTORDER_ACC     /* devacc_attr_dataorder */
  52 };
  53 
  54 static ddi_dma_attr_t sfxge_intr_dma_attr = {
  55         DMA_ATTR_V0,            /* dma_attr_version     */
  56         0,                      /* dma_attr_addr_lo     */
  57         0xffffffffffffffffull,  /* dma_attr_addr_hi     */
  58         0xffffffffffffffffull,  /* dma_attr_count_max   */
  59         EFX_INTR_SIZE,          /* dma_attr_align       */
  60         0xffffffff,             /* dma_attr_burstsizes  */
  61         1,                      /* dma_attr_minxfer     */
  62         0xffffffffffffffffull,  /* dma_attr_maxxfer     */
  63         0xffffffffffffffffull,  /* dma_attr_seg         */
  64         1,                      /* dma_attr_sgllen      */
  65         1,                      /* dma_attr_granular    */
  66         0                       /* dma_attr_flags       */
  67 };
  68 
  69 static unsigned int
  70 sfxge_intr_line(caddr_t arg1, caddr_t arg2)
  71 {
  72         sfxge_t *sp = (void *)arg1;
  73         efx_nic_t *enp = sp->s_enp;
  74         sfxge_intr_t *sip = &(sp->s_intr);
  75         unsigned int index;
  76         boolean_t fatal;
  77         uint32_t qmask;
  78         int rc;
  79 
  80         _NOTE(ARGUNUSED(arg2))
  81 
  82         ASSERT3U(sip->si_type, ==, EFX_INTR_LINE);
  83 
  84         if (sip->si_state != SFXGE_INTR_STARTED &&
  85             sip->si_state != SFXGE_INTR_TESTING) {
  86                 rc = DDI_INTR_UNCLAIMED;
  87                 goto done;
  88         }
  89 
  90         if (sip->si_state == SFXGE_INTR_TESTING) {
  91                 sip->si_mask |= 1;   /* only one interrupt */
  92                 rc = DDI_INTR_CLAIMED;
  93                 goto done;
  94         }
  95 
  96         efx_intr_status_line(enp, &fatal, &qmask);
  97 
  98         if (fatal) {
  99                 sfxge_intr_fatal(sp);
 100 
 101                 rc = DDI_INTR_CLAIMED;
 102                 goto done;
 103         }
 104 
 105         if (qmask != 0) {
 106                 for (index = 0; index < EFX_INTR_NEVQS; index++) {
 107                         if (qmask & (1 << index))
 108                                 (void) sfxge_ev_qpoll(sp, index);
 109                 }
 110 
 111                 sip->si_zero_count = 0;
 112                 sfxge_gld_rx_push(sp);
 113                 rc = DDI_INTR_CLAIMED;
 114                 goto done;
 115         }
 116 
 117         /*
 118          * bug15671/bug17203 workaround. Return CLAIMED for the first ISR=0
 119          * interrupt, and poll all evqs for work. For subsequent ISR=0
 120          * interrupts (the line must be shared in this case), just rearm the
 121          * event queues to ensure we don't miss an interrupt.
 122          */
 123         if (sip->si_zero_count++ == 0) {
 124                 for (index = 0; index < EFX_INTR_NEVQS; index++) {
 125                         if (sp->s_sep[index] != NULL)
 126                                 (void) sfxge_ev_qpoll(sp, index);
 127                 }
 128 
 129                 rc = DDI_INTR_CLAIMED;
 130         } else {
 131                 for (index = 0; index < EFX_INTR_NEVQS; index++) {
 132                         if (sp->s_sep[index] != NULL)
 133                                 (void) sfxge_ev_qprime(sp, index);
 134                 }
 135 
 136                 rc = DDI_INTR_UNCLAIMED;
 137         }
 138 
 139 done:
 140         return (rc);
 141 }
 142 
 143 static unsigned int
 144 sfxge_intr_message(caddr_t arg1, caddr_t arg2)
 145 {
 146         sfxge_t *sp = (void *)arg1;
 147         efx_nic_t *enp = sp->s_enp;
 148         sfxge_intr_t *sip = &(sp->s_intr);
 149         unsigned int index = (unsigned int)(uintptr_t)arg2;
 150         boolean_t fatal;
 151         int rc;
 152 
 153         ASSERT3U(sip->si_type, ==, EFX_INTR_MESSAGE);
 154 
 155         if (sip->si_state != SFXGE_INTR_STARTED &&
 156             sip->si_state != SFXGE_INTR_TESTING) {
 157                 rc = DDI_INTR_UNCLAIMED;
 158                 goto done;
 159         }
 160 
 161         if (sip->si_state == SFXGE_INTR_TESTING) {
 162                 uint64_t mask;
 163 
 164                 do {
 165                         mask = sip->si_mask;
 166                 } while (atomic_cas_64(&(sip->si_mask), mask,
 167                     mask | (1 << index)) != mask);
 168 
 169                 rc = DDI_INTR_CLAIMED;
 170                 goto done;
 171         }
 172 
 173         efx_intr_status_message(enp, index, &fatal);
 174 
 175         if (fatal) {
 176                 sfxge_intr_fatal(sp);
 177 
 178                 rc = DDI_INTR_CLAIMED;
 179                 goto done;
 180         }
 181 
 182         (void) sfxge_ev_qpoll(sp, index);
 183 
 184         sfxge_gld_rx_push(sp);
 185         rc = DDI_INTR_CLAIMED;
 186 
 187 done:
 188         return (rc);
 189 }
 190 
 191 static int
 192 sfxge_intr_bus_enable(sfxge_t *sp)
 193 {
 194         sfxge_intr_t *sip = &(sp->s_intr);
 195         ddi_intr_handler_t *handler;
 196         int add_index;
 197         int en_index;
 198         int err;
 199         int rc;
 200 
 201         /* Serialise all instances to avoid problems seen in bug31184. */
 202         mutex_enter(&sfxge_global_lock);
 203 
 204         switch (sip->si_type) {
 205         case EFX_INTR_MESSAGE:
 206                 handler = sfxge_intr_message;
 207                 break;
 208 
 209         case EFX_INTR_LINE:
 210                 handler = sfxge_intr_line;
 211                 break;
 212 
 213         default:
 214                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 215                     "bus_enable: unknown intr type (si_type=%d nalloc=%d)",
 216                     sip->si_type, sip->si_nalloc);
 217                 ASSERT(B_FALSE);
 218                 rc = EINVAL;
 219                 goto fail1;
 220         }
 221 
 222         /* Try to add the handlers */
 223         for (add_index = 0; add_index < sip->si_nalloc; add_index++) {
 224                 unsigned int pri;
 225 
 226                 /* This cannot fail unless given invalid inputs. */
 227                 err = ddi_intr_get_pri(sip->si_table[add_index], &pri);
 228                 ASSERT(err == DDI_SUCCESS);
 229 
 230                 DTRACE_PROBE2(pri, unsigned int, add_index, unsigned int, pri);
 231 
 232                 err = ddi_intr_add_handler(sip->si_table[add_index], handler,
 233                     (caddr_t)sp, (caddr_t)(uintptr_t)add_index);
 234                 if (err != DDI_SUCCESS) {
 235                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 236                             "bus_enable: ddi_intr_add_handler failed"
 237                             " err=%d (h=%p idx=%d nalloc=%d)",
 238                             err, (void *)sip->si_table[add_index], add_index,
 239                             sip->si_nalloc);
 240 
 241                         rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
 242                         goto fail2;
 243                 }
 244         }
 245 
 246         /* Get interrupt capabilities */
 247         err = ddi_intr_get_cap(sip->si_table[0], &(sip->si_cap));
 248         if (err != DDI_SUCCESS) {
 249                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 250                     "bus_enable: ddi_intr_get_cap failed"
 251                     " err=%d (h=%p idx=%d nalloc=%d)",
 252                     err, (void *)sip->si_table[0], 0, sip->si_nalloc);
 253 
 254                 if (err == DDI_EINVAL)
 255                         rc = EINVAL;
 256                 else if (err == DDI_ENOTSUP)
 257                         rc = ENOTSUP;
 258                 else
 259                         rc = EFAULT;
 260 
 261                 goto fail3;
 262         }
 263 
 264         /* Enable interrupts at the bus  */
 265         if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
 266                 en_index = 0; /* Silence gcc */
 267                 err = ddi_intr_block_enable(sip->si_table, sip->si_nalloc);
 268                 if (err != DDI_SUCCESS) {
 269                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 270                             "bus_enable: ddi_intr_block_enable failed"
 271                             " err=%d (table=%p nalloc=%d)",
 272                             err, (void *)sip->si_table, sip->si_nalloc);
 273 
 274                         rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
 275                         goto fail4;
 276                 }
 277         } else {
 278                 for (en_index = 0; en_index < sip->si_nalloc; en_index++) {
 279                         err = ddi_intr_enable(sip->si_table[en_index]);
 280                         if (err != DDI_SUCCESS) {
 281                                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 282                                     "bus_enable: ddi_intr_enable failed"
 283                                     " err=%d (h=%p idx=%d nalloc=%d)",
 284                                     err, (void *)sip->si_table[en_index],
 285                                     en_index, sip->si_nalloc);
 286 
 287                                 rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
 288                                 goto fail4;
 289                         }
 290                 }
 291         }
 292 
 293         mutex_exit(&sfxge_global_lock);
 294         return (0);
 295 
 296 fail4:
 297         DTRACE_PROBE(fail4);
 298 
 299         /* Disable the enabled handlers */
 300         if (!(sip->si_cap & DDI_INTR_FLAG_BLOCK)) {
 301                 while (--en_index >= 0) {
 302                         err = ddi_intr_disable(sip->si_table[en_index]);
 303                         if (err != DDI_SUCCESS) {
 304                                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 305                                     "bus_enable: ddi_intr_disable"
 306                                     " failed err=%d (h=%p idx=%d nalloc=%d)",
 307                                     err, (void *)sip->si_table[en_index],
 308                                     en_index, sip->si_nalloc);
 309                         }
 310                 }
 311         }
 312 
 313 fail3:
 314         DTRACE_PROBE(fail3);
 315 
 316         /* Remove all handlers */
 317         add_index = sip->si_nalloc;
 318 
 319 fail2:
 320         DTRACE_PROBE(fail2);
 321 
 322         /* Remove remaining handlers */
 323         while (--add_index >= 0) {
 324                 err = ddi_intr_remove_handler(sip->si_table[add_index]);
 325                 if (err != DDI_SUCCESS) {
 326                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 327                             "bus_enable: ddi_intr_remove_handler"
 328                             " failed err=%d (h=%p idx=%d nalloc=%d)",
 329                             err, (void *)sip->si_table[add_index], add_index,
 330                             sip->si_nalloc);
 331                 }
 332         }
 333 
 334 fail1:
 335         DTRACE_PROBE1(fail1, int, rc);
 336 
 337         mutex_exit(&sfxge_global_lock);
 338         return (rc);
 339 }
 340 
 341 static void
 342 sfxge_intr_bus_disable(sfxge_t *sp)
 343 {
 344         sfxge_intr_t *sip = &(sp->s_intr);
 345         int index;
 346         int err;
 347 
 348         /* Serialise all instances to avoid problems seen in bug31184. */
 349         mutex_enter(&sfxge_global_lock);
 350 
 351         /* Disable interrupts at the bus */
 352         if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
 353                 err = ddi_intr_block_disable(sip->si_table, sip->si_nalloc);
 354                 if (err != DDI_SUCCESS) {
 355                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 356                             "bus_disable: ddi_intr_block_disable"
 357                             " failed err=%d (table=%p nalloc=%d)",
 358                             err, (void *)sip->si_table, sip->si_nalloc);
 359                 }
 360         } else {
 361                 index = sip->si_nalloc;
 362                 while (--index >= 0) {
 363                         err = ddi_intr_disable(sip->si_table[index]);
 364                         if (err != DDI_SUCCESS) {
 365                                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 366                                     "bus_disable: ddi_intr_disable"
 367                                     " failed err=%d (h=%p idx=%d nalloc=%d)",
 368                                     err, (void *)sip->si_table[index], index,
 369                                     sip->si_nalloc);
 370                         }
 371                 }
 372         }
 373 
 374         sip->si_cap = 0;
 375 
 376         /* Remove all handlers */
 377         index = sip->si_nalloc;
 378         while (--index >= 0) {
 379                 err = ddi_intr_remove_handler(sip->si_table[index]);
 380                 if (err != DDI_SUCCESS) {
 381                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 382                             "bus_disable: ddi_intr_remove_handler"
 383                             " failed err=%d (h=%p idx=%d nalloc=%d)",
 384                             err, (void *)sip->si_table[index], index,
 385                             sip->si_nalloc);
 386                 }
 387         }
 388 
 389         mutex_exit(&sfxge_global_lock);
 390 }
 391 
 392 static int
 393 sfxge_intr_nic_enable(sfxge_t *sp)
 394 {
 395         sfxge_intr_t *sip = &(sp->s_intr);
 396         efsys_mem_t *esmp = &(sip->si_mem);
 397         efx_nic_t *enp = sp->s_enp;
 398         unsigned int index;
 399         uint64_t mask;
 400         unsigned int count;
 401         int rc;
 402 
 403         /* Zero the memory */
 404         bzero(esmp->esm_base, EFX_INTR_SIZE);
 405 
 406         /* Enable interrupts at the NIC */
 407         if ((rc = efx_intr_init(enp, sip->si_type, esmp)) != 0)
 408                 goto fail1;
 409 
 410         efx_intr_enable(enp);
 411 
 412         /* FIXME FIXME FIXME */
 413         if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
 414                 /* Disable interrupt test until supported on Huntington. */
 415                 return (0);
 416         }
 417         /* FIXME FIXME FIXME */
 418 
 419         /* Test the interrupts */
 420         mask = 0;
 421         for (index = 0; index < sip->si_nalloc; index++) {
 422                 mask |= (1 << index);
 423 
 424                 rc = efx_intr_trigger(enp, index);
 425                 ASSERT3U(rc, ==, 0);
 426         }
 427 
 428         /* Wait for the tests to complete */
 429         count = 0;
 430         do {
 431                 DTRACE_PROBE1(wait, unsigned int, count);
 432 
 433                 /* Spin for 1 ms */
 434                 drv_usecwait(1000);
 435 
 436                 /*
 437                  * Check to see that all the test interrupts have been
 438                  * processed.
 439                  */
 440                 if ((mask & sip->si_mask) == mask)
 441                         goto done;
 442 
 443         } while (++count < 20);
 444 
 445         rc = ETIMEDOUT;
 446         goto fail2;
 447 
 448 done:
 449         return (0);
 450 
 451 fail2:
 452         DTRACE_PROBE(fail2);
 453 
 454         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 455             "Interrupt test failed (mask=%"PRIx64" got=%"
 456             PRIx64"). NIC is disabled",
 457             mask, sip->si_mask);
 458 
 459         DTRACE_PROBE2(int_test_fail, uint64_t, mask, uint64_t, sip->si_mask);
 460 
 461         sip->si_mask = 0;
 462 
 463         /* Disable interrupts at the NIC */
 464         efx_intr_disable(enp);
 465         efx_intr_fini(enp);
 466 
 467 fail1:
 468         DTRACE_PROBE1(fail1, int, rc);
 469 
 470         return (rc);
 471 }
 472 
 473 static void
 474 sfxge_intr_nic_disable(sfxge_t *sp)
 475 {
 476         sfxge_intr_t *sip = &(sp->s_intr);
 477         efx_nic_t *enp = sp->s_enp;
 478 
 479         sip->si_mask = 0;
 480 
 481         /* Disable interrupts at the NIC */
 482         efx_intr_disable(enp);
 483         efx_intr_fini(enp);
 484 }
 485 
 486 static inline unsigned
 487 pow2_le(unsigned long n)
 488 {
 489         unsigned int order = 1;
 490         ASSERT3U(n, >, 0);
 491         while ((1ul << order) <= n) ++order;
 492         return (1ul << (order - 1));
 493 }
 494 
 495 int
 496 sfxge_intr_init(sfxge_t *sp)
 497 {
 498         dev_info_t *dip = sp->s_dip;
 499         sfxge_intr_t *sip = &(sp->s_intr);
 500         efsys_mem_t *esmp = &(sip->si_mem);
 501         sfxge_dma_buffer_attr_t dma_attr;
 502         int err;
 503         int rc;
 504         int types;
 505         int type;
 506         int index;
 507         unsigned int nalloc;
 508         int navail;
 509 
 510         SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
 511 
 512         ASSERT3U(sip->si_state, ==, SFXGE_INTR_UNINITIALIZED);
 513 
 514 #ifdef __sparc
 515         /* PSARC 2007/453 */
 516         (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
 517             "#msix-request", NULL, 0);
 518 #endif
 519 
 520         /* Get the map of supported interrupt types */
 521         err = ddi_intr_get_supported_types(dip, &types);
 522         if (err != DDI_SUCCESS) {
 523                 dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 524                     "intr_init: ddi_intr_get_supported_types failed err=%d",
 525                     err);
 526 
 527                 if (err == DDI_EINVAL)
 528                         rc = EINVAL;
 529                 else if (err == DDI_INTR_NOTFOUND)
 530                         rc = ENOENT;
 531                 else
 532                         rc = EFAULT;
 533 
 534                 goto fail1;
 535         }
 536 
 537         /* Choose most favourable type */
 538         if (types & DDI_INTR_TYPE_MSIX) {
 539                 DTRACE_PROBE(msix);
 540 
 541                 type = DDI_INTR_TYPE_MSIX;
 542                 sip->si_type = EFX_INTR_MESSAGE;
 543         } else {
 544                 DTRACE_PROBE(fixed);
 545 
 546                 ASSERT(types & DDI_INTR_TYPE_FIXED);
 547 
 548                 type = DDI_INTR_TYPE_FIXED;
 549                 sip->si_type = EFX_INTR_LINE;
 550         }
 551 
 552         /* Get the number of available interrupts */
 553         navail = 0;
 554         err = ddi_intr_get_navail(dip, type, &navail);
 555         if (err != DDI_SUCCESS) {
 556                 dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 557                     "intr_init: ddi_intr_get_navail failed err=%d", err);
 558 
 559                 if (err == DDI_EINVAL)
 560                         rc = EINVAL;
 561                 else if (err == DDI_INTR_NOTFOUND)
 562                         rc = ENOENT;
 563                 else
 564                         rc = EFAULT;
 565 
 566                 goto fail2;
 567         }
 568 
 569         /* Double-check */
 570         if (navail == 0) {
 571                 rc = ENOENT;
 572                 goto fail2;
 573         }
 574 
 575         /*
 576          * Allow greater number of MSI-X interrupts than CPUs.
 577          * This can be useful to prevent RX no desc drops; See task 32179.
 578          * Limit non MSI-X interrupts to a single instance.
 579          */
 580         if (type != DDI_INTR_TYPE_MSIX)
 581                 navail = 1;
 582         else
 583                 navail = min(navail, sfxge_rx_scale_prop_get(sp));
 584 
 585         DTRACE_PROBE1(navail, unsigned int, navail);
 586 
 587         /* Allocate a handle table */
 588         sip->si_table_size = navail * sizeof (ddi_intr_handle_t);
 589         sip->si_table = kmem_zalloc(sip->si_table_size, KM_SLEEP);
 590 
 591         /*
 592          * Allocate interrupt handles.
 593          * Serialise all device instances to avoid problems seen in bug31184.
 594          */
 595         mutex_enter(&sfxge_global_lock);
 596 
 597         err = ddi_intr_alloc(dip, sip->si_table, type, 0,
 598             navail, &(sip->si_nalloc), DDI_INTR_ALLOC_NORMAL);
 599 
 600         mutex_exit(&sfxge_global_lock);
 601 
 602         if (err != DDI_SUCCESS) {
 603                 dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 604                     "intr_init: ddi_intr_alloc failed err=%d"
 605                     " (navail=%d nalloc=%d)",
 606                     err, navail, sip->si_nalloc);
 607 
 608                 if (err == DDI_EINVAL)
 609                         rc = EINVAL;
 610                 else if (err == DDI_EAGAIN)
 611                         rc = EAGAIN;
 612                 else if (err == DDI_INTR_NOTFOUND)
 613                         rc = ENOENT;
 614                 else
 615                         rc = EFAULT;
 616 
 617                 goto fail3;
 618         }
 619 
 620         /* Double-check */
 621         if (sip->si_nalloc == 0) {
 622                 rc = ENOENT;
 623                 goto fail3;
 624         }
 625 
 626         /* Round down to a power of 2 */
 627         nalloc = pow2_le(sip->si_nalloc);
 628 
 629         /* Free off any excess handles */
 630         mutex_enter(&sfxge_global_lock);
 631 
 632         index = sip->si_nalloc;
 633         while (--index >= nalloc) {
 634                 (void) ddi_intr_free(sip->si_table[index]);
 635                 sip->si_table[index] = NULL;
 636         }
 637 
 638         mutex_exit(&sfxge_global_lock);
 639 
 640         sip->si_nalloc = nalloc;
 641         DTRACE_PROBE1(nalloc, unsigned int, sip->si_nalloc);
 642 
 643         dma_attr.sdba_dip        = sp->s_dip;
 644         dma_attr.sdba_dattrp     = &sfxge_intr_dma_attr;
 645         dma_attr.sdba_callback   = DDI_DMA_SLEEP;
 646         dma_attr.sdba_length     = EFX_INTR_SIZE;
 647         dma_attr.sdba_memflags   = DDI_DMA_CONSISTENT;
 648         dma_attr.sdba_devaccp    = &sfxge_intr_devacc;
 649         dma_attr.sdba_bindflags  = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
 650         dma_attr.sdba_maxcookies = 1;
 651         dma_attr.sdba_zeroinit   = B_TRUE;
 652 
 653         if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
 654                 goto fail4;
 655 
 656         /* Store the highest priority for convenience */
 657         sip->si_intr_pri = 0;
 658         for (index = 0; index < sip->si_nalloc; index++) {
 659                 uint_t pri;
 660                 if ((rc = ddi_intr_get_pri(sip->si_table[index], &pri)) !=  0)
 661                         goto fail5;
 662                 if (pri > sip->si_intr_pri)
 663                         sip->si_intr_pri = pri;
 664         }
 665 
 666         sip->si_state = SFXGE_INTR_INITIALIZED;
 667         return (0);
 668 
 669 fail5:
 670         DTRACE_PROBE(fail5);
 671 
 672 fail4:
 673         DTRACE_PROBE(fail4);
 674 
 675         /* Free interrupt handles */
 676         mutex_exit(&sfxge_global_lock);
 677 
 678         index = sip->si_nalloc;
 679         while (--index >= 0) {
 680                 err = ddi_intr_free(sip->si_table[index]);
 681                 if (err != DDI_SUCCESS) {
 682                         dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 683                             "intr_init: ddi_intr_free failed err=%d"
 684                             " (h=%p idx=%d nalloc=%d)",
 685                             err, (void *)sip->si_table[index], index,
 686                             sip->si_nalloc);
 687                 }
 688                 sip->si_table[index] = NULL;
 689         }
 690         sip->si_nalloc = 0;
 691 
 692         mutex_exit(&sfxge_global_lock);
 693 
 694 fail3:
 695         DTRACE_PROBE(fail3);
 696 
 697         /* Free the handle table */
 698         kmem_free(sip->si_table, sip->si_table_size);
 699         sip->si_table = NULL;
 700         sip->si_table_size = 0;
 701 
 702 fail2:
 703         DTRACE_PROBE(fail2);
 704 
 705         /* Clear the interrupt type */
 706         sip->si_type = EFX_INTR_INVALID;
 707 
 708 fail1:
 709         DTRACE_PROBE1(fail1, int, rc);
 710 
 711         SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
 712 
 713         return (rc);
 714 }
 715 
 716 int
 717 sfxge_intr_start(sfxge_t *sp)
 718 {
 719         sfxge_intr_t *sip = &(sp->s_intr);
 720         int rc;
 721 
 722         ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
 723 
 724         /* Enable interrupts at the bus */
 725         if ((rc = sfxge_intr_bus_enable(sp)) != 0)
 726                 goto fail1;
 727 
 728         sip->si_state = SFXGE_INTR_TESTING;
 729 
 730         /* Enable interrupts at the NIC */
 731         if ((rc = sfxge_intr_nic_enable(sp)) != 0)
 732                 goto fail2;
 733 
 734         sip->si_state = SFXGE_INTR_STARTED;
 735 
 736         return (0);
 737 
 738 fail2:
 739         DTRACE_PROBE(fail2);
 740 
 741         /* Disable interrupts at the bus */
 742         sfxge_intr_bus_disable(sp);
 743 
 744 fail1:
 745         DTRACE_PROBE1(fail1, int, rc);
 746 
 747         sip->si_state = SFXGE_INTR_INITIALIZED;
 748 
 749         return (rc);
 750 }
 751 
 752 void
 753 sfxge_intr_stop(sfxge_t *sp)
 754 {
 755         sfxge_intr_t *sip = &(sp->s_intr);
 756 
 757         ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
 758 
 759         sip->si_state = SFXGE_INTR_INITIALIZED;
 760 
 761         /* Disable interrupts at the NIC */
 762         sfxge_intr_nic_disable(sp);
 763 
 764         /* Disable interrupts at the bus */
 765         sfxge_intr_bus_disable(sp);
 766 }
 767 
 768 void
 769 sfxge_intr_fini(sfxge_t *sp)
 770 {
 771         sfxge_intr_t *sip = &(sp->s_intr);
 772         efsys_mem_t *esmp = &(sip->si_mem);
 773         int index;
 774         int err;
 775 
 776         ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
 777 
 778         sip->si_state = SFXGE_INTR_UNINITIALIZED;
 779 
 780         /* Tear down dma setup */
 781         sfxge_dma_buffer_destroy(esmp);
 782 
 783 
 784         /* Free interrupt handles */
 785         mutex_enter(&sfxge_global_lock);
 786 
 787         index = sip->si_nalloc;
 788         while (--index >= 0) {
 789                 err = ddi_intr_free(sip->si_table[index]);
 790                 if (err != DDI_SUCCESS) {
 791                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 792                             "intr_fini: ddi_intr_free failed err=%d"
 793                             " (h=%p idx=%d nalloc=%d)",
 794                             err, (void *)sip->si_table[index],
 795                             index, sip->si_nalloc);
 796                 }
 797                 sip->si_table[index] = NULL;
 798         }
 799         sip->si_nalloc = 0;
 800 
 801         mutex_exit(&sfxge_global_lock);
 802 
 803         /* Free the handle table */
 804         kmem_free(sip->si_table, sip->si_table_size);
 805         sip->si_table = NULL;
 806         sip->si_table_size = 0;
 807 
 808         /* Clear the interrupt type */
 809         sip->si_type = EFX_INTR_INVALID;
 810 
 811         SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
 812 }