1 /*
   2  * Copyright (c) 2008-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30 
  31 #include <sys/types.h>
  32 #include <sys/ddi.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/atomic.h>
  35 #include <sys/modctl.h>
  36 #include <sys/conf.h>
  37 #include <sys/ethernet.h>
  38 #include <sys/pci.h>
  39 #include <sys/pcie.h>
  40 
  41 #include "sfxge.h"
  42 
  43 #include "efx.h"
  44 
  45 
  46 /* Interrupt table DMA attributes */
  47 static ddi_device_acc_attr_t sfxge_intr_devacc = {
  48 
  49         DDI_DEVICE_ATTR_V0,     /* devacc_attr_version */
  50         DDI_NEVERSWAP_ACC,      /* devacc_attr_endian_flags */
  51         DDI_STRICTORDER_ACC     /* devacc_attr_dataorder */
  52 };
  53 
  54 static ddi_dma_attr_t sfxge_intr_dma_attr = {
  55         DMA_ATTR_V0,            /* dma_attr_version     */
  56         0,                      /* dma_attr_addr_lo     */
  57         0xffffffffffffffffull,  /* dma_attr_addr_hi     */
  58         0xffffffffffffffffull,  /* dma_attr_count_max   */
  59         EFX_INTR_SIZE,          /* dma_attr_align       */
  60         0xffffffff,             /* dma_attr_burstsizes  */
  61         1,                      /* dma_attr_minxfer     */
  62         0xffffffffffffffffull,  /* dma_attr_maxxfer     */
  63         0xffffffffffffffffull,  /* dma_attr_seg         */
  64         1,                      /* dma_attr_sgllen      */
  65         1,                      /* dma_attr_granular    */
  66         0                       /* dma_attr_flags       */
  67 };
  68 
  69 static unsigned int
  70 sfxge_intr_line(caddr_t arg1, caddr_t arg2)
  71 {
  72         sfxge_t *sp = (void *)arg1;
  73         efx_nic_t *enp = sp->s_enp;
  74         sfxge_intr_t *sip = &(sp->s_intr);
  75         unsigned int index;
  76         boolean_t fatal;
  77         uint32_t qmask;
  78         int rc;
  79 
  80         _NOTE(ARGUNUSED(arg2))
  81 
  82         ASSERT3U(sip->si_type, ==, EFX_INTR_LINE);
  83 
  84         if (sip->si_state != SFXGE_INTR_STARTED &&
  85             sip->si_state != SFXGE_INTR_TESTING) {
  86                 rc = DDI_INTR_UNCLAIMED;
  87                 goto done;
  88         }
  89 
  90         if (sip->si_state == SFXGE_INTR_TESTING) {
  91                 sip->si_mask |= 1;   /* only one interrupt */
  92                 rc = DDI_INTR_CLAIMED;
  93                 goto done;
  94         }
  95 
  96         efx_intr_status_line(enp, &fatal, &qmask);
  97 
  98         if (fatal) {
  99                 sfxge_intr_fatal(sp);
 100 
 101                 rc = DDI_INTR_CLAIMED;
 102                 goto done;
 103         }
 104 
 105         if (qmask != 0) {
 106                 for (index = 0; index < EFX_INTR_NEVQS; index++) {
 107                         if (qmask & (1 << index))
 108                                 (void) sfxge_ev_qpoll(sp, index);
 109                 }
 110 
 111                 sip->si_zero_count = 0;
 112                 sfxge_gld_rx_push(sp);
 113                 rc = DDI_INTR_CLAIMED;
 114                 goto done;
 115         }
 116 
 117         /*
 118          * bug15671/bug17203 workaround. Return CLAIMED for the first ISR=0
 119          * interrupt, and poll all evqs for work. For subsequent ISR=0
 120          * interrupts (the line must be shared in this case), just rearm the
 121          * event queues to ensure we don't miss an interrupt.
 122          */
 123         if (sip->si_zero_count++ == 0) {
 124                 for (index = 0; index < EFX_INTR_NEVQS; index++) {
 125                         if (sp->s_sep[index] != NULL)
 126                                 (void) sfxge_ev_qpoll(sp, index);
 127                 }
 128 
 129                 rc = DDI_INTR_CLAIMED;
 130         } else {
 131                 for (index = 0; index < EFX_INTR_NEVQS; index++) {
 132                         if (sp->s_sep[index] != NULL)
 133                                 (void) sfxge_ev_qprime(sp, index);
 134                 }
 135 
 136                 rc = DDI_INTR_UNCLAIMED;
 137         }
 138 
 139 done:
 140         return (rc);
 141 }
 142 
 143 static unsigned int
 144 sfxge_intr_message(caddr_t arg1, caddr_t arg2)
 145 {
 146         sfxge_t *sp = (void *)arg1;
 147         efx_nic_t *enp = sp->s_enp;
 148         sfxge_intr_t *sip = &(sp->s_intr);
 149         unsigned int index = (unsigned int)(uintptr_t)arg2;
 150         boolean_t fatal;
 151         int rc;
 152 
 153         ASSERT3U(sip->si_type, ==, EFX_INTR_MESSAGE);
 154 
 155         if (sip->si_state != SFXGE_INTR_STARTED &&
 156             sip->si_state != SFXGE_INTR_TESTING) {
 157                 rc = DDI_INTR_UNCLAIMED;
 158                 goto done;
 159         }
 160 
 161         if (sip->si_state == SFXGE_INTR_TESTING) {
 162                 uint64_t mask;
 163 
 164                 do {
 165                         mask = sip->si_mask;
 166                 } while (atomic_cas_64(&(sip->si_mask), mask,
 167                     mask | (1 << index)) != mask);
 168 
 169                 rc = DDI_INTR_CLAIMED;
 170                 goto done;
 171         }
 172 
 173         efx_intr_status_message(enp, index, &fatal);
 174 
 175         if (fatal) {
 176                 sfxge_intr_fatal(sp);
 177 
 178                 rc = DDI_INTR_CLAIMED;
 179                 goto done;
 180         }
 181 
 182         (void) sfxge_ev_qpoll(sp, index);
 183 
 184         sfxge_gld_rx_push(sp);
 185         rc = DDI_INTR_CLAIMED;
 186 
 187 done:
 188         return (rc);
 189 }
 190 
 191 static int
 192 sfxge_intr_bus_enable(sfxge_t *sp)
 193 {
 194         sfxge_intr_t *sip = &(sp->s_intr);
 195         ddi_intr_handler_t *handler;
 196         int add_index;
 197         int en_index;
 198         int err;
 199         int rc;
 200 
 201         /* Serialise all instances to avoid problems seen in bug31184. */
 202         mutex_enter(&sfxge_global_lock);
 203 
 204         switch (sip->si_type) {
 205         case EFX_INTR_MESSAGE:
 206                 handler = sfxge_intr_message;
 207                 break;
 208 
 209         case EFX_INTR_LINE:
 210                 handler = sfxge_intr_line;
 211                 break;
 212 
 213         default:
 214                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 215                     "bus_enable: unknown intr type (si_type=%d nalloc=%d)",
 216                     sip->si_type, sip->si_nalloc);
 217                 ASSERT(B_FALSE);
 218                 rc = EINVAL;
 219                 goto fail1;
 220         }
 221 
 222         /* Try to add the handlers */
 223         for (add_index = 0; add_index < sip->si_nalloc; add_index++) {
 224                 unsigned int pri;
 225 
 226                 (void) ddi_intr_get_pri(sip->si_table[add_index], &pri);
 227                 DTRACE_PROBE2(pri, unsigned int, add_index, unsigned int, pri);
 228 
 229                 err = ddi_intr_add_handler(sip->si_table[add_index], handler,
 230                     (caddr_t)sp, (caddr_t)(uintptr_t)add_index);
 231                 if (err != DDI_SUCCESS) {
 232                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 233                             "bus_enable: ddi_intr_add_handler failed"
 234                             " err=%d (h=%p idx=%d nalloc=%d)",
 235                             err, (void *)sip->si_table[add_index], add_index,
 236                             sip->si_nalloc);
 237 
 238                         rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
 239                         goto fail2;
 240                 }
 241         }
 242 
 243         /* Get interrupt capabilities */
 244         err = ddi_intr_get_cap(sip->si_table[0], &(sip->si_cap));
 245         if (err != DDI_SUCCESS) {
 246                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 247                     "bus_enable: ddi_intr_get_cap failed"
 248                     " err=%d (h=%p idx=%d nalloc=%d)",
 249                     err, (void *)sip->si_table[0], 0, sip->si_nalloc);
 250 
 251                 if (err == DDI_EINVAL)
 252                         rc = EINVAL;
 253                 else if (err == DDI_ENOTSUP)
 254                         rc = ENOTSUP;
 255                 else
 256                         rc = EFAULT;
 257 
 258                 goto fail3;
 259         }
 260 
 261         /* Enable interrupts at the bus  */
 262         if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
 263                 en_index = 0; /* Silence gcc */
 264                 err = ddi_intr_block_enable(sip->si_table, sip->si_nalloc);
 265                 if (err != DDI_SUCCESS) {
 266                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 267                             "bus_enable: ddi_intr_block_enable failed"
 268                             " err=%d (table=%p nalloc=%d)",
 269                             err, (void *)sip->si_table, sip->si_nalloc);
 270 
 271                         rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
 272                         goto fail4;
 273                 }
 274         } else {
 275                 for (en_index = 0; en_index < sip->si_nalloc; en_index++) {
 276                         err = ddi_intr_enable(sip->si_table[en_index]);
 277                         if (err != DDI_SUCCESS) {
 278                                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 279                                     "bus_enable: ddi_intr_enable failed"
 280                                     " err=%d (h=%p idx=%d nalloc=%d)",
 281                                     err, (void *)sip->si_table[en_index],
 282                                     en_index, sip->si_nalloc);
 283 
 284                                 rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
 285                                 goto fail4;
 286                         }
 287                 }
 288         }
 289 
 290         mutex_exit(&sfxge_global_lock);
 291         return (0);
 292 
 293 fail4:
 294         DTRACE_PROBE(fail4);
 295 
 296         /* Disable the enabled handlers */
 297         if (!(sip->si_cap & DDI_INTR_FLAG_BLOCK)) {
 298                 while (--en_index >= 0) {
 299                         err = ddi_intr_disable(sip->si_table[en_index]);
 300                         if (err != DDI_SUCCESS) {
 301                                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 302                                     "bus_enable: ddi_intr_disable"
 303                                     " failed err=%d (h=%p idx=%d nalloc=%d)",
 304                                     err, (void *)sip->si_table[en_index],
 305                                     en_index, sip->si_nalloc);
 306                         }
 307                 }
 308         }
 309 
 310 fail3:
 311         DTRACE_PROBE(fail3);
 312 
 313         /* Remove all handlers */
 314         add_index = sip->si_nalloc;
 315 
 316 fail2:
 317         DTRACE_PROBE(fail2);
 318 
 319         /* Remove remaining handlers */
 320         while (--add_index >= 0) {
 321                 err = ddi_intr_remove_handler(sip->si_table[add_index]);
 322                 if (err != DDI_SUCCESS) {
 323                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 324                             "bus_enable: ddi_intr_remove_handler"
 325                             " failed err=%d (h=%p idx=%d nalloc=%d)",
 326                             err, (void *)sip->si_table[add_index], add_index,
 327                             sip->si_nalloc);
 328                 }
 329         }
 330 
 331 fail1:
 332         DTRACE_PROBE1(fail1, int, rc);
 333 
 334         mutex_exit(&sfxge_global_lock);
 335         return (rc);
 336 }
 337 
 338 static void
 339 sfxge_intr_bus_disable(sfxge_t *sp)
 340 {
 341         sfxge_intr_t *sip = &(sp->s_intr);
 342         int index;
 343         int err;
 344 
 345         /* Serialise all instances to avoid problems seen in bug31184. */
 346         mutex_enter(&sfxge_global_lock);
 347 
 348         /* Disable interrupts at the bus */
 349         if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
 350                 err = ddi_intr_block_disable(sip->si_table, sip->si_nalloc);
 351                 if (err != DDI_SUCCESS) {
 352                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 353                             "bus_disable: ddi_intr_block_disable"
 354                             " failed err=%d (table=%p nalloc=%d)",
 355                             err, (void *)sip->si_table, sip->si_nalloc);
 356                 }
 357         } else {
 358                 index = sip->si_nalloc;
 359                 while (--index >= 0) {
 360                         err = ddi_intr_disable(sip->si_table[index]);
 361                         if (err != DDI_SUCCESS) {
 362                                 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 363                                     "bus_disable: ddi_intr_disable"
 364                                     " failed err=%d (h=%p idx=%d nalloc=%d)",
 365                                     err, (void *)sip->si_table[index], index,
 366                                     sip->si_nalloc);
 367                         }
 368                 }
 369         }
 370 
 371         sip->si_cap = 0;
 372 
 373         /* Remove all handlers */
 374         index = sip->si_nalloc;
 375         while (--index >= 0) {
 376                 err = ddi_intr_remove_handler(sip->si_table[index]);
 377                 if (err != DDI_SUCCESS) {
 378                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 379                             "bus_disable: ddi_intr_remove_handler"
 380                             " failed err=%d (h=%p idx=%d nalloc=%d)",
 381                             err, (void *)sip->si_table[index], index,
 382                             sip->si_nalloc);
 383                 }
 384         }
 385 
 386         mutex_exit(&sfxge_global_lock);
 387 }
 388 
 389 static int
 390 sfxge_intr_nic_enable(sfxge_t *sp)
 391 {
 392         sfxge_intr_t *sip = &(sp->s_intr);
 393         efsys_mem_t *esmp = &(sip->si_mem);
 394         efx_nic_t *enp = sp->s_enp;
 395         unsigned int index;
 396         uint64_t mask;
 397         unsigned int count;
 398         int rc;
 399 
 400         /* Zero the memory */
 401         bzero(esmp->esm_base, EFX_INTR_SIZE);
 402 
 403         /* Enable interrupts at the NIC */
 404         if ((rc = efx_intr_init(enp, sip->si_type, esmp)) != 0)
 405                 goto fail1;
 406 
 407         efx_intr_enable(enp);
 408 
 409         /* FIXME FIXME FIXME */
 410         if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
 411                 /* Disable interrupt test until supported on Huntington. */
 412                 return (0);
 413         }
 414         /* FIXME FIXME FIXME */
 415 
 416         /* Test the interrupts */
 417         mask = 0;
 418         for (index = 0; index < sip->si_nalloc; index++) {
 419                 mask |= (1 << index);
 420 
 421                 rc = efx_intr_trigger(enp, index);
 422                 ASSERT3U(rc, ==, 0);
 423         }
 424 
 425         /* Wait for the tests to complete */
 426         count = 0;
 427         do {
 428                 DTRACE_PROBE1(wait, unsigned int, count);
 429 
 430                 /* Spin for 1 ms */
 431                 drv_usecwait(1000);
 432 
 433                 /*
 434                  * Check to see that all the test interrupts have been
 435                  * processed.
 436                  */
 437                 if ((mask & sip->si_mask) == mask)
 438                         goto done;
 439 
 440         } while (++count < 20);
 441 
 442         rc = ETIMEDOUT;
 443         goto fail2;
 444 
 445 done:
 446         return (0);
 447 
 448 fail2:
 449         DTRACE_PROBE(fail2);
 450 
 451         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 452             "Interrupt test failed (mask=%"PRIx64" got=%"
 453             PRIx64"). NIC is disabled",
 454             mask, sip->si_mask);
 455 
 456         DTRACE_PROBE2(int_test_fail, uint64_t, mask, uint64_t, sip->si_mask);
 457 
 458         sip->si_mask = 0;
 459 
 460         /* Disable interrupts at the NIC */
 461         efx_intr_disable(enp);
 462         efx_intr_fini(enp);
 463 
 464 fail1:
 465         DTRACE_PROBE1(fail1, int, rc);
 466 
 467         return (rc);
 468 }
 469 
 470 static void
 471 sfxge_intr_nic_disable(sfxge_t *sp)
 472 {
 473         sfxge_intr_t *sip = &(sp->s_intr);
 474         efx_nic_t *enp = sp->s_enp;
 475 
 476         sip->si_mask = 0;
 477 
 478         /* Disable interrupts at the NIC */
 479         efx_intr_disable(enp);
 480         efx_intr_fini(enp);
 481 }
 482 
 483 inline unsigned pow2_le(unsigned long n) {
 484         unsigned int order = 1;
 485         ASSERT3U(n, >, 0);
 486         while ((1ul << order) <= n) ++order;
 487         return (1ul << (order - 1));
 488 }
 489 
 490 int
 491 sfxge_intr_init(sfxge_t *sp)
 492 {
 493         dev_info_t *dip = sp->s_dip;
 494         sfxge_intr_t *sip = &(sp->s_intr);
 495         efsys_mem_t *esmp = &(sip->si_mem);
 496         sfxge_dma_buffer_attr_t dma_attr;
 497         int err;
 498         int rc;
 499         int types;
 500         int type;
 501         int index;
 502         unsigned int nalloc;
 503         int navail;
 504 
 505         SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
 506 
 507         ASSERT3U(sip->si_state, ==, SFXGE_INTR_UNINITIALIZED);
 508 
 509 #ifdef __sparc
 510         /* PSARC 2007/453 */
 511         (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
 512             "#msix-request", NULL, 0);
 513 #endif
 514 
 515         /* Get the map of supported interrupt types */
 516         err = ddi_intr_get_supported_types(dip, &types);
 517         if (err != DDI_SUCCESS) {
 518                 dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 519                     "intr_init: ddi_intr_get_supported_types failed err=%d",
 520                     err);
 521 
 522                 if (err == DDI_EINVAL)
 523                         rc = EINVAL;
 524                 else if (err == DDI_INTR_NOTFOUND)
 525                         rc = ENOENT;
 526                 else
 527                         rc = EFAULT;
 528 
 529                 goto fail1;
 530         }
 531 
 532         /* Choose most favourable type */
 533         if (types & DDI_INTR_TYPE_MSIX) {
 534                 DTRACE_PROBE(msix);
 535 
 536                 type = DDI_INTR_TYPE_MSIX;
 537                 sip->si_type = EFX_INTR_MESSAGE;
 538         } else {
 539                 DTRACE_PROBE(fixed);
 540 
 541                 ASSERT(types & DDI_INTR_TYPE_FIXED);
 542 
 543                 type = DDI_INTR_TYPE_FIXED;
 544                 sip->si_type = EFX_INTR_LINE;
 545         }
 546 
 547         /* Get the number of available interrupts */
 548         navail = 0;
 549         err = ddi_intr_get_navail(dip, type, &navail);
 550         if (err != DDI_SUCCESS) {
 551                 dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 552                     "intr_init: ddi_intr_get_navail failed err=%d", err);
 553 
 554                 if (err == DDI_EINVAL)
 555                         rc = EINVAL;
 556                 else if (err == DDI_INTR_NOTFOUND)
 557                         rc = ENOENT;
 558                 else
 559                         rc = EFAULT;
 560 
 561                 goto fail2;
 562         }
 563 
 564         /* Double-check */
 565         if (navail == 0) {
 566                 rc = ENOENT;
 567                 goto fail2;
 568         }
 569 
 570         /*
 571          * Allow greater number of MSI-X interrupts than CPUs.
 572          * This can be useful to prevent RX no desc drops; See task 32179.
 573          * Limit non MSI-X interrupts to a single instance.
 574          */
 575         if (type != DDI_INTR_TYPE_MSIX)
 576                 navail = 1;
 577         else
 578                 navail = min(navail, sfxge_rx_scale_prop_get(sp));
 579 
 580         DTRACE_PROBE1(navail, unsigned int, navail);
 581 
 582         /* Allocate a handle table */
 583         sip->si_table_size = navail * sizeof (ddi_intr_handle_t);
 584         sip->si_table = kmem_zalloc(sip->si_table_size, KM_SLEEP);
 585 
 586         /*
 587          * Allocate interrupt handles.
 588          * Serialise all device instances to avoid problems seen in bug31184.
 589          */
 590         mutex_enter(&sfxge_global_lock);
 591 
 592         err = ddi_intr_alloc(dip, sip->si_table, type, 0,
 593             navail, &(sip->si_nalloc), DDI_INTR_ALLOC_NORMAL);
 594 
 595         mutex_exit(&sfxge_global_lock);
 596 
 597         if (err != DDI_SUCCESS) {
 598                 dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 599                     "intr_init: ddi_intr_alloc failed err=%d"
 600                     " (navail=%d nalloc=%d)",
 601                     err, navail, sip->si_nalloc);
 602 
 603                 if (err == DDI_EINVAL)
 604                         rc = EINVAL;
 605                 else if (err == DDI_EAGAIN)
 606                         rc = EAGAIN;
 607                 else if (err == DDI_INTR_NOTFOUND)
 608                         rc = ENOENT;
 609                 else
 610                         rc = EFAULT;
 611 
 612                 goto fail3;
 613         }
 614 
 615         /* Double-check */
 616         if (sip->si_nalloc == 0) {
 617                 rc = ENOENT;
 618                 goto fail3;
 619         }
 620 
 621         /* Round down to a power of 2 */
 622         nalloc = pow2_le(sip->si_nalloc);
 623 
 624         /* Free off any excess handles */
 625         mutex_enter(&sfxge_global_lock);
 626 
 627         index = sip->si_nalloc;
 628         while (--index >= nalloc) {
 629                 (void) ddi_intr_free(sip->si_table[index]);
 630                 sip->si_table[index] = NULL;
 631         }
 632 
 633         mutex_exit(&sfxge_global_lock);
 634 
 635         sip->si_nalloc = nalloc;
 636         DTRACE_PROBE1(nalloc, unsigned int, sip->si_nalloc);
 637 
 638         dma_attr.sdba_dip        = sp->s_dip;
 639         dma_attr.sdba_dattrp     = &sfxge_intr_dma_attr;
 640         dma_attr.sdba_callback   = DDI_DMA_SLEEP;
 641         dma_attr.sdba_length     = EFX_INTR_SIZE;
 642         dma_attr.sdba_memflags   = DDI_DMA_CONSISTENT;
 643         dma_attr.sdba_devaccp    = &sfxge_intr_devacc;
 644         dma_attr.sdba_bindflags  = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
 645         dma_attr.sdba_maxcookies = 1;
 646         dma_attr.sdba_zeroinit   = B_TRUE;
 647 
 648         if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
 649                 goto fail4;
 650 
 651         /* Store the highest priority for convenience */
 652         sip->si_intr_pri = 0;
 653         for (index = 0; index < sip->si_nalloc; index++) {
 654                 uint_t pri;
 655                 if ((rc = ddi_intr_get_pri(sip->si_table[index], &pri)) !=  0)
 656                         goto fail5;
 657                 if (pri > sip->si_intr_pri)
 658                         sip->si_intr_pri = pri;
 659         }
 660 
 661         sip->si_state = SFXGE_INTR_INITIALIZED;
 662         return (0);
 663 
 664 fail5:
 665         DTRACE_PROBE(fail5);
 666 
 667 fail4:
 668         DTRACE_PROBE(fail4);
 669 
 670         /* Free interrupt handles */
 671         mutex_exit(&sfxge_global_lock);
 672 
 673         index = sip->si_nalloc;
 674         while (--index >= 0) {
 675                 err = ddi_intr_free(sip->si_table[index]);
 676                 if (err != DDI_SUCCESS) {
 677                         dev_err(dip, CE_WARN, SFXGE_CMN_ERR
 678                             "intr_init: ddi_intr_free failed err=%d"
 679                             " (h=%p idx=%d nalloc=%d)",
 680                             err, (void *)sip->si_table[index], index,
 681                             sip->si_nalloc);
 682                 }
 683                 sip->si_table[index] = NULL;
 684         }
 685         sip->si_nalloc = 0;
 686 
 687         mutex_exit(&sfxge_global_lock);
 688 
 689 fail3:
 690         DTRACE_PROBE(fail3);
 691 
 692         /* Free the handle table */
 693         kmem_free(sip->si_table, sip->si_table_size);
 694         sip->si_table = NULL;
 695         sip->si_table_size = 0;
 696 
 697 fail2:
 698         DTRACE_PROBE(fail2);
 699 
 700         /* Clear the interrupt type */
 701         sip->si_type = EFX_INTR_INVALID;
 702 
 703 fail1:
 704         DTRACE_PROBE1(fail1, int, rc);
 705 
 706         SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
 707 
 708         return (rc);
 709 }
 710 
 711 int
 712 sfxge_intr_start(sfxge_t *sp)
 713 {
 714         sfxge_intr_t *sip = &(sp->s_intr);
 715         int rc;
 716 
 717         ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
 718 
 719         /* Enable interrupts at the bus */
 720         if ((rc = sfxge_intr_bus_enable(sp)) != 0)
 721                 goto fail1;
 722 
 723         sip->si_state = SFXGE_INTR_TESTING;
 724 
 725         /* Enable interrupts at the NIC */
 726         if ((rc = sfxge_intr_nic_enable(sp)) != 0)
 727                 goto fail2;
 728 
 729         sip->si_state = SFXGE_INTR_STARTED;
 730 
 731         return (0);
 732 
 733 fail2:
 734         DTRACE_PROBE(fail2);
 735 
 736         /* Disable interrupts at the bus */
 737         sfxge_intr_bus_disable(sp);
 738 
 739 fail1:
 740         DTRACE_PROBE1(fail1, int, rc);
 741 
 742         sip->si_state = SFXGE_INTR_INITIALIZED;
 743 
 744         return (rc);
 745 }
 746 
 747 void
 748 sfxge_intr_stop(sfxge_t *sp)
 749 {
 750         sfxge_intr_t *sip = &(sp->s_intr);
 751 
 752         ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
 753 
 754         sip->si_state = SFXGE_INTR_INITIALIZED;
 755 
 756         /* Disable interrupts at the NIC */
 757         sfxge_intr_nic_disable(sp);
 758 
 759         /* Disable interrupts at the bus */
 760         sfxge_intr_bus_disable(sp);
 761 }
 762 
 763 void
 764 sfxge_intr_fini(sfxge_t *sp)
 765 {
 766         sfxge_intr_t *sip = &(sp->s_intr);
 767         efsys_mem_t *esmp = &(sip->si_mem);
 768         int index;
 769         int err;
 770 
 771         ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
 772 
 773         sip->si_state = SFXGE_INTR_UNINITIALIZED;
 774 
 775         /* Tear down dma setup */
 776         sfxge_dma_buffer_destroy(esmp);
 777 
 778 
 779         /* Free interrupt handles */
 780         mutex_enter(&sfxge_global_lock);
 781 
 782         index = sip->si_nalloc;
 783         while (--index >= 0) {
 784                 err = ddi_intr_free(sip->si_table[index]);
 785                 if (err != DDI_SUCCESS) {
 786                         dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
 787                             "intr_fini: ddi_intr_free failed err=%d"
 788                             " (h=%p idx=%d nalloc=%d)",
 789                             err, (void *)sip->si_table[index],
 790                             index, sip->si_nalloc);
 791                 }
 792                 sip->si_table[index] = NULL;
 793         }
 794         sip->si_nalloc = 0;
 795 
 796         mutex_exit(&sfxge_global_lock);
 797 
 798         /* Free the handle table */
 799         kmem_free(sip->si_table, sip->si_table_size);
 800         sip->si_table = NULL;
 801         sip->si_table_size = 0;
 802 
 803         /* Clear the interrupt type */
 804         sip->si_type = EFX_INTR_INVALID;
 805 
 806         SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
 807 }