Print this page 
7813 mpt_sas does not like concurrent HBA resets
Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c
          +++ new/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c
↓ open down ↓ 548 lines elided ↑ open up ↑ 
 549  549          NULL,                   /* bus operations */
 550  550          mptsas_power,           /* power management */
 551  551  #ifdef  __sparc
 552  552          ddi_quiesce_not_needed
 553  553  #else
 554  554          mptsas_quiesce          /* quiesce */
 555  555  #endif  /* __sparc */
 556  556  };
 557  557  
 558  558  
 559      -#define MPTSAS_MOD_STRING "MPTSAS HBA Driver 00.00.00.24"
      559 +#define MPTSAS_MOD_STRING "MPTSAS HBA Driver 00.00.00.24X"
 560  560  
 561  561  static struct modldrv modldrv = {
 562  562          &mod_driverops, /* Type of module. This one is a driver */
 563  563          MPTSAS_MOD_STRING, /* Name of the module. */
 564  564          &mptsas_ops,    /* driver ops */
 565  565  };
 566  566  
 567  567  static struct modlinkage modlinkage = {
 568  568          MODREV_1, &modldrv, NULL
 569  569  };
↓ open down ↓ 758 lines elided ↑ open up ↑ 
1328 1328  
1329 1329          /*
1330 1330           * Disable hardware interrupt since we're not ready to
1331 1331           * handle it yet.
1332 1332           */
1333 1333          MPTSAS_DISABLE_INTR(mpt);
1334 1334          if (mptsas_register_intrs(mpt) == FALSE)
1335 1335                  goto fail;
1336 1336          intr_added++;
1337 1337  
     1338 +        /*
     1339 +         * The mutex to protect task management during reset
     1340 +         */
     1341 +        mutex_init(&mpt->m_taskmgmt_mutex, NULL, MUTEX_SPIN,
     1342 +            DDI_INTR_PRI(mpt->m_intr_pri));
     1343 +
1338 1344          /* Initialize mutex used in interrupt handler */
1339 1345          mutex_init(&mpt->m_mutex, NULL, MUTEX_DRIVER,
1340 1346              DDI_INTR_PRI(mpt->m_intr_pri));
1341 1347          mutex_init(&mpt->m_passthru_mutex, NULL, MUTEX_DRIVER, NULL);
1342 1348          mutex_init(&mpt->m_tx_waitq_mutex, NULL, MUTEX_DRIVER,
1343 1349              DDI_INTR_PRI(mpt->m_intr_pri));
1344 1350          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
1345 1351                  mutex_init(&mpt->m_phy_info[i].smhba_info.phy_mutex,
1346 1352                      NULL, MUTEX_DRIVER,
1347 1353                      DDI_INTR_PRI(mpt->m_intr_pri));
↓ open down ↓ 274 lines elided ↑ open up ↑ 
1622 1628                  if (event_taskq_create) {
1623 1629                          ddi_taskq_destroy(mpt->m_event_taskq);
1624 1630                  }
1625 1631                  if (dr_taskq_create) {
1626 1632                          ddi_taskq_destroy(mpt->m_dr_taskq);
1627 1633                  }
1628 1634                  if (mutex_init_done) {
1629 1635                          mutex_destroy(&mpt->m_tx_waitq_mutex);
1630 1636                          mutex_destroy(&mpt->m_passthru_mutex);
1631 1637                          mutex_destroy(&mpt->m_mutex);
     1638 +                        mutex_destroy(&mpt->m_taskmgmt_mutex);
1632 1639                          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
1633 1640                                  mutex_destroy(
1634 1641                                      &mpt->m_phy_info[i].smhba_info.phy_mutex);
1635 1642                          }
1636 1643                          cv_destroy(&mpt->m_cv);
1637 1644                          cv_destroy(&mpt->m_passthru_cv);
1638 1645                          cv_destroy(&mpt->m_fw_cv);
1639 1646                          cv_destroy(&mpt->m_config_cv);
1640 1647                          cv_destroy(&mpt->m_fw_diag_cv);
1641 1648                          cv_destroy(&mpt->m_extreq_sense_refcount_cv);
↓ open down ↓ 399 lines elided ↑ open up ↑ 
2041 2048                  if (pm_lower_power(dip, 0, PM_LEVEL_D3) != DDI_SUCCESS)
2042 2049                          mptsas_log(mpt, CE_WARN,
2043 2050                              "!mptsas%d: Lower power request failed "
2044 2051                              "during detach, ignoring.",
2045 2052                              mpt->m_instance);
2046 2053          }
2047 2054  
2048 2055          mutex_destroy(&mpt->m_tx_waitq_mutex);
2049 2056          mutex_destroy(&mpt->m_passthru_mutex);
2050 2057          mutex_destroy(&mpt->m_mutex);
     2058 +        mutex_destroy(&mpt->m_taskmgmt_mutex);
2051 2059          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
2052 2060                  mutex_destroy(&mpt->m_phy_info[i].smhba_info.phy_mutex);
2053 2061          }
2054 2062          cv_destroy(&mpt->m_cv);
2055 2063          cv_destroy(&mpt->m_passthru_cv);
2056 2064          cv_destroy(&mpt->m_fw_cv);
2057 2065          cv_destroy(&mpt->m_config_cv);
2058 2066          cv_destroy(&mpt->m_fw_diag_cv);
2059 2067          cv_destroy(&mpt->m_extreq_sense_refcount_cv);
2060 2068
↓ open down ↓ 341 lines elided ↑ open up ↑ 
2402 2410                                  break;
2403 2411                          }
2404 2412                          delay(drv_usectohz(10000));
2405 2413                  }
2406 2414                  /*
2407 2415                   * If IOC is not in operational state, try to hard reset it.
2408 2416                   */
2409 2417                  if ((ioc_status & MPI2_IOC_STATE_MASK) !=
2410 2418                      MPI2_IOC_STATE_OPERATIONAL) {
2411 2419                          mpt->m_softstate &= ~MPTSAS_SS_MSG_UNIT_RESET;
2412      -                        if (mptsas_restart_ioc(mpt) == DDI_FAILURE) {
     2420 +                        if (mptsas_reset_handler(mpt) == DDI_FAILURE) {
2413 2421                                  mptsas_log(mpt, CE_WARN,
2414 2422                                      "mptsas_power: hard reset failed");
2415 2423                                  mutex_exit(&mpt->m_mutex);
2416 2424                                  return (DDI_FAILURE);
2417 2425                          }
2418 2426                  }
2419 2427                  mpt->m_power_level = PM_LEVEL_D0;
2420 2428                  break;
2421 2429          case PM_LEVEL_D3:
2422 2430                  NDBG11(("mptsas%d: turning power OFF.", mpt->m_instance));
↓ open down ↓ 996 lines elided ↑ open up ↑ 
3419 3427                  mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
3420 3428          }
3421 3429  
3422 3430          /*
3423 3431           * If HBA is being reset, the DevHandles are being re-initialized,
3424 3432           * which means that they could be invalid even if the target is still
3425 3433           * attached.  Check if being reset and if DevHandle is being
3426 3434           * re-initialized.  If this is the case, return BUSY so the I/O can be
3427 3435           * retried later.
3428 3436           */
     3437 +        mutex_enter(&mpt->m_taskmgmt_mutex);
3429 3438          if ((ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) && mpt->m_in_reset) {
3430 3439                  mptsas_set_pkt_reason(mpt, cmd, CMD_RESET, STAT_BUS_RESET);
3431 3440                  if (cmd->cmd_flags & CFLAG_TXQ) {
3432 3441                          mptsas_doneq_add(mpt, cmd);
3433 3442                          mptsas_doneq_empty(mpt);
     3443 +                        mutex_exit(&mpt->m_taskmgmt_mutex);
3434 3444                          return (rval);
3435 3445                  } else {
     3446 +                        mutex_exit(&mpt->m_taskmgmt_mutex);
3436 3447                          return (TRAN_BUSY);
3437 3448                  }
3438 3449          }
     3450 +        mutex_exit(&mpt->m_taskmgmt_mutex);
3439 3451  
3440 3452          /*
3441 3453           * If device handle has already been invalidated, just
3442 3454           * fail the command. In theory, command from scsi_vhci
3443 3455           * client is impossible send down command with invalid
3444 3456           * devhdl since devhdl is set after path offline, target
3445 3457           * driver is not suppose to select a offlined path.
3446 3458           */
3447 3459          if (ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) {
3448 3460                  NDBG3(("rejecting command, it might because invalid devhdl "
↓ open down ↓ 235 lines elided ↑ open up ↑ 
3684 3696                          int failure;
3685 3697  
3686 3698                          /*
3687 3699                           * We are going to allocate external packet space which
3688 3700                           * might include the sense data buffer for DMA so we
3689 3701                           * need to increase the reference counter here.  In a
3690 3702                           * case the HBA is in reset we just simply free the
3691 3703                           * allocated packet and bail out.
3692 3704                           */
3693 3705                          mutex_enter(&mpt->m_mutex);
3694      -                        if (mpt->m_in_reset) {
     3706 +                        mutex_enter(&mpt->m_taskmgmt_mutex);
     3707 +                        if (mpt->m_in_reset == TRUE) {
     3708 +                                mutex_exit(&mpt->m_taskmgmt_mutex);
3695 3709                                  mutex_exit(&mpt->m_mutex);
3696 3710  
3697 3711                                  cmd->cmd_flags = CFLAG_FREE;
3698 3712                                  kmem_cache_free(mpt->m_kmem_cache, cmd);
3699 3713                                  return (NULL);
3700 3714                          }
     3715 +                        mutex_exit(&mpt->m_taskmgmt_mutex);
3701 3716                          mpt->m_extreq_sense_refcount++;
3702 3717                          ASSERT(mpt->m_extreq_sense_refcount > 0);
3703 3718                          mutex_exit(&mpt->m_mutex);
3704 3719  
3705 3720                          /*
3706 3721                           * if extern alloc fails, all will be
3707 3722                           * deallocated, including cmd
3708 3723                           */
3709 3724                          failure = mptsas_pkt_alloc_extern(mpt, cmd,
3710 3725                              cmdlen, tgtlen, statuslen, kf);
↓ open down ↓ 1654 lines elided ↑ open up ↑ 
5365 5380                   */
5366 5381                  if (mptsas_handle_event_sync(args) == DDI_SUCCESS) {
5367 5382                          /*
5368 5383                           * Would not return main process,
5369 5384                           * just let taskq resolve ack action
5370 5385                           * and ack would be sent in taskq thread
5371 5386                           */
5372 5387                          NDBG20(("send mptsas_handle_event_sync success"));
5373 5388                  }
5374 5389  
5375      -                if (mpt->m_in_reset) {
     5390 +                mutex_enter(&mpt->m_taskmgmt_mutex);
     5391 +                if (mpt->m_in_reset == TRUE) {
5376 5392                          NDBG20(("dropping event received during reset"));
     5393 +                        mutex_exit(&mpt->m_taskmgmt_mutex);
5377 5394                          return;
5378 5395                  }
     5396 +                mutex_exit(&mpt->m_taskmgmt_mutex);
5379 5397  
5380 5398                  if ((ddi_taskq_dispatch(mpt->m_event_taskq, mptsas_handle_event,
5381 5399                      (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
5382 5400                          mptsas_log(mpt, CE_WARN, "No memory available"
5383 5401                          "for dispatch taskq");
5384 5402                          /*
5385 5403                           * Return the reply frame to the free queue.
5386 5404                           */
5387 5405                          ddi_put32(mpt->m_acc_free_queue_hdl,
5388 5406                              &((uint32_t *)(void *)
↓ open down ↓ 946 lines elided ↑ open up ↑ 
6335 6353  
6336 6354                  }
6337 6355                  ASSERT(parent);
6338 6356  handle_topo_change:
6339 6357  
6340 6358                  mutex_enter(&mpt->m_mutex);
6341 6359                  /*
6342 6360                   * If HBA is being reset, don't perform operations depending
6343 6361                   * on the IOC. We must free the topo list, however.
6344 6362                   */
6345      -                if (!mpt->m_in_reset)
     6363 +
     6364 +                mutex_enter(&mpt->m_taskmgmt_mutex);
     6365 +                if (mpt->m_in_reset == FALSE)
6346 6366                          mptsas_handle_topo_change(topo_node, parent);
6347      -                else
6348      -                        NDBG20(("skipping topo change received during reset"));
     6367 +                mutex_exit(&mpt->m_taskmgmt_mutex);
     6368 +
6349 6369                  save_node = topo_node;
6350 6370                  topo_node = topo_node->next;
6351 6371                  ASSERT(save_node);
6352 6372                  kmem_free(save_node, sizeof (mptsas_topo_change_list_t));
6353 6373                  mutex_exit(&mpt->m_mutex);
6354 6374  
6355 6375                  if ((flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) ||
6356 6376                      (flags == MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED) ||
6357 6377                      (flags == MPTSAS_TOPO_FLAG_RAID_ASSOCIATED)) {
6358 6378                          /*
↓ open down ↓ 1248 lines elided ↑ open up ↑ 
7607 7627          uint_t                          iocstatus;
7608 7628  
7609 7629          replyh_arg = (m_replyh_arg_t *)args;
7610 7630          rfm = replyh_arg->rfm;
7611 7631          mpt = replyh_arg->mpt;
7612 7632  
7613 7633          mutex_enter(&mpt->m_mutex);
7614 7634          /*
7615 7635           * If HBA is being reset, drop incoming event.
7616 7636           */
7617      -        if (mpt->m_in_reset) {
     7637 +        mutex_enter(&mpt->m_taskmgmt_mutex);
     7638 +        if (mpt->m_in_reset == TRUE) {
7618 7639                  NDBG20(("dropping event received prior to reset"));
     7640 +                mutex_exit(&mpt->m_taskmgmt_mutex);
7619 7641                  mutex_exit(&mpt->m_mutex);
7620 7642                  return;
7621 7643          }
     7644 +        mutex_exit(&mpt->m_taskmgmt_mutex);
7622 7645  
7623 7646          eventreply = (pMpi2EventNotificationReply_t)
7624 7647              (mpt->m_reply_frame + (rfm -
7625 7648              (mpt->m_reply_frame_dma_addr & 0xffffffffu)));
7626 7649          event = ddi_get16(mpt->m_acc_reply_frame_hdl, &eventreply->Event);
7627 7650  
7628 7651          if (iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
7629 7652              &eventreply->IOCStatus)) {
7630 7653                  if (iocstatus == MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
7631 7654                          mptsas_log(mpt, CE_WARN,
↓ open down ↓ 2283 lines elided ↑ open up ↑ 
9915 9938  
9916 9939                  /*
9917 9940                   * Check if controller is in a FAULT state. If so, reset it.
9918 9941                   */
9919 9942                  doorbell = ddi_get32(mpt->m_datap, &mpt->m_reg->Doorbell);
9920 9943                  if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
9921 9944                          doorbell &= MPI2_DOORBELL_DATA_MASK;
9922 9945                          mptsas_log(mpt, CE_WARN, "MPT Firmware Fault, "
9923 9946                              "code: %04x", doorbell);
9924 9947                          mpt->m_softstate &= ~MPTSAS_SS_MSG_UNIT_RESET;
9925      -                        if ((mptsas_restart_ioc(mpt)) == DDI_FAILURE) {
     9948 +                        if ((mptsas_reset_handler(mpt)) == DDI_FAILURE) {
9926 9949                                  mptsas_log(mpt, CE_WARN, "Reset failed"
9927 9950                                      "after fault was detected");
9928 9951                          }
9929 9952                  }
9930 9953  
9931 9954                  /*
9932 9955                   * For now, always call mptsas_watchsubr.
9933 9956                   */
9934 9957                  mptsas_watchsubr(mpt);
9935 9958
↓ open down ↓ 1236 lines elided ↑ open up ↑ 
11172 11195          if (pt_flags & MPTSAS_DATAOUT_ALLOCATED) {
11173 11196                  if (mptsas_check_dma_handle(dataout_dma_state.handle) !=
11174 11197                      DDI_SUCCESS) {
11175 11198                          ddi_fm_service_impact(mpt->m_dip,
11176 11199                              DDI_SERVICE_UNAFFECTED);
11177 11200                          status = EFAULT;
11178 11201                  }
11179 11202                  mptsas_dma_free(&dataout_dma_state);
11180 11203          }
11181 11204          if (pt_flags & MPTSAS_CMD_TIMEOUT) {
11182      -                if ((mptsas_restart_ioc(mpt)) == DDI_FAILURE) {
11183      -                        mptsas_log(mpt, CE_WARN, "mptsas_restart_ioc failed");
     11205 +                if ((mptsas_reset_handler(mpt)) == DDI_FAILURE) {
     11206 +                        mptsas_log(mpt, CE_WARN, "mptsas_reset_handler failed");
11184 11207                  }
11185 11208          }
11186 11209          if (request_msg)
11187 11210                  kmem_free(request_msg, request_size);
11188 11211          NDBG27(("mptsas_do_passthru: Done status 0x%x", status));
11189 11212  
11190 11213          return (status);
11191 11214  }
11192 11215  
11193 11216  static int
↓ open down ↓ 1484 lines elided ↑ open up ↑ 
12678 12701                              (caddr_t)(long)flashdata.PtrBuffer,
12679 12702                              flashdata.ImageSize, flashdata.ImageType, mode)) {
12680 12703                                  status = EFAULT;
12681 12704                          }
12682 12705  
12683 12706                          /*
12684 12707                           * Reset the chip to start using the new
12685 12708                           * firmware.  Reset if failed also.
12686 12709                           */
12687 12710                          mpt->m_softstate &= ~MPTSAS_SS_MSG_UNIT_RESET;
12688      -                        if (mptsas_restart_ioc(mpt) == DDI_FAILURE) {
     12711 +                        if (mptsas_reset_handler(mpt) == DDI_FAILURE) {
12689 12712                                  status = EFAULT;
12690 12713                          }
12691 12714                          mutex_exit(&mpt->m_mutex);
12692 12715                          break;
12693 12716                  case MPTIOCTL_PASS_THRU:
12694 12717                          /*
12695 12718                           * The user has requested to pass through a command to
12696 12719                           * be executed by the MPT firmware.  Call our routine
12697 12720                           * which does this.  Only allow one passthru IOCTL at
12698 12721                           * one time. Other threads will block on
↓ open down ↓ 51 lines elided ↑ open up ↑ 
12750 12773                          mptsas_read_pci_info(mpt, &pci_info);
12751 12774                          mutex_exit(&mpt->m_mutex);
12752 12775                          if (ddi_copyout((void *)(&pci_info), (void *)data,
12753 12776                              sizeof (mptsas_pci_info_t), mode) != 0) {
12754 12777                                  status = EFAULT;
12755 12778                          }
12756 12779                          break;
12757 12780                  case MPTIOCTL_RESET_ADAPTER:
12758 12781                          mutex_enter(&mpt->m_mutex);
12759 12782                          mpt->m_softstate &= ~MPTSAS_SS_MSG_UNIT_RESET;
12760      -                        if ((mptsas_restart_ioc(mpt)) == DDI_FAILURE) {
     12783 +                        if ((mptsas_reset_handler(mpt)) == DDI_FAILURE) {
12761 12784                                  mptsas_log(mpt, CE_WARN, "reset adapter IOCTL "
12762 12785                                      "failed");
12763 12786                                  status = EFAULT;
12764 12787                          }
12765 12788                          mutex_exit(&mpt->m_mutex);
12766 12789                          break;
12767 12790                  case MPTIOCTL_DIAG_ACTION:
12768 12791                          /*
12769 12792                           * The user has done a diag buffer action.  Call our
12770 12793                           * routine which does this.  Only allow one diag action
↓ open down ↓ 46 lines elided ↑ open up ↑ 
12817 12840                          status = scsi_hba_ioctl(dev, cmd, data, mode, credp,
12818 12841                              rval);
12819 12842                          break;
12820 12843          }
12821 12844  
12822 12845  out:
12823 12846          return (status);
12824 12847  }
12825 12848  
12826 12849  int
12827      -mptsas_restart_ioc(mptsas_t *mpt)
     12850 +mptsas_reset_handler(mptsas_t *mpt)
12828 12851  {
12829 12852          int             rval = DDI_SUCCESS;
12830 12853          mptsas_target_t *ptgt = NULL;
12831 12854  
12832 12855          ASSERT(mutex_owned(&mpt->m_mutex));
12833 12856  
12834 12857          /*
12835      -         * Set a flag telling I/O path that we're processing a reset.  This is
12836      -         * needed because after the reset is complete, the hash table still
     12858 +         * Set a flag telling task management we are processing a reset.  This
     12859 +         * is needed because after the reset is complete, the hash table still
12837 12860           * needs to be rebuilt.  If I/Os are started before the hash table is
12838 12861           * rebuilt, I/O errors will occur.  This flag allows I/Os to be marked
12839 12862           * so that they can be retried.
12840 12863           */
     12864 +        mutex_enter(&mpt->m_taskmgmt_mutex);
     12865 +        if (mpt->m_in_reset == TRUE) {
     12866 +                mutex_exit(&mpt->m_taskmgmt_mutex);
     12867 +                return (DDI_FAILURE);
     12868 +        }
12841 12869          mpt->m_in_reset = TRUE;
     12870 +        mutex_exit(&mpt->m_taskmgmt_mutex);
12842 12871  
12843 12872          /*
12844 12873           * Wait until all the allocated sense data buffers for DMA are freed.
12845 12874           */
12846 12875          while (mpt->m_extreq_sense_refcount > 0)
12847 12876                  cv_wait(&mpt->m_extreq_sense_refcount_cv, &mpt->m_mutex);
12848 12877  
12849 12878          /*
12850 12879           * Set all throttles to HOLD
12851 12880           */
↓ open down ↓ 44 lines elided ↑ open up ↑ 
12896 12925          mptsas_restart_hba(mpt);
12897 12926  
12898 12927          if (rval != DDI_SUCCESS) {
12899 12928                  mptsas_fm_ereport(mpt, DDI_FM_DEVICE_NO_RESPONSE);
12900 12929                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_LOST);
12901 12930          }
12902 12931  
12903 12932          /*
12904 12933           * Clear the reset flag so that I/Os can continue.
12905 12934           */
     12935 +        mutex_enter(&mpt->m_taskmgmt_mutex);
12906 12936          mpt->m_in_reset = FALSE;
     12937 +        mutex_exit(&mpt->m_taskmgmt_mutex);
12907 12938  
12908 12939          return (rval);
12909 12940  }
12910 12941  
12911 12942  static int
12912 12943  mptsas_init_chip(mptsas_t *mpt, int first_time)
12913 12944  {
12914 12945          ddi_dma_cookie_t        cookie;
12915 12946          uint32_t                i;
12916 12947          int                     rval;
↓ open down ↓ 3702 lines elided ↑ open up ↑ 
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX