Print this page
Code reconciliation with other base.
hg changesets 607a5b46a793..b706c96317c3
Fix ncpus for early boot config
Purge the ack to the interrupt before exiting mptsas_intr()
Changes from code review
Optimise slot alocation through rotor and the use
of per target mutex in start path.
Update tx waitq's code.
Create 2 threads, divide the workflow and deliver
to the hardware from the threads.
Optimise mutex's and code paths.
Split out offline target code.
Test timeouts code
Add support for more than 8 MSI-X interrupts.
Tidy up interrupt assignement and card ID messages.
Enable Fast Path for capable devices.
Merge fixes for Illumos issue 4819, fix mpt_sas command timeout handling.
Tweeks debug flags.
Default to process done commands all in threads if only 1 interrupt.
Lint and cstyle fixes.
Fix problem with running against 64bit msgaddr attributes for DMA.
Default is now to run like this.
Don't take tx_waiq_mutex if draining isn't set.
Fixes for Illumos issue 4682.
Fix hang bug to do with tx_wq.
Re-arrange mptsas_poll() to disable interrupts before issuing the
command.
Improve the tx_waitq code path.
Major rework of mutexes.
During normal operation do not grab m_mutex during interrupt.
Use reply post queues instead.
Make a few variable non static so you can change in /etc/system.
Fixes to some address arithmetic using 32bit values.
Distribute command done processing around the threads.
Improved auto-request sense memory usage.
Fix for Nexenta commit 36c74113a21
OS-91 mptsas does inquiry without setting pkt_time
Add comment about testing.
Test firmware version of 2008 controllers for MSI-X Compatibility.
Re-arrange mptsas_intr() to reduce number of spurious interrupts.
Fix bug in mptsas_free_post_queue().
Change mptsas_doneq_mv() to not loop.
Should not need m_in_callback flag. It prevents concurrent
command completion processing.
Added code to support using MSI-X interrupts across multiple
reply queues. Not tested with anything other than 3008 yet.
Change output "mptsas%d" -> "mptsas3%d".
Add SAS3 specific messages (12.0Gb).
Allow over-ride for interrupt type.
Restrict pre MPI2.5 to MSI interrupts.
Allow watchdog timout to work for mptsas_smhba_setup() in attach().
Merge fixes for "4403 mpt_sas panic when pulling a drive", commit f7d0d869a9ae78d
Use MSI-X interrupts, just one for now.
Pre-allocate array for request sense buffers, similar to command frames.
No more messing about with scsi_alloc_consistent_buf().
Add rolling buffer for *all* debug messages.
Improve mdb module and seperate out into mpt_sas3.
Initial modifications using the code changes present between
the LSI source code for FreeBSD drivers. Specifically the changes
between from mpslsi-source-17.00.00.00 -> mpslsi-source-03.00.00.00.
This mainly involves using a different scatter/gather element in
frame setup.
Change some obvious references sas -> sas3.
Changes to enable driver to compile.
Header paths, object lists, etc.

Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/io/scsi/adapters/mpt_sas3/mptsas3.c
          +++ new/usr/src/uts/common/io/scsi/adapters/mpt_sas3/mptsas3.c
↓ open down ↓ 13 lines elided ↑ open up ↑
  14   14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15   15   * If applicable, add the following below this CDDL HEADER, with the
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  
  22   22  /*
  23   23   * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  24      - * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
       24 + * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
  25   25   * Copyright (c) 2014, Joyent, Inc. All rights reserved.
       26 + * Copyright (c) 2014, Tegile Systems Inc. All rights reserved.
       27 + * Copyright 2014 OmniTI Computer Consulting, Inc. All rights reserved.
  26   28   */
  27   29  
  28   30  /*
  29   31   * Copyright (c) 2000 to 2010, LSI Corporation.
  30   32   * All rights reserved.
  31   33   *
  32   34   * Redistribution and use in source and binary forms of all code within
  33   35   * this file that is exclusively owned by LSI, with or without
  34   36   * modification, is permitted provided that, in addition to the CDDL 1.0
  35   37   * License requirements, the following conditions are met:
↓ open down ↓ 10 lines elided ↑ open up ↑
  46   48   * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  47   49   * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  48   50   * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  49   51   * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  50   52   * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  51   53   * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  52   54   * DAMAGE.
  53   55   */
  54   56  
  55   57  /*
  56      - * mptsas - This is a driver based on LSI Logic's MPT2.0 interface.
       58 + * mptsas3 - This is a driver based on LSI Logic's MPT2.0/2.5 interface.
  57   59   *
  58   60   */
  59   61  
  60   62  #if defined(lint) || defined(DEBUG)
  61   63  #define MPTSAS_DEBUG
  62   64  #endif
  63   65  
  64   66  /*
  65   67   * standard header files.
  66   68   */
↓ open down ↓ 4 lines elided ↑ open up ↑
  71   73  #include <sys/policy.h>
  72   74  #include <sys/model.h>
  73   75  #include <sys/sysevent.h>
  74   76  #include <sys/sysevent/eventdefs.h>
  75   77  #include <sys/sysevent/dr.h>
  76   78  #include <sys/sata/sata_defs.h>
  77   79  #include <sys/scsi/generic/sas.h>
  78   80  #include <sys/scsi/impl/scsi_sas.h>
  79   81  
  80   82  #pragma pack(1)
  81      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_type.h>
  82      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2.h>
  83      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_cnfg.h>
  84      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_init.h>
  85      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_ioc.h>
  86      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_sas.h>
  87      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_tool.h>
  88      -#include <sys/scsi/adapters/mpt_sas/mpi/mpi2_raid.h>
       83 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_type.h>
       84 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2.h>
       85 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_cnfg.h>
       86 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_init.h>
       87 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_ioc.h>
       88 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_sas.h>
       89 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_tool.h>
       90 +#include <sys/scsi/adapters/mpt_sas3/mpi/mpi2_raid.h>
  89   91  #pragma pack()
  90   92  
  91   93  /*
  92   94   * private header files.
  93   95   *
  94   96   */
  95   97  #include <sys/scsi/impl/scsi_reset_notify.h>
  96      -#include <sys/scsi/adapters/mpt_sas/mptsas_var.h>
  97      -#include <sys/scsi/adapters/mpt_sas/mptsas_ioctl.h>
  98      -#include <sys/scsi/adapters/mpt_sas/mptsas_smhba.h>
  99      -#include <sys/scsi/adapters/mpt_sas/mptsas_hash.h>
       98 +#include <sys/scsi/adapters/mpt_sas3/mptsas3_var.h>
       99 +#include <sys/scsi/adapters/mpt_sas3/mptsas3_ioctl.h>
      100 +#include <sys/scsi/adapters/mpt_sas3/mptsas3_smhba.h>
      101 +#include <sys/scsi/adapters/mpt_sas3/mptsas3_hash.h>
 100  102  #include <sys/raidioctl.h>
 101  103  
 102  104  #include <sys/fs/dv_node.h>     /* devfs_clean */
 103  105  
 104  106  /*
 105  107   * FMA header files
 106  108   */
 107  109  #include <sys/ddifm.h>
 108  110  #include <sys/fm/protocol.h>
 109  111  #include <sys/fm/util.h>
↓ open down ↓ 27 lines elided ↑ open up ↑
 137  139  static int mptsas_hba_setup(mptsas_t *mpt);
 138  140  static void mptsas_hba_teardown(mptsas_t *mpt);
 139  141  static int mptsas_config_space_init(mptsas_t *mpt);
 140  142  static void mptsas_config_space_fini(mptsas_t *mpt);
 141  143  static void mptsas_iport_register(mptsas_t *mpt);
 142  144  static int mptsas_smp_setup(mptsas_t *mpt);
 143  145  static void mptsas_smp_teardown(mptsas_t *mpt);
 144  146  static int mptsas_cache_create(mptsas_t *mpt);
 145  147  static void mptsas_cache_destroy(mptsas_t *mpt);
 146  148  static int mptsas_alloc_request_frames(mptsas_t *mpt);
      149 +static int mptsas_alloc_sense_bufs(mptsas_t *mpt);
 147  150  static int mptsas_alloc_reply_frames(mptsas_t *mpt);
 148  151  static int mptsas_alloc_free_queue(mptsas_t *mpt);
 149  152  static int mptsas_alloc_post_queue(mptsas_t *mpt);
      153 +static void mptsas_free_post_queue(mptsas_t *mpt);
 150  154  static void mptsas_alloc_reply_args(mptsas_t *mpt);
 151  155  static int mptsas_alloc_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd);
 152  156  static void mptsas_free_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd);
 153  157  static int mptsas_init_chip(mptsas_t *mpt, int first_time);
 154  158  
 155  159  /*
 156  160   * SCSA function prototypes
 157  161   */
 158  162  static int mptsas_scsi_start(struct scsi_address *ap, struct scsi_pkt *pkt);
 159  163  static int mptsas_scsi_reset(struct scsi_address *ap, int level);
↓ open down ↓ 34 lines elided ↑ open up ↑
 194  198  
 195  199  static int mptsas_quiesce_bus(mptsas_t *mpt);
 196  200  static int mptsas_unquiesce_bus(mptsas_t *mpt);
 197  201  
 198  202  static int mptsas_alloc_handshake_msg(mptsas_t *mpt, size_t alloc_size);
 199  203  static void mptsas_free_handshake_msg(mptsas_t *mpt);
 200  204  
 201  205  static void mptsas_ncmds_checkdrain(void *arg);
 202  206  
 203  207  static int mptsas_prepare_pkt(mptsas_cmd_t *cmd);
 204      -static int mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *sp);
 205      -static int mptsas_accept_txwq_and_pkt(mptsas_t *mpt, mptsas_cmd_t *sp);
 206      -static void mptsas_accept_tx_waitq(mptsas_t *mpt);
      208 +static void mptsas_retry_pkt(mptsas_t *mpt, mptsas_cmd_t *sp);
      209 +static int mptsas_save_cmd_to_slot(mptsas_t *mpt, mptsas_cmd_t *cmd);
      210 +static int mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *sp,
      211 +    int *tran_rval);
      212 +static void mptsas_accept_tx_waitqs(mptsas_t *mpt);
      213 +static void mptsas_unblock_tx_waitqs(mptsas_t *mpt);
      214 +static void mptsas_drain_tx_waitq(mptsas_t *mpt, mptsas_tx_waitqueue_t *txwq);
      215 +static int mptsas_check_targ_intxtion(mptsas_target_t *ptgt, int cmd_pkt_flags);
 207  216  
 208  217  static int mptsas_do_detach(dev_info_t *dev);
 209  218  static int mptsas_do_scsi_reset(mptsas_t *mpt, uint16_t devhdl);
 210  219  static int mptsas_do_scsi_abort(mptsas_t *mpt, int target, int lun,
 211  220      struct scsi_pkt *pkt);
 212  221  static int mptsas_scsi_capchk(char *cap, int tgtonly, int *cidxp);
 213  222  
 214  223  static void mptsas_handle_qfull(mptsas_t *mpt, mptsas_cmd_t *cmd);
 215  224  static void mptsas_handle_event(void *args);
 216  225  static int mptsas_handle_event_sync(void *args);
↓ open down ↓ 3 lines elided ↑ open up ↑
 220  229  
 221  230  static void mptsas_restart_cmd(void *);
 222  231  
 223  232  static void mptsas_flush_hba(mptsas_t *mpt);
 224  233  static void mptsas_flush_target(mptsas_t *mpt, ushort_t target, int lun,
 225  234          uint8_t tasktype);
 226  235  static void mptsas_set_pkt_reason(mptsas_t *mpt, mptsas_cmd_t *cmd,
 227  236      uchar_t reason, uint_t stat);
 228  237  
 229  238  static uint_t mptsas_intr(caddr_t arg1, caddr_t arg2);
 230      -static void mptsas_process_intr(mptsas_t *mpt,
      239 +static void mptsas_process_intr(mptsas_t *mpt, mptsas_reply_pqueue_t *rpqp,
 231  240      pMpi2ReplyDescriptorsUnion_t reply_desc_union);
 232  241  static void mptsas_handle_scsi_io_success(mptsas_t *mpt,
 233      -    pMpi2ReplyDescriptorsUnion_t reply_desc);
      242 +    mptsas_reply_pqueue_t *rpqp, pMpi2ReplyDescriptorsUnion_t reply_desc);
 234  243  static void mptsas_handle_address_reply(mptsas_t *mpt,
 235  244      pMpi2ReplyDescriptorsUnion_t reply_desc);
 236  245  static int mptsas_wait_intr(mptsas_t *mpt, int polltime);
 237  246  static void mptsas_sge_setup(mptsas_t *mpt, mptsas_cmd_t *cmd,
 238  247      uint32_t *control, pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl);
 239  248  
 240  249  static void mptsas_watch(void *arg);
 241  250  static void mptsas_watchsubr(mptsas_t *mpt);
 242      -static void mptsas_cmd_timeout(mptsas_t *mpt, uint16_t devhdl);
      251 +static void mptsas_cmd_timeout(mptsas_t *mpt, mptsas_target_t *ptgt);
 243  252  
 244  253  static void mptsas_start_passthru(mptsas_t *mpt, mptsas_cmd_t *cmd);
 245  254  static int mptsas_do_passthru(mptsas_t *mpt, uint8_t *request, uint8_t *reply,
 246  255      uint8_t *data, uint32_t request_size, uint32_t reply_size,
 247      -    uint32_t data_size, uint32_t direction, uint8_t *dataout,
      256 +    uint32_t data_size, uint8_t direction, uint8_t *dataout,
 248  257      uint32_t dataout_size, short timeout, int mode);
 249  258  static int mptsas_free_devhdl(mptsas_t *mpt, uint16_t devhdl);
 250  259  
 251  260  static uint8_t mptsas_get_fw_diag_buffer_number(mptsas_t *mpt,
 252  261      uint32_t unique_id);
 253  262  static void mptsas_start_diag(mptsas_t *mpt, mptsas_cmd_t *cmd);
 254  263  static int mptsas_post_fw_diag_buffer(mptsas_t *mpt,
 255  264      mptsas_fw_diagnostic_buffer_t *pBuffer, uint32_t *return_code);
 256  265  static int mptsas_release_fw_diag_buffer(mptsas_t *mpt,
 257  266      mptsas_fw_diagnostic_buffer_t *pBuffer, uint32_t *return_code,
↓ open down ↓ 34 lines elided ↑ open up ↑
 292  301      mptsas_target_t *ptgt, uchar_t *cdb, int cdblen, struct buf *data_bp,
 293  302      int *resid);
 294  303  
 295  304  static int mptsas_alloc_active_slots(mptsas_t *mpt, int flag);
 296  305  static void mptsas_free_active_slots(mptsas_t *mpt);
 297  306  static int mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd);
 298  307  
 299  308  static void mptsas_restart_hba(mptsas_t *mpt);
 300  309  static void mptsas_restart_waitq(mptsas_t *mpt);
 301  310  
 302      -static void mptsas_deliver_doneq_thread(mptsas_t *mpt);
      311 +static void mptsas_deliver_doneq_thread(mptsas_t *mpt,
      312 +    mptsas_done_list_t *dlist);
 303  313  static void mptsas_doneq_add(mptsas_t *mpt, mptsas_cmd_t *cmd);
 304      -static void mptsas_doneq_mv(mptsas_t *mpt, uint64_t t);
      314 +static void mptsas_rpdoneq_add(mptsas_t *mpt, mptsas_reply_pqueue_t *rpqp,
      315 +    mptsas_cmd_t *cmd);
      316 +static void mptsas_doneq_mv(mptsas_done_list_t *from,
      317 +    mptsas_doneq_thread_list_t *item);
 305  318  
 306  319  static mptsas_cmd_t *mptsas_doneq_thread_rm(mptsas_t *mpt, uint64_t t);
 307  320  static void mptsas_doneq_empty(mptsas_t *mpt);
 308      -static void mptsas_doneq_thread(mptsas_doneq_thread_arg_t *arg);
      321 +static void mptsas_rpdoneq_empty(mptsas_reply_pqueue_t *rpqp);
      322 +static void mptsas_doneq_thread(mptsas_thread_arg_t *arg);
      323 +static void mptsas_tx_waitq_thread(mptsas_thread_arg_t *arg);
 309  324  
 310  325  static mptsas_cmd_t *mptsas_waitq_rm(mptsas_t *mpt);
 311  326  static void mptsas_waitq_delete(mptsas_t *mpt, mptsas_cmd_t *cmd);
 312      -static mptsas_cmd_t *mptsas_tx_waitq_rm(mptsas_t *mpt);
 313      -static void mptsas_tx_waitq_delete(mptsas_t *mpt, mptsas_cmd_t *cmd);
 314      -
 315  327  
 316  328  static void mptsas_start_watch_reset_delay();
 317  329  static void mptsas_setup_bus_reset_delay(mptsas_t *mpt);
 318  330  static void mptsas_watch_reset_delay(void *arg);
 319  331  static int mptsas_watch_reset_delay_subr(mptsas_t *mpt);
      332 +static void mptsas_set_throttle(struct mptsas *mpt, mptsas_target_t *ptgt,
      333 +    int what);
      334 +static void mptsas_set_throttle_mtx(struct mptsas *mpt, mptsas_target_t *ptgt,
      335 +    int what);
      336 +static void mptsas_remove_cmd_nomtx(mptsas_t *mpt, mptsas_cmd_t *cmd);
 320  337  
 321  338  /*
 322  339   * helper functions
 323  340   */
 324  341  static void mptsas_dump_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd);
 325  342  
 326  343  static dev_info_t *mptsas_find_child(dev_info_t *pdip, char *name);
 327  344  static dev_info_t *mptsas_find_child_phy(dev_info_t *pdip, uint8_t phy);
 328  345  static dev_info_t *mptsas_find_child_addr(dev_info_t *pdip, uint64_t sasaddr,
 329  346      int lun);
↓ open down ↓ 32 lines elided ↑ open up ↑
 362  379  /*
 363  380   * Enumeration / DR functions
 364  381   */
 365  382  static void mptsas_config_all(dev_info_t *pdip);
 366  383  static int mptsas_config_one_addr(dev_info_t *pdip, uint64_t sasaddr, int lun,
 367  384      dev_info_t **lundip);
 368  385  static int mptsas_config_one_phy(dev_info_t *pdip, uint8_t phy, int lun,
 369  386      dev_info_t **lundip);
 370  387  
 371  388  static int mptsas_config_target(dev_info_t *pdip, mptsas_target_t *ptgt);
 372      -static int mptsas_offline_target(dev_info_t *pdip, char *name);
      389 +static int mptsas_offline_targetdev(dev_info_t *pdip, char *name);
      390 +static void mptsas_offline_target(mptsas_t *mpt, mptsas_target_t *ptgt,
      391 +    uint8_t topo_flags, dev_info_t *parent);
 373  392  
 374  393  static int mptsas_config_raid(dev_info_t *pdip, uint16_t target,
 375  394      dev_info_t **dip);
 376  395  
 377  396  static int mptsas_config_luns(dev_info_t *pdip, mptsas_target_t *ptgt);
 378  397  static int mptsas_probe_lun(dev_info_t *pdip, int lun,
 379  398      dev_info_t **dip, mptsas_target_t *ptgt);
 380  399  
 381  400  static int mptsas_create_lun(dev_info_t *pdip, struct scsi_inquiry *sd_inq,
 382  401      dev_info_t **dip, mptsas_target_t *ptgt, int lun);
↓ open down ↓ 35 lines elided ↑ open up ↑
 418  437   */
 419  438  static int mptsas_get_pci_cap(mptsas_t *mpt);
 420  439  static int mptsas_init_pm(mptsas_t *mpt);
 421  440  
 422  441  /*
 423  442   * MPT MSI tunable:
 424  443   *
 425  444   * By default MSI is enabled on all supported platforms.
 426  445   */
 427  446  boolean_t mptsas_enable_msi = B_TRUE;
      447 +boolean_t mptsas_enable_msix = B_TRUE;
 428  448  boolean_t mptsas_physical_bind_failed_page_83 = B_FALSE;
 429  449  
      450 +/*
      451 + * Global switch for use of MPI2.5 FAST PATH.
      452 + */
      453 +boolean_t mptsas3_use_fastpath = B_TRUE;
      454 +
 430  455  static int mptsas_register_intrs(mptsas_t *);
 431  456  static void mptsas_unregister_intrs(mptsas_t *);
 432  457  static int mptsas_add_intrs(mptsas_t *, int);
 433  458  static void mptsas_rem_intrs(mptsas_t *);
 434  459  
 435  460  /*
 436  461   * FMA Prototypes
 437  462   */
 438  463  static void mptsas_fm_init(mptsas_t *mpt);
 439  464  static void mptsas_fm_fini(mptsas_t *mpt);
 440  465  static int mptsas_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void *);
 441  466  
 442  467  extern pri_t minclsyspri, maxclsyspri;
      468 +/*
      469 + * NCPUS is used to determine some optimal configurations for number
      470 + * of threads created to perform specific jobs. If we are invoked because
      471 + * a disk is part of the root file system ncpus may still be 1 so check
      472 + * boot_ncpus as well.
      473 + */
      474 +extern int ncpus, boot_ncpus;
      475 +#define NCPUS   max(ncpus, boot_ncpus)
 443  476  
 444  477  /*
 445  478   * This device is created by the SCSI pseudo nexus driver (SCSI vHCI).  It is
 446  479   * under this device that the paths to a physical device are created when
 447  480   * MPxIO is used.
 448  481   */
 449  482  extern dev_info_t       *scsi_vhci_dip;
 450  483  
 451  484  /*
 452  485   * Tunable timeout value for Inquiry VPD page 0x83
 453  486   * By default the value is 30 seconds.
 454  487   */
 455  488  int mptsas_inq83_retry_timeout = 30;
 456  489  
 457  490  /*
      491 + * Tunable for default SCSI pkt timeout. Defaults to 5 seconds, which should
      492 + * be plenty for INQUIRY and REPORT_LUNS, which are the only commands currently
      493 + * issued by mptsas directly.
      494 + */
      495 +int mptsas_scsi_pkt_time = 5;
      496 +
      497 +/*
 458  498   * This is used to allocate memory for message frame storage, not for
 459  499   * data I/O DMA. All message frames must be stored in the first 4G of
 460  500   * physical memory.
 461  501   */
 462  502  ddi_dma_attr_t mptsas_dma_attrs = {
 463  503          DMA_ATTR_V0,    /* attribute layout version             */
 464  504          0x0ull,         /* address low - should be 0 (longlong) */
 465  505          0xffffffffull,  /* address high - 32-bit max range      */
 466  506          0x00ffffffull,  /* count max - max DMA object size      */
 467  507          4,              /* allocation alignment requirements    */
↓ open down ↓ 15 lines elided ↑ open up ↑
 483  523          0x0ull,         /* address low - should be 0 (longlong) */
 484  524          0xffffffffffffffffull,  /* address high - 64-bit max    */
 485  525          0x00ffffffull,  /* count max - max DMA object size      */
 486  526          4,              /* allocation alignment requirements    */
 487  527          0x78,           /* burstsizes - binary encoded values   */
 488  528          1,              /* minxfer - gran. of DMA engine        */
 489  529          0x00ffffffull,  /* maxxfer - gran. of DMA engine        */
 490  530          0xffffffffull,  /* max segment size (DMA boundary)      */
 491  531          MPTSAS_MAX_DMA_SEGS, /* scatter/gather list length      */
 492  532          512,            /* granularity - device transfer size   */
 493      -        DDI_DMA_RELAXED_ORDERING        /* flags, enable relaxed ordering */
      533 +        0               /* flags, set to 0 */
 494  534  };
 495  535  
 496  536  ddi_device_acc_attr_t mptsas_dev_attr = {
 497  537          DDI_DEVICE_ATTR_V1,
 498  538          DDI_STRUCTURE_LE_ACC,
 499  539          DDI_STRICTORDER_ACC,
 500  540          DDI_DEFAULT_ACC
 501  541  };
 502  542  
 503  543  static struct cb_ops mptsas_cb_ops = {
↓ open down ↓ 34 lines elided ↑ open up ↑
 538  578          NULL,                   /* bus operations */
 539  579          mptsas_power,           /* power management */
 540  580  #ifdef  __sparc
 541  581          ddi_quiesce_not_needed
 542  582  #else
 543  583          mptsas_quiesce          /* quiesce */
 544  584  #endif  /* __sparc */
 545  585  };
 546  586  
 547  587  
 548      -#define MPTSAS_MOD_STRING "MPTSAS HBA Driver 00.00.00.24"
      588 +#define MPTSAS_MOD_STRING "MPTSAS3 HBA Driver 00.00.01"
 549  589  
 550  590  static struct modldrv modldrv = {
 551  591          &mod_driverops, /* Type of module. This one is a driver */
 552  592          MPTSAS_MOD_STRING, /* Name of the module. */
 553  593          &mptsas_ops,    /* driver ops */
 554  594  };
 555  595  
 556  596  static struct modlinkage modlinkage = {
 557  597          MODREV_1, &modldrv, NULL
 558  598  };
↓ open down ↓ 2 lines elided ↑ open up ↑
 561  601  #define LUN64_PROP      "lun64"
 562  602  #define SAS_PROP        "sas-mpt"
 563  603  #define MDI_GUID        "wwn"
 564  604  #define NDI_GUID        "guid"
 565  605  #define MPTSAS_DEV_GONE "mptsas_dev_gone"
 566  606  
 567  607  /*
 568  608   * Local static data
 569  609   */
 570  610  #if defined(MPTSAS_DEBUG)
 571      -uint32_t mptsas_debug_flags = 0;
      611 +uint32_t mptsas_debug_flags = 0x0;
      612 +/*
      613 + * Flags to ignore these messages in local debug ring buffer.
      614 + * Default is to ignore the watchsubr() output which normally happens
      615 + * every second.
      616 + */
      617 +uint32_t mptsas_dbglog_imask = 0x40000000;
      618 +uint32_t mptsas_test_timeout = 0;
 572  619  #endif  /* defined(MPTSAS_DEBUG) */
 573  620  uint32_t mptsas_debug_resets = 0;
 574  621  
 575  622  static kmutex_t         mptsas_global_mutex;
 576      -static void             *mptsas_state;          /* soft state ptr */
      623 +static void             *mptsas3_state;         /* soft state ptr */
 577  624  static krwlock_t        mptsas_global_rwlock;
 578  625  
 579  626  static kmutex_t         mptsas_log_mutex;
 580  627  static char             mptsas_log_buf[256];
 581  628  _NOTE(MUTEX_PROTECTS_DATA(mptsas_log_mutex, mptsas_log_buf))
 582  629  
 583  630  static mptsas_t *mptsas_head, *mptsas_tail;
 584  631  static clock_t mptsas_scsi_watchdog_tick;
 585  632  static clock_t mptsas_tick;
 586  633  static timeout_id_t mptsas_reset_watch;
 587  634  static timeout_id_t mptsas_timeout_id;
 588  635  static int mptsas_timeouts_enabled = 0;
      636 +
      637 +/*
      638 + * Maximum number of MSI-X interrupts any instance of mptsas3 can use.
      639 + * Note that if you want to increase this you may have to also bump the
      640 + * value of ddi_msix_alloc_limit which defaults to 8.
      641 + * Set to zero to fall back to other interrupt types.
      642 + */
      643 +int mptsas3_max_msix_intrs = 8;
      644 +
      645 +/*
      646 + * Default length for extended auto request sense buffers.
      647 + * All sense buffers need to be under the same alloc because there
      648 + * is only one common top 32bits (of 64bits) address register.
      649 + * Most requests only require 32 bytes, but some request >256.
      650 + * We use rmalloc()/rmfree() on this additional memory to manage the
      651 + * "extended" requests.
      652 + */
      653 +int mptsas_extreq_sense_bufsize = 256*64;
      654 +
      655 +/*
      656 + * Believe that all software resrictions of having to run with DMA
      657 + * attributes to limit allocation to the first 4G are removed.
      658 + * However, this flag remains to enable quick switchback should suspicious
      659 + * problems emerge.
      660 + * Note that scsi_alloc_consistent_buf() does still adhering to allocating
      661 + * 32 bit addressable memory, but we can cope if that is changed now.
      662 + */
      663 +int mptsas_use_64bit_msgaddr = 1;
      664 +
 589  665  /*
 590  666   * warlock directives
 591  667   */
 592  668  _NOTE(SCHEME_PROTECTS_DATA("unique per pkt", scsi_pkt \
 593  669          mptsas_cmd NcrTableIndirect buf scsi_cdb scsi_status))
 594  670  _NOTE(SCHEME_PROTECTS_DATA("unique per pkt", smp_pkt))
 595  671  _NOTE(SCHEME_PROTECTS_DATA("stable data", scsi_device scsi_address))
 596  672  _NOTE(SCHEME_PROTECTS_DATA("No Mutex Needed", mptsas_tgt_private))
 597  673  _NOTE(SCHEME_PROTECTS_DATA("No Mutex Needed", scsi_hba_tran::tran_tgt_private))
 598  674  
↓ open down ↓ 13 lines elided ↑ open up ↑
 612  688   */
 613  689  int
 614  690  _init(void)
 615  691  {
 616  692          int status;
 617  693          /* CONSTCOND */
 618  694          ASSERT(NO_COMPETING_THREADS);
 619  695  
 620  696          NDBG0(("_init"));
 621  697  
 622      -        status = ddi_soft_state_init(&mptsas_state, MPTSAS_SIZE,
      698 +        status = ddi_soft_state_init(&mptsas3_state, MPTSAS_SIZE,
 623  699              MPTSAS_INITIAL_SOFT_SPACE);
 624  700          if (status != 0) {
 625  701                  return (status);
 626  702          }
 627  703  
 628  704          if ((status = scsi_hba_init(&modlinkage)) != 0) {
 629      -                ddi_soft_state_fini(&mptsas_state);
      705 +                ddi_soft_state_fini(&mptsas3_state);
 630  706                  return (status);
 631  707          }
 632  708  
 633  709          mutex_init(&mptsas_global_mutex, NULL, MUTEX_DRIVER, NULL);
 634  710          rw_init(&mptsas_global_rwlock, NULL, RW_DRIVER, NULL);
 635  711          mutex_init(&mptsas_log_mutex, NULL, MUTEX_DRIVER, NULL);
 636  712  
 637  713          if ((status = mod_install(&modlinkage)) != 0) {
 638  714                  mutex_destroy(&mptsas_log_mutex);
 639  715                  rw_destroy(&mptsas_global_rwlock);
 640  716                  mutex_destroy(&mptsas_global_mutex);
 641      -                ddi_soft_state_fini(&mptsas_state);
      717 +                ddi_soft_state_fini(&mptsas3_state);
 642  718                  scsi_hba_fini(&modlinkage);
 643  719          }
 644  720  
 645  721          return (status);
 646  722  }
 647  723  
 648  724  /*
 649  725   * Notes:
 650  726   *      - scsi_hba_fini(9F) uninitializes SCSI HBA modules
 651  727   */
 652  728  int
 653  729  _fini(void)
 654  730  {
 655  731          int     status;
 656  732          /* CONSTCOND */
 657  733          ASSERT(NO_COMPETING_THREADS);
 658  734  
 659  735          NDBG0(("_fini"));
 660  736  
 661  737          if ((status = mod_remove(&modlinkage)) == 0) {
 662      -                ddi_soft_state_fini(&mptsas_state);
      738 +                ddi_soft_state_fini(&mptsas3_state);
 663  739                  scsi_hba_fini(&modlinkage);
 664  740                  mutex_destroy(&mptsas_global_mutex);
 665  741                  rw_destroy(&mptsas_global_rwlock);
 666  742                  mutex_destroy(&mptsas_log_mutex);
 667  743          }
 668  744          return (status);
 669  745  }
 670  746  
 671  747  /*
 672  748   * The loadable-module _info(9E) entry point
↓ open down ↓ 85 lines elided ↑ open up ↑
 758  834  }
 759  835  
 760  836  static void
 761  837  mptsas_destroy_hashes(mptsas_t *mpt)
 762  838  {
 763  839          mptsas_target_t *tp;
 764  840          mptsas_smp_t *sp;
 765  841  
 766  842          for (tp = refhash_first(mpt->m_targets); tp != NULL;
 767  843              tp = refhash_next(mpt->m_targets, tp)) {
      844 +                mutex_destroy(&tp->m_t_mutex);
 768  845                  refhash_remove(mpt->m_targets, tp);
 769  846          }
 770  847          for (sp = refhash_first(mpt->m_smp_targets); sp != NULL;
 771  848              sp = refhash_next(mpt->m_smp_targets, sp)) {
 772  849                  refhash_remove(mpt->m_smp_targets, sp);
 773  850          }
 774  851          refhash_destroy(mpt->m_targets);
 775  852          refhash_destroy(mpt->m_smp_targets);
 776  853          mpt->m_targets = NULL;
 777  854          mpt->m_smp_targets = NULL;
↓ open down ↓ 14 lines elided ↑ open up ↑
 792  869          int                     rval = DDI_FAILURE;
 793  870          int                     i = 0;
 794  871          uint8_t                 numphys = 0;
 795  872          uint8_t                 phy_id;
 796  873          uint8_t                 phy_port = 0;
 797  874          uint16_t                attached_devhdl = 0;
 798  875          uint32_t                dev_info;
 799  876          uint64_t                attached_sas_wwn;
 800  877          uint16_t                dev_hdl;
 801  878          uint16_t                pdev_hdl;
 802      -        uint16_t                bay_num, enclosure;
      879 +        uint16_t                bay_num, enclosure, io_flags;
 803  880          char                    attached_wwnstr[MPTSAS_WWN_STRLEN];
 804  881  
 805  882          /* CONSTCOND */
 806  883          ASSERT(NO_COMPETING_THREADS);
 807  884  
 808  885          switch (cmd) {
 809  886          case DDI_ATTACH:
 810  887                  break;
 811  888  
 812  889          case DDI_RESUME:
↓ open down ↓ 141 lines elided ↑ open up ↑
 954 1031                  return (DDI_FAILURE);
 955 1032          }
 956 1033          mptsas_smhba_set_all_phy_props(mpt, dip, numphys, phy_mask,
 957 1034              &attached_devhdl);
 958 1035  
 959 1036          mutex_enter(&mpt->m_mutex);
 960 1037          page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
 961 1038              MPI2_SAS_DEVICE_PGAD_FORM_MASK) | (uint32_t)attached_devhdl;
 962 1039          rval = mptsas_get_sas_device_page0(mpt, page_address, &dev_hdl,
 963 1040              &attached_sas_wwn, &dev_info, &phy_port, &phy_id,
 964      -            &pdev_hdl, &bay_num, &enclosure);
     1041 +            &pdev_hdl, &bay_num, &enclosure, &io_flags);
 965 1042          if (rval != DDI_SUCCESS) {
 966 1043                  mptsas_log(mpt, CE_WARN,
 967 1044                      "Failed to get device page0 for handle:%d",
 968 1045                      attached_devhdl);
 969 1046                  mutex_exit(&mpt->m_mutex);
 970 1047                  return (DDI_FAILURE);
 971 1048          }
 972 1049  
 973 1050          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
 974 1051                  bzero(phymask, sizeof (phymask));
↓ open down ↓ 36 lines elided ↑ open up ↑
1011 1088   *      Set up all device state and allocate data structures,
1012 1089   *      mutexes, condition variables, etc. for device operation.
1013 1090   *      Add interrupts needed.
1014 1091   *      Return DDI_SUCCESS if device is ready, else return DDI_FAILURE.
1015 1092   */
1016 1093  static int
1017 1094  mptsas_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
1018 1095  {
1019 1096          mptsas_t                *mpt = NULL;
1020 1097          int                     instance, i, j;
1021      -        int                     doneq_thread_num;
1022      -        char                    intr_added = 0;
     1098 +        int                     q_thread_num;
1023 1099          char                    map_setup = 0;
1024 1100          char                    config_setup = 0;
1025 1101          char                    hba_attach_setup = 0;
1026 1102          char                    smp_attach_setup = 0;
1027 1103          char                    mutex_init_done = 0;
1028 1104          char                    event_taskq_create = 0;
1029 1105          char                    dr_taskq_create = 0;
1030 1106          char                    doneq_thread_create = 0;
     1107 +        char                    txwq_thread_create = 0;
     1108 +        char                    added_watchdog = 0;
1031 1109          scsi_hba_tran_t         *hba_tran;
1032 1110          uint_t                  mem_bar = MEM_SPACE;
1033 1111          int                     rval = DDI_FAILURE;
1034 1112  
1035 1113          /* CONSTCOND */
1036 1114          ASSERT(NO_COMPETING_THREADS);
1037 1115  
1038 1116          if (scsi_hba_iport_unit_address(dip)) {
1039 1117                  return (mptsas_iport_attach(dip, cmd));
1040 1118          }
↓ open down ↓ 94 lines elided ↑ open up ↑
1135 1213          default:
1136 1214                  return (DDI_FAILURE);
1137 1215  
1138 1216          }
1139 1217  
1140 1218          instance = ddi_get_instance(dip);
1141 1219  
1142 1220          /*
1143 1221           * Allocate softc information.
1144 1222           */
1145      -        if (ddi_soft_state_zalloc(mptsas_state, instance) != DDI_SUCCESS) {
     1223 +        if (ddi_soft_state_zalloc(mptsas3_state, instance) != DDI_SUCCESS) {
1146 1224                  mptsas_log(NULL, CE_WARN,
1147 1225                      "mptsas%d: cannot allocate soft state", instance);
1148 1226                  goto fail;
1149 1227          }
1150 1228  
1151      -        mpt = ddi_get_soft_state(mptsas_state, instance);
     1229 +        mpt = ddi_get_soft_state(mptsas3_state, instance);
1152 1230  
1153 1231          if (mpt == NULL) {
1154 1232                  mptsas_log(NULL, CE_WARN,
1155 1233                      "mptsas%d: cannot get soft state", instance);
1156 1234                  goto fail;
1157 1235          }
1158 1236  
1159 1237          /* Indicate that we are 'sizeof (scsi_*(9S))' clean. */
1160 1238          scsi_size_clean(dip);
1161 1239  
1162 1240          mpt->m_dip = dip;
1163 1241          mpt->m_instance = instance;
1164 1242  
1165 1243          /* Make a per-instance copy of the structures */
1166 1244          mpt->m_io_dma_attr = mptsas_dma_attrs64;
1167      -        mpt->m_msg_dma_attr = mptsas_dma_attrs;
     1245 +        if (mptsas_use_64bit_msgaddr) {
     1246 +                mpt->m_msg_dma_attr = mptsas_dma_attrs64;
     1247 +        } else {
     1248 +                mpt->m_msg_dma_attr = mptsas_dma_attrs;
     1249 +        }
1168 1250          mpt->m_reg_acc_attr = mptsas_dev_attr;
1169 1251          mpt->m_dev_acc_attr = mptsas_dev_attr;
1170 1252  
1171 1253          /*
     1254 +         * Round down the arq sense buffer size to nearest 16 bytes.
     1255 +         */
     1256 +        mpt->m_req_sense_size = EXTCMDS_STATUS_SIZE;
     1257 +
     1258 +        /*
1172 1259           * Initialize FMA
1173 1260           */
1174 1261          mpt->m_fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, mpt->m_dip,
1175 1262              DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
1176 1263              DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
1177 1264              DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
1178 1265  
1179 1266          mptsas_fm_init(mpt);
1180 1267  
1181 1268          if (mptsas_alloc_handshake_msg(mpt,
↓ open down ↓ 33 lines elided ↑ open up ↑
1215 1302           */
1216 1303          if ((mpt->m_dr_taskq = ddi_taskq_create(dip,
1217 1304              "mptsas_dr_taskq",
1218 1305              1, TASKQ_DEFAULTPRI, 0)) == NULL) {
1219 1306                  mptsas_log(mpt, CE_NOTE, "ddi_taskq_create for discovery "
1220 1307                      "failed");
1221 1308                  goto fail;
1222 1309          }
1223 1310          dr_taskq_create++;
1224 1311  
     1312 +        cv_init(&mpt->m_qthread_cv, NULL, CV_DRIVER, NULL);
     1313 +        mutex_init(&mpt->m_qthread_mutex, NULL, MUTEX_DRIVER, NULL);
     1314 +
     1315 +        i = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
     1316 +            0, "mptsas_enable_txwq_prop", NCPUS > 1);
     1317 +        if (i) {
     1318 +                mpt->m_txwq_thread_n = NUM_TX_WAITQ;
     1319 +                mpt->m_txwq_enabled = FALSE;
     1320 +                if (ddi_prop_get_int(DDI_DEV_T_ANY, dip,
     1321 +                    0, "mptsas_allow_txwq_jumping", 0)) {
     1322 +                        mpt->m_txwq_allow_q_jumping = TRUE;
     1323 +                }
     1324 +                i = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
     1325 +                    0, "mptsas_txwq_threashold_prop", 80000);
     1326 +                mpt->m_txwq_thread_threshold = (uint16_t)i;
     1327 +        } else {
     1328 +                mpt->m_txwq_thread_n = 0;
     1329 +                mpt->m_txwq_enabled = FALSE;
     1330 +        }
     1331 +
     1332 +        if (mpt->m_txwq_thread_n) {
     1333 +                mutex_enter(&mpt->m_qthread_mutex);
     1334 +                for (j = 0; j < NUM_TX_WAITQ; j++) {
     1335 +                        mutex_init(&mpt->m_tx_waitq[j].txwq_mutex, NULL,
     1336 +                            MUTEX_DRIVER,
     1337 +                            NULL);
     1338 +                        cv_init(&mpt->m_tx_waitq[j].txwq_cv, NULL, CV_DRIVER,
     1339 +                            NULL);
     1340 +                        cv_init(&mpt->m_tx_waitq[j].txwq_drain_cv, NULL,
     1341 +                            CV_DRIVER, NULL);
     1342 +                        mpt->m_tx_waitq[j].txwq_active = TRUE;
     1343 +                        mpt->m_tx_waitq[j].txwq_draining = FALSE;
     1344 +                        mpt->m_tx_waitq[j].txwq_cmdq = NULL;
     1345 +                        mpt->m_tx_waitq[j].txwq_qtail =
     1346 +                            &mpt->m_tx_waitq[j].txwq_cmdq;
     1347 +                        mutex_enter(&mpt->m_tx_waitq[j].txwq_mutex);
     1348 +                        mpt->m_tx_waitq[j].arg.mpt = mpt;
     1349 +                        mpt->m_tx_waitq[j].arg.t = j;
     1350 +                        mpt->m_tx_waitq[j].txwq_threadp =
     1351 +                            thread_create(NULL, 0, mptsas_tx_waitq_thread,
     1352 +                            &mpt->m_tx_waitq[j].arg,
     1353 +                            0, &p0, TS_RUN, maxclsyspri - 10);
     1354 +                        mutex_exit(&mpt->m_tx_waitq[j].txwq_mutex);
     1355 +                }
     1356 +                mutex_exit(&mpt->m_qthread_mutex);
     1357 +                txwq_thread_create++;
     1358 +        }
     1359 +
1225 1360          mpt->m_doneq_thread_threshold = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
1226 1361              0, "mptsas_doneq_thread_threshold_prop", 10);
1227 1362          mpt->m_doneq_length_threshold = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
1228 1363              0, "mptsas_doneq_length_threshold_prop", 8);
1229 1364          mpt->m_doneq_thread_n = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
1230      -            0, "mptsas_doneq_thread_n_prop", 8);
     1365 +            0, "mptsas_doneq_thread_n_prop", min(NCPUS, 8));
1231 1366  
1232 1367          if (mpt->m_doneq_thread_n) {
1233      -                cv_init(&mpt->m_doneq_thread_cv, NULL, CV_DRIVER, NULL);
1234      -                mutex_init(&mpt->m_doneq_mutex, NULL, MUTEX_DRIVER, NULL);
1235      -
1236      -                mutex_enter(&mpt->m_doneq_mutex);
     1368 +                mutex_enter(&mpt->m_qthread_mutex);
1237 1369                  mpt->m_doneq_thread_id =
1238 1370                      kmem_zalloc(sizeof (mptsas_doneq_thread_list_t)
1239 1371                      * mpt->m_doneq_thread_n, KM_SLEEP);
1240 1372  
1241 1373                  for (j = 0; j < mpt->m_doneq_thread_n; j++) {
1242 1374                          cv_init(&mpt->m_doneq_thread_id[j].cv, NULL,
1243 1375                              CV_DRIVER, NULL);
1244 1376                          mutex_init(&mpt->m_doneq_thread_id[j].mutex, NULL,
1245 1377                              MUTEX_DRIVER, NULL);
1246 1378                          mutex_enter(&mpt->m_doneq_thread_id[j].mutex);
1247 1379                          mpt->m_doneq_thread_id[j].flag |=
1248 1380                              MPTSAS_DONEQ_THREAD_ACTIVE;
1249 1381                          mpt->m_doneq_thread_id[j].arg.mpt = mpt;
1250 1382                          mpt->m_doneq_thread_id[j].arg.t = j;
1251 1383                          mpt->m_doneq_thread_id[j].threadp =
1252 1384                              thread_create(NULL, 0, mptsas_doneq_thread,
1253 1385                              &mpt->m_doneq_thread_id[j].arg,
1254      -                            0, &p0, TS_RUN, minclsyspri);
1255      -                        mpt->m_doneq_thread_id[j].donetail =
1256      -                            &mpt->m_doneq_thread_id[j].doneq;
     1386 +                            0, &p0, TS_RUN, maxclsyspri - 10);
     1387 +                        mpt->m_doneq_thread_id[j].dlist.dl_tail =
     1388 +                            &mpt->m_doneq_thread_id[j].dlist.dl_q;
1257 1389                          mutex_exit(&mpt->m_doneq_thread_id[j].mutex);
1258 1390                  }
1259      -                mutex_exit(&mpt->m_doneq_mutex);
     1391 +                mutex_exit(&mpt->m_qthread_mutex);
1260 1392                  doneq_thread_create++;
1261 1393          }
1262 1394  
1263      -        /* Initialize mutex used in interrupt handler */
1264      -        mutex_init(&mpt->m_mutex, NULL, MUTEX_DRIVER,
1265      -            DDI_INTR_PRI(mpt->m_intr_pri));
     1395 +        /*
     1396 +         * Disable hardware interrupt since we're not ready to
     1397 +         * handle it yet.
     1398 +         */
     1399 +        MPTSAS_DISABLE_INTR(mpt);
     1400 +
     1401 +        /*
     1402 +         * Initialize mutex used in interrupt handler.
     1403 +         * We don't support hi-level so the mutex's are all adaptive
     1404 +         * and we don't want to register the interrupts until we get
     1405 +         * the chip type information from _init_chip() below.
     1406 +         * Otherwise we would use DDI_INTR_PRI(mpt->m_intr_pri)
     1407 +         * rather than NULL in the mutex_init() calls.
     1408 +         */
     1409 +        mutex_init(&mpt->m_mutex, NULL, MUTEX_DRIVER, NULL);
1266 1410          mutex_init(&mpt->m_passthru_mutex, NULL, MUTEX_DRIVER, NULL);
1267      -        mutex_init(&mpt->m_tx_waitq_mutex, NULL, MUTEX_DRIVER,
1268      -            DDI_INTR_PRI(mpt->m_intr_pri));
1269 1411          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
1270 1412                  mutex_init(&mpt->m_phy_info[i].smhba_info.phy_mutex,
1271      -                    NULL, MUTEX_DRIVER,
1272      -                    DDI_INTR_PRI(mpt->m_intr_pri));
     1413 +                    NULL, MUTEX_DRIVER, NULL);
1273 1414          }
1274 1415  
1275 1416          cv_init(&mpt->m_cv, NULL, CV_DRIVER, NULL);
1276 1417          cv_init(&mpt->m_passthru_cv, NULL, CV_DRIVER, NULL);
1277 1418          cv_init(&mpt->m_fw_cv, NULL, CV_DRIVER, NULL);
1278 1419          cv_init(&mpt->m_config_cv, NULL, CV_DRIVER, NULL);
1279 1420          cv_init(&mpt->m_fw_diag_cv, NULL, CV_DRIVER, NULL);
1280 1421          mutex_init_done++;
1281 1422  
1282      -        /*
1283      -         * Disable hardware interrupt since we're not ready to
1284      -         * handle it yet.
1285      -         */
1286      -        MPTSAS_DISABLE_INTR(mpt);
1287      -        if (mptsas_register_intrs(mpt) == FALSE)
1288      -                goto fail;
1289      -        intr_added++;
1290      -
1291 1423          mutex_enter(&mpt->m_mutex);
1292 1424          /*
1293 1425           * Initialize power management component
1294 1426           */
1295 1427          if (mpt->m_options & MPTSAS_OPT_PM) {
1296 1428                  if (mptsas_init_pm(mpt)) {
1297 1429                          mutex_exit(&mpt->m_mutex);
1298 1430                          mptsas_log(mpt, CE_WARN, "mptsas pm initialization "
1299 1431                              "failed");
1300 1432                          goto fail;
↓ open down ↓ 24 lines elided ↑ open up ↑
1325 1457                      "mptsas_get_sas_io_unit_page_hndshk failed!");
1326 1458                  goto fail;
1327 1459          }
1328 1460  
1329 1461          if (mptsas_get_manufacture_page0(mpt) == DDI_FAILURE) {
1330 1462                  mptsas_log(mpt, CE_WARN,
1331 1463                      "mptsas_get_manufacture_page0 failed!");
1332 1464                  goto fail;
1333 1465          }
1334 1466  
     1467 +        /*
     1468 +         * If we only have one interrupt the default for doneq_thread_threshold
     1469 +         * should be 0 so that all completion processing goes to the threads.
     1470 +         * Only change it if it wasn't set from .conf file.
     1471 +         */
     1472 +        if (mpt->m_doneq_thread_n != 0 &&
     1473 +            ddi_prop_exists(DDI_DEV_T_ANY, dip,
     1474 +            0, "mptsas_doneq_length_threshold_prop") == 0 &&
     1475 +            mpt->m_intr_cnt == 1) {
     1476 +                mpt->m_doneq_length_threshold = 0;
     1477 +        }
     1478 +
     1479 +
1335 1480          mutex_exit(&mpt->m_mutex);
1336 1481  
1337 1482          /*
1338 1483           * Register the iport for multiple port HBA
1339 1484           */
1340 1485          mptsas_iport_register(mpt);
1341 1486  
1342 1487          /*
1343 1488           * initialize SCSI HBA transport structure
1344 1489           */
↓ open down ↓ 13 lines elided ↑ open up ↑
1358 1503          if (mpt->m_scsi_reset_delay == 0) {
1359 1504                  mptsas_log(mpt, CE_NOTE,
1360 1505                      "scsi_reset_delay of 0 is not recommended,"
1361 1506                      " resetting to SCSI_DEFAULT_RESET_DELAY\n");
1362 1507                  mpt->m_scsi_reset_delay = SCSI_DEFAULT_RESET_DELAY;
1363 1508          }
1364 1509  
1365 1510          /*
1366 1511           * Initialize the wait and done FIFO queue
1367 1512           */
1368      -        mpt->m_donetail = &mpt->m_doneq;
     1513 +        mpt->m_dlist.dl_tail = &mpt->m_dlist.dl_q;
1369 1514          mpt->m_waitqtail = &mpt->m_waitq;
1370      -        mpt->m_tx_waitqtail = &mpt->m_tx_waitq;
1371      -        mpt->m_tx_draining = 0;
1372 1515  
1373 1516          /*
1374 1517           * ioc cmd queue initialize
1375 1518           */
1376 1519          mpt->m_ioc_event_cmdtail = &mpt->m_ioc_event_cmdq;
1377 1520          mpt->m_dev_handle = 0xFFFF;
1378 1521  
1379 1522          MPTSAS_ENABLE_INTR(mpt);
1380 1523  
1381 1524          /*
1382 1525           * enable event notification
1383 1526           */
1384 1527          mutex_enter(&mpt->m_mutex);
1385 1528          if (mptsas_ioc_enable_event_notification(mpt)) {
1386 1529                  mutex_exit(&mpt->m_mutex);
1387 1530                  goto fail;
1388 1531          }
1389 1532          mutex_exit(&mpt->m_mutex);
1390 1533  
1391 1534          /*
1392      -         * Initialize PHY info for smhba
     1535 +         * used for mptsas_watch
     1536 +         */
     1537 +        mptsas_list_add(mpt);
     1538 +
     1539 +        mutex_enter(&mptsas_global_mutex);
     1540 +        if (mptsas_timeouts_enabled == 0) {
     1541 +                mptsas_scsi_watchdog_tick = ddi_prop_get_int(DDI_DEV_T_ANY,
     1542 +                    dip, 0, "scsi-watchdog-tick", DEFAULT_WD_TICK);
     1543 +
     1544 +                mptsas_tick = mptsas_scsi_watchdog_tick *
     1545 +                    drv_usectohz((clock_t)1000000);
     1546 +
     1547 +                mptsas_timeout_id = timeout(mptsas_watch, NULL, mptsas_tick);
     1548 +                mptsas_timeouts_enabled = 1;
     1549 +        }
     1550 +        mutex_exit(&mptsas_global_mutex);
     1551 +        added_watchdog++;
     1552 +
     1553 +        /*
     1554 +         * Initialize PHY info for smhba.
     1555 +         * This requires watchdog to be enabled otherwise if interrupts
     1556 +         * don't work the system will hang.
1393 1557           */
1394 1558          if (mptsas_smhba_setup(mpt)) {
1395 1559                  mptsas_log(mpt, CE_WARN, "mptsas phy initialization "
1396 1560                      "failed");
1397 1561                  goto fail;
1398 1562          }
1399 1563  
1400 1564          /* Check all dma handles allocated in attach */
1401 1565          if ((mptsas_check_dma_handle(mpt->m_dma_req_frame_hdl)
1402 1566              != DDI_SUCCESS) ||
     1567 +            (mptsas_check_dma_handle(mpt->m_dma_req_sense_hdl)
     1568 +            != DDI_SUCCESS) ||
1403 1569              (mptsas_check_dma_handle(mpt->m_dma_reply_frame_hdl)
1404 1570              != DDI_SUCCESS) ||
1405 1571              (mptsas_check_dma_handle(mpt->m_dma_free_queue_hdl)
1406 1572              != DDI_SUCCESS) ||
1407 1573              (mptsas_check_dma_handle(mpt->m_dma_post_queue_hdl)
1408 1574              != DDI_SUCCESS) ||
1409 1575              (mptsas_check_dma_handle(mpt->m_hshk_dma_hdl)
1410 1576              != DDI_SUCCESS)) {
1411 1577                  goto fail;
1412 1578          }
1413 1579  
1414 1580          /* Check all acc handles allocated in attach */
1415 1581          if ((mptsas_check_acc_handle(mpt->m_datap) != DDI_SUCCESS) ||
1416 1582              (mptsas_check_acc_handle(mpt->m_acc_req_frame_hdl)
1417 1583              != DDI_SUCCESS) ||
     1584 +            (mptsas_check_acc_handle(mpt->m_acc_req_sense_hdl)
     1585 +            != DDI_SUCCESS) ||
1418 1586              (mptsas_check_acc_handle(mpt->m_acc_reply_frame_hdl)
1419 1587              != DDI_SUCCESS) ||
1420 1588              (mptsas_check_acc_handle(mpt->m_acc_free_queue_hdl)
1421 1589              != DDI_SUCCESS) ||
1422 1590              (mptsas_check_acc_handle(mpt->m_acc_post_queue_hdl)
1423 1591              != DDI_SUCCESS) ||
1424 1592              (mptsas_check_acc_handle(mpt->m_hshk_acc_hdl)
1425 1593              != DDI_SUCCESS) ||
1426 1594              (mptsas_check_acc_handle(mpt->m_config_handle)
1427 1595              != DDI_SUCCESS)) {
1428 1596                  goto fail;
1429 1597          }
1430 1598  
1431 1599          /*
1432 1600           * After this point, we are not going to fail the attach.
1433 1601           */
1434      -        /*
1435      -         * used for mptsas_watch
1436      -         */
1437      -        mptsas_list_add(mpt);
1438      -
1439      -        mutex_enter(&mptsas_global_mutex);
1440      -        if (mptsas_timeouts_enabled == 0) {
1441      -                mptsas_scsi_watchdog_tick = ddi_prop_get_int(DDI_DEV_T_ANY,
1442      -                    dip, 0, "scsi-watchdog-tick", DEFAULT_WD_TICK);
1443      -
1444      -                mptsas_tick = mptsas_scsi_watchdog_tick *
1445      -                    drv_usectohz((clock_t)1000000);
1446      -
1447      -                mptsas_timeout_id = timeout(mptsas_watch, NULL, mptsas_tick);
1448      -                mptsas_timeouts_enabled = 1;
1449      -        }
1450      -        mutex_exit(&mptsas_global_mutex);
1451 1602  
1452 1603          /* Print message of HBA present */
1453 1604          ddi_report_dev(dip);
1454 1605  
1455 1606          /* report idle status to pm framework */
1456 1607          if (mpt->m_options & MPTSAS_OPT_PM) {
1457 1608                  (void) pm_idle_component(dip, 0);
1458 1609          }
1459 1610  
1460 1611          return (DDI_SUCCESS);
1461 1612  
1462 1613  fail:
1463 1614          mptsas_log(mpt, CE_WARN, "attach failed");
1464 1615          mptsas_fm_ereport(mpt, DDI_FM_DEVICE_NO_RESPONSE);
1465 1616          ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_LOST);
1466 1617          if (mpt) {
1467      -                mutex_enter(&mptsas_global_mutex);
     1618 +                /* deallocate in reverse order */
     1619 +                if (added_watchdog) {
     1620 +                        mptsas_list_del(mpt);
     1621 +                        mutex_enter(&mptsas_global_mutex);
1468 1622  
1469      -                if (mptsas_timeout_id && (mptsas_head == NULL)) {
1470      -                        timeout_id_t tid = mptsas_timeout_id;
1471      -                        mptsas_timeouts_enabled = 0;
1472      -                        mptsas_timeout_id = 0;
     1623 +                        if (mptsas_timeout_id && (mptsas_head == NULL)) {
     1624 +                                timeout_id_t tid = mptsas_timeout_id;
     1625 +                                mptsas_timeouts_enabled = 0;
     1626 +                                mptsas_timeout_id = 0;
     1627 +                                mutex_exit(&mptsas_global_mutex);
     1628 +                                (void) untimeout(tid);
     1629 +                                mutex_enter(&mptsas_global_mutex);
     1630 +                        }
1473 1631                          mutex_exit(&mptsas_global_mutex);
1474      -                        (void) untimeout(tid);
1475      -                        mutex_enter(&mptsas_global_mutex);
1476 1632                  }
1477      -                mutex_exit(&mptsas_global_mutex);
1478      -                /* deallocate in reverse order */
     1633 +
1479 1634                  mptsas_cache_destroy(mpt);
1480 1635  
1481 1636                  if (smp_attach_setup) {
1482 1637                          mptsas_smp_teardown(mpt);
1483 1638                  }
1484 1639                  if (hba_attach_setup) {
1485 1640                          mptsas_hba_teardown(mpt);
1486 1641                  }
1487 1642  
1488 1643                  if (mpt->m_targets)
1489 1644                          refhash_destroy(mpt->m_targets);
1490 1645                  if (mpt->m_smp_targets)
1491 1646                          refhash_destroy(mpt->m_smp_targets);
1492 1647  
1493 1648                  if (mpt->m_active) {
1494 1649                          mptsas_free_active_slots(mpt);
1495 1650                  }
1496      -                if (intr_added) {
     1651 +                if (mpt->m_intr_cnt) {
1497 1652                          mptsas_unregister_intrs(mpt);
1498 1653                  }
1499 1654  
1500 1655                  if (doneq_thread_create) {
1501      -                        mutex_enter(&mpt->m_doneq_mutex);
1502      -                        doneq_thread_num = mpt->m_doneq_thread_n;
1503      -                        for (j = 0; j < mpt->m_doneq_thread_n; j++) {
     1656 +                        mutex_enter(&mpt->m_qthread_mutex);
     1657 +                        q_thread_num = mpt->m_doneq_thread_n;
     1658 +                        for (j = 0; j < q_thread_num; j++) {
1504 1659                                  mutex_enter(&mpt->m_doneq_thread_id[j].mutex);
1505 1660                                  mpt->m_doneq_thread_id[j].flag &=
1506 1661                                      (~MPTSAS_DONEQ_THREAD_ACTIVE);
1507 1662                                  cv_signal(&mpt->m_doneq_thread_id[j].cv);
1508 1663                                  mutex_exit(&mpt->m_doneq_thread_id[j].mutex);
1509 1664                          }
1510 1665                          while (mpt->m_doneq_thread_n) {
1511      -                                cv_wait(&mpt->m_doneq_thread_cv,
1512      -                                    &mpt->m_doneq_mutex);
     1666 +                                cv_wait(&mpt->m_qthread_cv,
     1667 +                                    &mpt->m_qthread_mutex);
1513 1668                          }
1514      -                        for (j = 0; j < doneq_thread_num; j++) {
     1669 +                        for (j = 0; j < q_thread_num; j++) {
1515 1670                                  cv_destroy(&mpt->m_doneq_thread_id[j].cv);
1516 1671                                  mutex_destroy(&mpt->m_doneq_thread_id[j].mutex);
1517 1672                          }
1518 1673                          kmem_free(mpt->m_doneq_thread_id,
1519 1674                              sizeof (mptsas_doneq_thread_list_t)
1520      -                            * doneq_thread_num);
1521      -                        mutex_exit(&mpt->m_doneq_mutex);
1522      -                        cv_destroy(&mpt->m_doneq_thread_cv);
1523      -                        mutex_destroy(&mpt->m_doneq_mutex);
     1675 +                            * q_thread_num);
     1676 +                        mutex_exit(&mpt->m_qthread_mutex);
     1677 +                }
     1678 +                if (txwq_thread_create) {
     1679 +                        mutex_enter(&mpt->m_qthread_mutex);
     1680 +                        q_thread_num = mpt->m_txwq_thread_n;
     1681 +                        for (j = 0; j < q_thread_num; j++) {
     1682 +                                mutex_enter(&mpt->m_tx_waitq[j].txwq_mutex);
     1683 +                                mpt->m_tx_waitq[j].txwq_active = FALSE;
     1684 +                                cv_signal(&mpt->m_tx_waitq[j].txwq_cv);
     1685 +                                mutex_exit(&mpt->m_tx_waitq[j].txwq_mutex);
     1686 +                        }
     1687 +                        while (mpt->m_txwq_thread_n) {
     1688 +                                cv_wait(&mpt->m_qthread_cv,
     1689 +                                    &mpt->m_qthread_mutex);
     1690 +                        }
     1691 +                        for (j = 0; j < q_thread_num; j++) {
     1692 +                                cv_destroy(&mpt->m_tx_waitq[j].txwq_cv);
     1693 +                                cv_destroy(&mpt->m_tx_waitq[j].txwq_drain_cv);
     1694 +                                mutex_destroy(&mpt->m_tx_waitq[j].txwq_mutex);
     1695 +                        }
1524 1696                  }
1525 1697                  if (event_taskq_create) {
1526 1698                          ddi_taskq_destroy(mpt->m_event_taskq);
1527 1699                  }
1528 1700                  if (dr_taskq_create) {
1529 1701                          ddi_taskq_destroy(mpt->m_dr_taskq);
1530 1702                  }
1531 1703                  if (mutex_init_done) {
1532      -                        mutex_destroy(&mpt->m_tx_waitq_mutex);
     1704 +                        mutex_destroy(&mpt->m_qthread_mutex);
1533 1705                          mutex_destroy(&mpt->m_passthru_mutex);
1534 1706                          mutex_destroy(&mpt->m_mutex);
1535 1707                          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
1536 1708                                  mutex_destroy(
1537 1709                                      &mpt->m_phy_info[i].smhba_info.phy_mutex);
1538 1710                          }
     1711 +                        cv_destroy(&mpt->m_qthread_cv);
1539 1712                          cv_destroy(&mpt->m_cv);
1540 1713                          cv_destroy(&mpt->m_passthru_cv);
1541 1714                          cv_destroy(&mpt->m_fw_cv);
1542 1715                          cv_destroy(&mpt->m_config_cv);
1543 1716                          cv_destroy(&mpt->m_fw_diag_cv);
1544 1717                  }
1545 1718  
1546 1719                  if (map_setup) {
1547 1720                          mptsas_cfg_fini(mpt);
1548 1721                  }
1549 1722                  if (config_setup) {
1550 1723                          mptsas_config_space_fini(mpt);
1551 1724                  }
1552 1725                  mptsas_free_handshake_msg(mpt);
1553 1726                  mptsas_hba_fini(mpt);
1554 1727  
1555 1728                  mptsas_fm_fini(mpt);
1556      -                ddi_soft_state_free(mptsas_state, instance);
     1729 +                ddi_soft_state_free(mptsas3_state, instance);
1557 1730                  ddi_prop_remove_all(dip);
1558 1731          }
1559 1732          return (DDI_FAILURE);
1560 1733  }
1561 1734  
1562 1735  static int
1563 1736  mptsas_suspend(dev_info_t *devi)
1564 1737  {
1565 1738          mptsas_t        *mpt, *g;
1566 1739          scsi_hba_tran_t *tran;
↓ open down ↓ 193 lines elided ↑ open up ↑
1760 1933  
1761 1934  static int
1762 1935  mptsas_do_detach(dev_info_t *dip)
1763 1936  {
1764 1937          mptsas_t        *mpt;
1765 1938          scsi_hba_tran_t *tran;
1766 1939          int             circ = 0;
1767 1940          int             circ1 = 0;
1768 1941          mdi_pathinfo_t  *pip = NULL;
1769 1942          int             i;
1770      -        int             doneq_thread_num = 0;
     1943 +        int             q_thread_num = 0;
1771 1944  
1772 1945          NDBG0(("mptsas_do_detach: dip=0x%p", (void *)dip));
1773 1946  
1774 1947          if ((tran = ndi_flavorv_get(dip, SCSA_FLAVOR_SCSI_DEVICE)) == NULL)
1775 1948                  return (DDI_FAILURE);
1776 1949  
1777 1950          mpt = TRAN2MPT(tran);
1778 1951          if (!mpt) {
1779 1952                  return (DDI_FAILURE);
1780 1953          }
↓ open down ↓ 27 lines elided ↑ open up ↑
1808 1981                  return (DDI_SUCCESS);
1809 1982          }
1810 1983  
1811 1984          /* Make sure power level is D0 before accessing registers */
1812 1985          if (mpt->m_options & MPTSAS_OPT_PM) {
1813 1986                  (void) pm_busy_component(dip, 0);
1814 1987                  if (mpt->m_power_level != PM_LEVEL_D0) {
1815 1988                          if (pm_raise_power(dip, 0, PM_LEVEL_D0) !=
1816 1989                              DDI_SUCCESS) {
1817 1990                                  mptsas_log(mpt, CE_WARN,
1818      -                                    "mptsas%d: Raise power request failed.",
     1991 +                                    "mptsas3%d: Raise power request failed.",
1819 1992                                      mpt->m_instance);
1820 1993                                  (void) pm_idle_component(dip, 0);
1821 1994                                  return (DDI_FAILURE);
1822 1995                          }
1823 1996                  }
1824 1997          }
1825 1998  
1826 1999          /*
1827 2000           * Send RAID action system shutdown to sync IR.  After action, send a
1828 2001           * Message Unit Reset. Since after that DMA resource will be freed,
↓ open down ↓ 3 lines elided ↑ open up ↑
1832 2005          MPTSAS_DISABLE_INTR(mpt);
1833 2006          mptsas_raid_action_system_shutdown(mpt);
1834 2007          mpt->m_softstate |= MPTSAS_SS_MSG_UNIT_RESET;
1835 2008          (void) mptsas_ioc_reset(mpt, FALSE);
1836 2009          mutex_exit(&mpt->m_mutex);
1837 2010          mptsas_rem_intrs(mpt);
1838 2011          ddi_taskq_destroy(mpt->m_event_taskq);
1839 2012          ddi_taskq_destroy(mpt->m_dr_taskq);
1840 2013  
1841 2014          if (mpt->m_doneq_thread_n) {
1842      -                mutex_enter(&mpt->m_doneq_mutex);
1843      -                doneq_thread_num = mpt->m_doneq_thread_n;
     2015 +                mutex_enter(&mpt->m_qthread_mutex);
     2016 +                q_thread_num = mpt->m_doneq_thread_n;
1844 2017                  for (i = 0; i < mpt->m_doneq_thread_n; i++) {
1845 2018                          mutex_enter(&mpt->m_doneq_thread_id[i].mutex);
1846 2019                          mpt->m_doneq_thread_id[i].flag &=
1847 2020                              (~MPTSAS_DONEQ_THREAD_ACTIVE);
1848 2021                          cv_signal(&mpt->m_doneq_thread_id[i].cv);
1849 2022                          mutex_exit(&mpt->m_doneq_thread_id[i].mutex);
1850 2023                  }
1851 2024                  while (mpt->m_doneq_thread_n) {
1852      -                        cv_wait(&mpt->m_doneq_thread_cv,
1853      -                            &mpt->m_doneq_mutex);
     2025 +                        cv_wait(&mpt->m_qthread_cv,
     2026 +                            &mpt->m_qthread_mutex);
1854 2027                  }
1855      -                for (i = 0;  i < doneq_thread_num; i++) {
     2028 +                for (i = 0;  i < q_thread_num; i++) {
1856 2029                          cv_destroy(&mpt->m_doneq_thread_id[i].cv);
1857 2030                          mutex_destroy(&mpt->m_doneq_thread_id[i].mutex);
1858 2031                  }
1859 2032                  kmem_free(mpt->m_doneq_thread_id,
1860 2033                      sizeof (mptsas_doneq_thread_list_t)
1861      -                    * doneq_thread_num);
1862      -                mutex_exit(&mpt->m_doneq_mutex);
1863      -                cv_destroy(&mpt->m_doneq_thread_cv);
1864      -                mutex_destroy(&mpt->m_doneq_mutex);
     2034 +                    * q_thread_num);
     2035 +                mutex_exit(&mpt->m_qthread_mutex);
     2036 +        }
     2037 +        if (mpt->m_txwq_thread_n) {
     2038 +                mutex_enter(&mpt->m_qthread_mutex);
     2039 +                q_thread_num = mpt->m_txwq_thread_n;
     2040 +                for (i = 0; i < q_thread_num; i++) {
     2041 +                        mutex_enter(&mpt->m_tx_waitq[i].txwq_mutex);
     2042 +                        mpt->m_tx_waitq[i].txwq_active = FALSE;
     2043 +                        cv_signal(&mpt->m_tx_waitq[i].txwq_cv);
     2044 +                        mutex_exit(&mpt->m_tx_waitq[i].txwq_mutex);
     2045 +                }
     2046 +                while (mpt->m_txwq_thread_n) {
     2047 +                        cv_wait(&mpt->m_qthread_cv,
     2048 +                            &mpt->m_qthread_mutex);
     2049 +                }
     2050 +                for (i = 0; i < q_thread_num; i++) {
     2051 +                        cv_destroy(&mpt->m_tx_waitq[i].txwq_cv);
     2052 +                        cv_destroy(&mpt->m_tx_waitq[i].txwq_drain_cv);
     2053 +                        mutex_destroy(&mpt->m_tx_waitq[i].txwq_mutex);
     2054 +                }
1865 2055          }
1866 2056  
1867 2057          scsi_hba_reset_notify_tear_down(mpt->m_reset_notify_listf);
1868 2058  
1869 2059          mptsas_list_del(mpt);
1870 2060  
1871 2061          /*
1872 2062           * Cancel timeout threads for this mpt
1873 2063           */
1874 2064          mutex_enter(&mpt->m_mutex);
↓ open down ↓ 60 lines elided ↑ open up ↑
1935 2125          /* deallocate everything that was allocated in mptsas_attach */
1936 2126          mptsas_cache_destroy(mpt);
1937 2127  
1938 2128          mptsas_hba_fini(mpt);
1939 2129          mptsas_cfg_fini(mpt);
1940 2130  
1941 2131          /* Lower the power informing PM Framework */
1942 2132          if (mpt->m_options & MPTSAS_OPT_PM) {
1943 2133                  if (pm_lower_power(dip, 0, PM_LEVEL_D3) != DDI_SUCCESS)
1944 2134                          mptsas_log(mpt, CE_WARN,
1945      -                            "!mptsas%d: Lower power request failed "
     2135 +                            "!mptsas3%d: Lower power request failed "
1946 2136                              "during detach, ignoring.",
1947 2137                              mpt->m_instance);
1948 2138          }
1949 2139  
1950      -        mutex_destroy(&mpt->m_tx_waitq_mutex);
     2140 +        mutex_destroy(&mpt->m_qthread_mutex);
1951 2141          mutex_destroy(&mpt->m_passthru_mutex);
1952 2142          mutex_destroy(&mpt->m_mutex);
1953 2143          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
1954 2144                  mutex_destroy(&mpt->m_phy_info[i].smhba_info.phy_mutex);
1955 2145          }
     2146 +        cv_destroy(&mpt->m_qthread_cv);
1956 2147          cv_destroy(&mpt->m_cv);
1957 2148          cv_destroy(&mpt->m_passthru_cv);
1958 2149          cv_destroy(&mpt->m_fw_cv);
1959 2150          cv_destroy(&mpt->m_config_cv);
1960 2151          cv_destroy(&mpt->m_fw_diag_cv);
1961 2152  
1962 2153  
1963 2154          mptsas_smp_teardown(mpt);
1964 2155          mptsas_hba_teardown(mpt);
1965 2156  
1966 2157          mptsas_config_space_fini(mpt);
1967 2158  
1968 2159          mptsas_free_handshake_msg(mpt);
1969 2160  
1970 2161          mptsas_fm_fini(mpt);
1971      -        ddi_soft_state_free(mptsas_state, ddi_get_instance(dip));
     2162 +        ddi_soft_state_free(mptsas3_state, ddi_get_instance(dip));
1972 2163          ddi_prop_remove_all(dip);
1973 2164  
1974 2165          return (DDI_SUCCESS);
1975 2166  }
1976 2167  
1977 2168  static void
1978 2169  mptsas_list_add(mptsas_t *mpt)
1979 2170  {
1980 2171          rw_enter(&mptsas_global_rwlock, RW_WRITER);
1981 2172  
↓ open down ↓ 32 lines elided ↑ open up ↑
2014 2205                  mptsas_tail = m;
2015 2206          }
2016 2207          rw_exit(&mptsas_global_rwlock);
2017 2208  }
2018 2209  
2019 2210  static int
2020 2211  mptsas_alloc_handshake_msg(mptsas_t *mpt, size_t alloc_size)
2021 2212  {
2022 2213          ddi_dma_attr_t  task_dma_attrs;
2023 2214  
     2215 +        mpt->m_hshk_dma_size = 0;
2024 2216          task_dma_attrs = mpt->m_msg_dma_attr;
2025 2217          task_dma_attrs.dma_attr_sgllen = 1;
2026 2218          task_dma_attrs.dma_attr_granular = (uint32_t)(alloc_size);
2027 2219  
2028 2220          /* allocate Task Management ddi_dma resources */
2029 2221          if (mptsas_dma_addr_create(mpt, task_dma_attrs,
2030 2222              &mpt->m_hshk_dma_hdl, &mpt->m_hshk_acc_hdl, &mpt->m_hshk_memp,
2031 2223              alloc_size, NULL) == FALSE) {
2032 2224                  return (DDI_FAILURE);
2033 2225          }
2034 2226          mpt->m_hshk_dma_size = alloc_size;
2035 2227  
2036 2228          return (DDI_SUCCESS);
2037 2229  }
2038 2230  
2039 2231  static void
2040 2232  mptsas_free_handshake_msg(mptsas_t *mpt)
2041 2233  {
     2234 +        if (mpt->m_hshk_dma_size == 0)
     2235 +                return;
2042 2236          mptsas_dma_addr_destroy(&mpt->m_hshk_dma_hdl, &mpt->m_hshk_acc_hdl);
2043 2237          mpt->m_hshk_dma_size = 0;
2044 2238  }
2045 2239  
2046 2240  static int
2047 2241  mptsas_hba_setup(mptsas_t *mpt)
2048 2242  {
2049 2243          scsi_hba_tran_t         *hba_tran;
2050 2244          int                     tran_flags;
2051 2245  
↓ open down ↓ 157 lines elided ↑ open up ↑
2209 2403  
2210 2404  static int
2211 2405  mptsas_cache_create(mptsas_t *mpt)
2212 2406  {
2213 2407          int instance = mpt->m_instance;
2214 2408          char buf[64];
2215 2409  
2216 2410          /*
2217 2411           * create kmem cache for packets
2218 2412           */
2219      -        (void) sprintf(buf, "mptsas%d_cache", instance);
     2413 +        (void) sprintf(buf, "mptsas3%d_cache", instance);
2220 2414          mpt->m_kmem_cache = kmem_cache_create(buf,
2221      -            sizeof (struct mptsas_cmd) + scsi_pkt_size(), 8,
     2415 +            sizeof (struct mptsas_cmd) + scsi_pkt_size(), 16,
2222 2416              mptsas_kmem_cache_constructor, mptsas_kmem_cache_destructor,
2223 2417              NULL, (void *)mpt, NULL, 0);
2224 2418  
2225 2419          if (mpt->m_kmem_cache == NULL) {
2226 2420                  mptsas_log(mpt, CE_WARN, "creating kmem cache failed");
2227 2421                  return (FALSE);
2228 2422          }
2229 2423  
2230 2424          /*
2231 2425           * create kmem cache for extra SGL frames if SGL cannot
2232 2426           * be accomodated into main request frame.
2233 2427           */
2234      -        (void) sprintf(buf, "mptsas%d_cache_frames", instance);
     2428 +        (void) sprintf(buf, "mptsas3%d_cache_frames", instance);
2235 2429          mpt->m_cache_frames = kmem_cache_create(buf,
2236      -            sizeof (mptsas_cache_frames_t), 8,
     2430 +            sizeof (mptsas_cache_frames_t), 16,
2237 2431              mptsas_cache_frames_constructor, mptsas_cache_frames_destructor,
2238 2432              NULL, (void *)mpt, NULL, 0);
2239 2433  
2240 2434          if (mpt->m_cache_frames == NULL) {
2241 2435                  mptsas_log(mpt, CE_WARN, "creating cache for frames failed");
2242 2436                  return (FALSE);
2243 2437          }
2244 2438  
2245 2439          return (TRUE);
2246 2440  }
↓ open down ↓ 19 lines elided ↑ open up ↑
2266 2460          _NOTE(ARGUNUSED(component))
2267 2461  #endif
2268 2462          mptsas_t        *mpt;
2269 2463          int             rval = DDI_SUCCESS;
2270 2464          int             polls = 0;
2271 2465          uint32_t        ioc_status;
2272 2466  
2273 2467          if (scsi_hba_iport_unit_address(dip) != 0)
2274 2468                  return (DDI_SUCCESS);
2275 2469  
2276      -        mpt = ddi_get_soft_state(mptsas_state, ddi_get_instance(dip));
     2470 +        mpt = ddi_get_soft_state(mptsas3_state, ddi_get_instance(dip));
2277 2471          if (mpt == NULL) {
2278 2472                  return (DDI_FAILURE);
2279 2473          }
2280 2474  
2281 2475          mutex_enter(&mpt->m_mutex);
2282 2476  
2283 2477          /*
2284 2478           * If the device is busy, don't lower its power level
2285 2479           */
2286 2480          if (mpt->m_busy && (mpt->m_power_level > level)) {
2287 2481                  mutex_exit(&mpt->m_mutex);
2288 2482                  return (DDI_FAILURE);
2289 2483          }
2290 2484          switch (level) {
2291 2485          case PM_LEVEL_D0:
2292      -                NDBG11(("mptsas%d: turning power ON.", mpt->m_instance));
     2486 +                NDBG11(("mptsas3%d: turning power ON.", mpt->m_instance));
2293 2487                  MPTSAS_POWER_ON(mpt);
2294 2488                  /*
2295 2489                   * Wait up to 30 seconds for IOC to come out of reset.
2296 2490                   */
2297 2491                  while (((ioc_status = ddi_get32(mpt->m_datap,
2298 2492                      &mpt->m_reg->Doorbell)) &
2299 2493                      MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
2300 2494                          if (polls++ > 3000) {
2301 2495                                  break;
2302 2496                          }
↓ open down ↓ 8 lines elided ↑ open up ↑
2311 2505                          if (mptsas_restart_ioc(mpt) == DDI_FAILURE) {
2312 2506                                  mptsas_log(mpt, CE_WARN,
2313 2507                                      "mptsas_power: hard reset failed");
2314 2508                                  mutex_exit(&mpt->m_mutex);
2315 2509                                  return (DDI_FAILURE);
2316 2510                          }
2317 2511                  }
2318 2512                  mpt->m_power_level = PM_LEVEL_D0;
2319 2513                  break;
2320 2514          case PM_LEVEL_D3:
2321      -                NDBG11(("mptsas%d: turning power OFF.", mpt->m_instance));
     2515 +                NDBG11(("mptsas3%d: turning power OFF.", mpt->m_instance));
2322 2516                  MPTSAS_POWER_OFF(mpt);
2323 2517                  break;
2324 2518          default:
2325      -                mptsas_log(mpt, CE_WARN, "mptsas%d: unknown power level <%x>.",
     2519 +                mptsas_log(mpt, CE_WARN, "mptsas3%d: unknown power level <%x>.",
2326 2520                      mpt->m_instance, level);
2327 2521                  rval = DDI_FAILURE;
2328 2522                  break;
2329 2523          }
2330 2524          mutex_exit(&mpt->m_mutex);
2331 2525          return (rval);
2332 2526  }
2333 2527  
2334 2528  /*
2335 2529   * Initialize configuration space and figure out which
↓ open down ↓ 157 lines elided ↑ open up ↑
2493 2687  mptsas_alloc_request_frames(mptsas_t *mpt)
2494 2688  {
2495 2689          ddi_dma_attr_t          frame_dma_attrs;
2496 2690          caddr_t                 memp;
2497 2691          ddi_dma_cookie_t        cookie;
2498 2692          size_t                  mem_size;
2499 2693  
2500 2694          /*
2501 2695           * re-alloc when it has already alloced
2502 2696           */
2503      -        mptsas_dma_addr_destroy(&mpt->m_dma_req_frame_hdl,
2504      -            &mpt->m_acc_req_frame_hdl);
     2697 +        if (mpt->m_dma_flags & MPTSAS_REQ_FRAME) {
     2698 +                mptsas_dma_addr_destroy(&mpt->m_dma_req_frame_hdl,
     2699 +                    &mpt->m_acc_req_frame_hdl);
     2700 +                mpt->m_dma_flags &= ~MPTSAS_REQ_FRAME;
     2701 +        }
2505 2702  
2506 2703          /*
2507 2704           * The size of the request frame pool is:
2508 2705           *   Number of Request Frames * Request Frame Size
2509 2706           */
2510 2707          mem_size = mpt->m_max_requests * mpt->m_req_frame_size;
2511 2708  
2512 2709          /*
2513 2710           * set the DMA attributes.  System Request Message Frames must be
2514 2711           * aligned on a 16-byte boundry.
↓ open down ↓ 17 lines elided ↑ open up ↑
2532 2729           * address is the address mpt uses to fill in the frame.
2533 2730           */
2534 2731          mpt->m_req_frame_dma_addr = cookie.dmac_laddress;
2535 2732          mpt->m_req_frame = memp;
2536 2733  
2537 2734          /*
2538 2735           * Clear the request frame pool.
2539 2736           */
2540 2737          bzero(mpt->m_req_frame, mem_size);
2541 2738  
     2739 +        mpt->m_dma_flags |= MPTSAS_REQ_FRAME;
     2740 +        return (DDI_SUCCESS);
     2741 +}
     2742 +
     2743 +static int
     2744 +mptsas_alloc_sense_bufs(mptsas_t *mpt)
     2745 +{
     2746 +        ddi_dma_attr_t          sense_dma_attrs;
     2747 +        caddr_t                 memp;
     2748 +        ddi_dma_cookie_t        cookie;
     2749 +        size_t                  mem_size;
     2750 +        int                     num_extrqsense_bufs;
     2751 +
     2752 +        /*
     2753 +         * re-alloc when it has already alloced
     2754 +         */
     2755 +        if (mpt->m_dma_flags & MPTSAS_REQ_SENSE) {
     2756 +                rmfreemap(mpt->m_erqsense_map);
     2757 +                mptsas_dma_addr_destroy(&mpt->m_dma_req_sense_hdl,
     2758 +                    &mpt->m_acc_req_sense_hdl);
     2759 +                mpt->m_dma_flags &= ~MPTSAS_REQ_SENSE;
     2760 +        }
     2761 +
     2762 +        /*
     2763 +         * The size of the request sense pool is:
     2764 +         *   (Number of Request Frames - 2 ) * Request Sense Size +
     2765 +         *   extra memory for extended sense requests.
     2766 +         */
     2767 +        mem_size = ((mpt->m_max_requests - 2) * mpt->m_req_sense_size) +
     2768 +            mptsas_extreq_sense_bufsize;
     2769 +
     2770 +        /*
     2771 +         * set the DMA attributes.  ARQ buffers
     2772 +         * aligned on a 16-byte boundry.
     2773 +         */
     2774 +        sense_dma_attrs = mpt->m_msg_dma_attr;
     2775 +        sense_dma_attrs.dma_attr_align = 16;
     2776 +        sense_dma_attrs.dma_attr_sgllen = 1;
     2777 +
     2778 +        /*
     2779 +         * allocate the request sense buffer pool.
     2780 +         */
     2781 +        if (mptsas_dma_addr_create(mpt, sense_dma_attrs,
     2782 +            &mpt->m_dma_req_sense_hdl, &mpt->m_acc_req_sense_hdl, &memp,
     2783 +            mem_size, &cookie) == FALSE) {
     2784 +                return (DDI_FAILURE);
     2785 +        }
     2786 +
     2787 +        /*
     2788 +         * Store the request sense base memory address.  This chip uses this
     2789 +         * address to dma the request sense data.  The second
     2790 +         * address is the address mpt uses to access the data.
     2791 +         * The third is the base for the extended rqsense buffers.
     2792 +         */
     2793 +        mpt->m_req_sense_dma_addr = cookie.dmac_laddress;
     2794 +        mpt->m_req_sense = memp;
     2795 +        memp += (mpt->m_max_requests - 2) * mpt->m_req_sense_size;
     2796 +        mpt->m_extreq_sense = memp;
     2797 +
     2798 +        /*
     2799 +         * The extra memory is divided up into multiples of the base
     2800 +         * buffer size in order to allocate via rmalloc().
     2801 +         * Note that the rmallocmap cannot start at zero!
     2802 +         */
     2803 +        num_extrqsense_bufs = mptsas_extreq_sense_bufsize /
     2804 +            mpt->m_req_sense_size;
     2805 +        mpt->m_erqsense_map = rmallocmap_wait(num_extrqsense_bufs);
     2806 +        rmfree(mpt->m_erqsense_map, num_extrqsense_bufs, 1);
     2807 +
     2808 +        /*
     2809 +         * Clear the pool.
     2810 +         */
     2811 +        bzero(mpt->m_req_sense, mem_size);
     2812 +
     2813 +        mpt->m_dma_flags |= MPTSAS_REQ_SENSE;
2542 2814          return (DDI_SUCCESS);
2543 2815  }
2544 2816  
2545 2817  static int
2546 2818  mptsas_alloc_reply_frames(mptsas_t *mpt)
2547 2819  {
2548 2820          ddi_dma_attr_t          frame_dma_attrs;
2549 2821          caddr_t                 memp;
2550 2822          ddi_dma_cookie_t        cookie;
2551 2823          size_t                  mem_size;
2552 2824  
2553 2825          /*
2554 2826           * re-alloc when it has already alloced
2555 2827           */
2556      -        mptsas_dma_addr_destroy(&mpt->m_dma_reply_frame_hdl,
2557      -            &mpt->m_acc_reply_frame_hdl);
     2828 +        if (mpt->m_dma_flags & MPTSAS_REPLY_FRAME) {
     2829 +                mptsas_dma_addr_destroy(&mpt->m_dma_reply_frame_hdl,
     2830 +                    &mpt->m_acc_reply_frame_hdl);
     2831 +                mpt->m_dma_flags &= ~MPTSAS_REPLY_FRAME;
     2832 +        }
2558 2833  
2559 2834          /*
2560 2835           * The size of the reply frame pool is:
2561 2836           *   Number of Reply Frames * Reply Frame Size
2562 2837           */
2563 2838          mem_size = mpt->m_max_replies * mpt->m_reply_frame_size;
2564 2839  
2565 2840          /*
2566 2841           * set the DMA attributes.   System Reply Message Frames must be
2567 2842           * aligned on a 4-byte boundry.  This is the default.
↓ open down ↓ 16 lines elided ↑ open up ↑
2584 2859           * address is the address mpt uses to process the frame.
2585 2860           */
2586 2861          mpt->m_reply_frame_dma_addr = cookie.dmac_laddress;
2587 2862          mpt->m_reply_frame = memp;
2588 2863  
2589 2864          /*
2590 2865           * Clear the reply frame pool.
2591 2866           */
2592 2867          bzero(mpt->m_reply_frame, mem_size);
2593 2868  
     2869 +        mpt->m_dma_flags |= MPTSAS_REPLY_FRAME;
2594 2870          return (DDI_SUCCESS);
2595 2871  }
2596 2872  
2597 2873  static int
2598 2874  mptsas_alloc_free_queue(mptsas_t *mpt)
2599 2875  {
2600 2876          ddi_dma_attr_t          frame_dma_attrs;
2601 2877          caddr_t                 memp;
2602 2878          ddi_dma_cookie_t        cookie;
2603 2879          size_t                  mem_size;
2604 2880  
2605 2881          /*
2606 2882           * re-alloc when it has already alloced
2607 2883           */
2608      -        mptsas_dma_addr_destroy(&mpt->m_dma_free_queue_hdl,
2609      -            &mpt->m_acc_free_queue_hdl);
     2884 +        if (mpt->m_dma_flags & MPTSAS_FREE_QUEUE) {
     2885 +                mptsas_dma_addr_destroy(&mpt->m_dma_free_queue_hdl,
     2886 +                    &mpt->m_acc_free_queue_hdl);
     2887 +                mpt->m_dma_flags &= ~MPTSAS_FREE_QUEUE;
     2888 +        }
2610 2889  
2611 2890          /*
2612 2891           * The reply free queue size is:
2613 2892           *   Reply Free Queue Depth * 4
2614 2893           * The "4" is the size of one 32 bit address (low part of 64-bit
2615 2894           *   address)
2616 2895           */
2617 2896          mem_size = mpt->m_free_queue_depth * 4;
2618 2897  
2619 2898          /*
↓ open down ↓ 19 lines elided ↑ open up ↑
2639 2918           * is the address mpt uses to manage the queue.
2640 2919           */
2641 2920          mpt->m_free_queue_dma_addr = cookie.dmac_laddress;
2642 2921          mpt->m_free_queue = memp;
2643 2922  
2644 2923          /*
2645 2924           * Clear the reply free queue memory.
2646 2925           */
2647 2926          bzero(mpt->m_free_queue, mem_size);
2648 2927  
     2928 +        mpt->m_dma_flags |= MPTSAS_FREE_QUEUE;
2649 2929          return (DDI_SUCCESS);
2650 2930  }
2651 2931  
     2932 +static void
     2933 +mptsas_free_post_queue(mptsas_t *mpt)
     2934 +{
     2935 +        mptsas_reply_pqueue_t   *rpqp;
     2936 +        int                     i;
     2937 +
     2938 +        if (mpt->m_dma_flags & MPTSAS_POST_QUEUE) {
     2939 +                mptsas_dma_addr_destroy(&mpt->m_dma_post_queue_hdl,
     2940 +                    &mpt->m_acc_post_queue_hdl);
     2941 +                rpqp = mpt->m_rep_post_queues;
     2942 +                for (i = 0; i < mpt->m_post_reply_qcount; i++) {
     2943 +                        mutex_destroy(&rpqp->rpq_mutex);
     2944 +                        rpqp++;
     2945 +                }
     2946 +                kmem_free(mpt->m_rep_post_queues,
     2947 +                    sizeof (mptsas_reply_pqueue_t) *
     2948 +                    mpt->m_post_reply_qcount);
     2949 +                mpt->m_dma_flags &= ~MPTSAS_POST_QUEUE;
     2950 +        }
     2951 +}
     2952 +
2652 2953  static int
2653 2954  mptsas_alloc_post_queue(mptsas_t *mpt)
2654 2955  {
2655 2956          ddi_dma_attr_t          frame_dma_attrs;
2656 2957          caddr_t                 memp;
2657 2958          ddi_dma_cookie_t        cookie;
2658 2959          size_t                  mem_size;
     2960 +        mptsas_reply_pqueue_t   *rpqp;
     2961 +        int                     i;
2659 2962  
2660 2963          /*
2661 2964           * re-alloc when it has already alloced
2662 2965           */
2663      -        mptsas_dma_addr_destroy(&mpt->m_dma_post_queue_hdl,
2664      -            &mpt->m_acc_post_queue_hdl);
     2966 +        mptsas_free_post_queue(mpt);
2665 2967  
2666 2968          /*
2667 2969           * The reply descriptor post queue size is:
2668 2970           *   Reply Descriptor Post Queue Depth * 8
2669 2971           * The "8" is the size of each descriptor (8 bytes or 64 bits).
2670 2972           */
2671      -        mem_size = mpt->m_post_queue_depth * 8;
     2973 +        mpt->m_post_reply_qcount = mpt->m_intr_cnt;
     2974 +        mem_size = mpt->m_post_queue_depth * 8 * mpt->m_post_reply_qcount;
2672 2975  
2673 2976          /*
2674 2977           * set the DMA attributes.  The Reply Descriptor Post Queue must be
2675 2978           * aligned on a 16-byte boundry.
2676 2979           */
2677 2980          frame_dma_attrs = mpt->m_msg_dma_attr;
2678 2981          frame_dma_attrs.dma_attr_align = 16;
2679 2982          frame_dma_attrs.dma_attr_sgllen = 1;
2680 2983  
2681 2984          /*
2682      -         * allocate the reply post queue
     2985 +         * Allocate the reply post queue(s).
     2986 +         * MPI2.5 introduces a method to allocate multiple queues
     2987 +         * using a redirect table. For now stick to one contiguous
     2988 +         * chunck. This can get as big as 1Mbyte for 16 queues.
     2989 +         * The spec gives no indication that the queue size can be
     2990 +         * reduced if you have many of them.
2683 2991           */
2684 2992          if (mptsas_dma_addr_create(mpt, frame_dma_attrs,
2685 2993              &mpt->m_dma_post_queue_hdl, &mpt->m_acc_post_queue_hdl, &memp,
2686 2994              mem_size, &cookie) == FALSE) {
2687 2995                  return (DDI_FAILURE);
2688 2996          }
2689 2997  
2690 2998          /*
2691 2999           * Store the reply descriptor post queue memory address.  This chip
2692 3000           * uses this address to write to the reply descriptor post queue.  The
2693 3001           * second address is the address mpt uses to manage the queue.
2694 3002           */
2695 3003          mpt->m_post_queue_dma_addr = cookie.dmac_laddress;
2696 3004          mpt->m_post_queue = memp;
2697 3005  
     3006 +        mpt->m_rep_post_queues = kmem_zalloc(sizeof (mptsas_reply_pqueue_t) *
     3007 +            mpt->m_post_reply_qcount, KM_SLEEP);
     3008 +        rpqp = mpt->m_rep_post_queues;
     3009 +        for (i = 0; i < mpt->m_post_reply_qcount; i++) {
     3010 +                rpqp->rpq_queue = memp;
     3011 +                mutex_init(&rpqp->rpq_mutex, NULL, MUTEX_DRIVER, NULL);
     3012 +                rpqp->rpq_dlist.dl_tail = &rpqp->rpq_dlist.dl_q;
     3013 +                rpqp->rpq_num = (uint8_t)i;
     3014 +                memp += (mpt->m_post_queue_depth * 8);
     3015 +                rpqp++;
     3016 +        }
     3017 +
2698 3018          /*
2699 3019           * Clear the reply post queue memory.
2700 3020           */
2701 3021          bzero(mpt->m_post_queue, mem_size);
2702 3022  
     3023 +        mpt->m_dma_flags |= MPTSAS_POST_QUEUE;
2703 3024          return (DDI_SUCCESS);
2704 3025  }
2705 3026  
2706 3027  static void
2707 3028  mptsas_alloc_reply_args(mptsas_t *mpt)
2708 3029  {
2709 3030          if (mpt->m_replyh_args == NULL) {
2710 3031                  mpt->m_replyh_args = kmem_zalloc(sizeof (m_replyh_arg_t) *
2711 3032                      mpt->m_max_replies, KM_SLEEP);
     3033 +        } else {
     3034 +                bzero(mpt->m_replyh_args, sizeof (m_replyh_arg_t) *
     3035 +                    mpt->m_max_replies);
2712 3036          }
2713 3037  }
2714 3038  
2715 3039  static int
2716 3040  mptsas_alloc_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd)
2717 3041  {
2718 3042          mptsas_cache_frames_t   *frames = NULL;
2719 3043          if (cmd->cmd_extra_frames == NULL) {
2720 3044                  frames = kmem_cache_alloc(mpt->m_cache_frames, KM_NOSLEEP);
2721 3045                  if (frames == NULL) {
↓ open down ↓ 22 lines elided ↑ open up ↑
2744 3068  }
2745 3069  
2746 3070  static void
2747 3071  mptsas_hba_fini(mptsas_t *mpt)
2748 3072  {
2749 3073          NDBG0(("mptsas_hba_fini"));
2750 3074  
2751 3075          /*
2752 3076           * Free up any allocated memory
2753 3077           */
2754      -        mptsas_dma_addr_destroy(&mpt->m_dma_req_frame_hdl,
2755      -            &mpt->m_acc_req_frame_hdl);
     3078 +        if (mpt->m_dma_flags & MPTSAS_REQ_FRAME) {
     3079 +                mptsas_dma_addr_destroy(&mpt->m_dma_req_frame_hdl,
     3080 +                    &mpt->m_acc_req_frame_hdl);
     3081 +        }
     3082 +
     3083 +        if (mpt->m_dma_flags & MPTSAS_REQ_SENSE) {
     3084 +                rmfreemap(mpt->m_erqsense_map);
     3085 +                mptsas_dma_addr_destroy(&mpt->m_dma_req_sense_hdl,
     3086 +                    &mpt->m_acc_req_sense_hdl);
     3087 +        }
2756 3088  
2757      -        mptsas_dma_addr_destroy(&mpt->m_dma_reply_frame_hdl,
2758      -            &mpt->m_acc_reply_frame_hdl);
     3089 +        if (mpt->m_dma_flags & MPTSAS_REPLY_FRAME) {
     3090 +                mptsas_dma_addr_destroy(&mpt->m_dma_reply_frame_hdl,
     3091 +                    &mpt->m_acc_reply_frame_hdl);
     3092 +        }
2759 3093  
2760      -        mptsas_dma_addr_destroy(&mpt->m_dma_free_queue_hdl,
2761      -            &mpt->m_acc_free_queue_hdl);
     3094 +        if (mpt->m_dma_flags & MPTSAS_FREE_QUEUE) {
     3095 +                mptsas_dma_addr_destroy(&mpt->m_dma_free_queue_hdl,
     3096 +                    &mpt->m_acc_free_queue_hdl);
     3097 +        }
2762 3098  
2763      -        mptsas_dma_addr_destroy(&mpt->m_dma_post_queue_hdl,
2764      -            &mpt->m_acc_post_queue_hdl);
     3099 +        mptsas_free_post_queue(mpt);
2765 3100  
2766 3101          if (mpt->m_replyh_args != NULL) {
2767 3102                  kmem_free(mpt->m_replyh_args, sizeof (m_replyh_arg_t)
2768 3103                      * mpt->m_max_replies);
2769 3104          }
2770 3105  }
2771 3106  
2772 3107  static int
2773 3108  mptsas_name_child(dev_info_t *lun_dip, char *name, int len)
2774 3109  {
↓ open down ↓ 258 lines elided ↑ open up ↑
3033 3368   *      - if NO_INTR, tran_start must poll device for command completion
3034 3369   */
3035 3370  static int
3036 3371  mptsas_scsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
3037 3372  {
3038 3373  #ifndef __lock_lint
3039 3374          _NOTE(ARGUNUSED(ap))
3040 3375  #endif
3041 3376          mptsas_t        *mpt = PKT2MPT(pkt);
3042 3377          mptsas_cmd_t    *cmd = PKT2CMD(pkt);
3043      -        int             rval;
     3378 +        int             rval, start;
     3379 +        uint8_t         pref;
3044 3380          mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
     3381 +        mptsas_tx_waitqueue_t *txwq;
3045 3382  
3046 3383          NDBG1(("mptsas_scsi_start: pkt=0x%p", (void *)pkt));
3047 3384          ASSERT(ptgt);
3048 3385          if (ptgt == NULL)
3049 3386                  return (TRAN_FATAL_ERROR);
3050 3387  
3051 3388          /*
3052 3389           * prepare the pkt before taking mutex.
3053 3390           */
3054 3391          rval = mptsas_prepare_pkt(cmd);
↓ open down ↓ 28 lines elided ↑ open up ↑
3083 3420           * Under the tx_waitq mutex, record whether a thread is draining
3084 3421           * the tx_waitq.  An IO requesting thread that finds the instance
3085 3422           * mutex contended appends to the tx_waitq and while holding the
3086 3423           * tx_wait mutex, if the draining flag is not set, sets it and then
3087 3424           * proceeds to spin for the instance mutex. This scheme ensures that
3088 3425           * the last cmd in a burst be processed.
3089 3426           *
3090 3427           * we enable this feature only when the helper threads are enabled,
3091 3428           * at which we think the loads are heavy.
3092 3429           *
3093      -         * per instance mutex m_tx_waitq_mutex is introduced to protect the
3094      -         * m_tx_waitqtail, m_tx_waitq, m_tx_draining.
     3430 +         * per instance, per queue mutex m_tx_waitq[i].txwq_mutex is
     3431 +         * introduced to protect the txwq_qtail, txwq_cmdq, txwq_len
3095 3432           */
3096 3433  
3097      -        if (mpt->m_doneq_thread_n) {
3098      -                if (mutex_tryenter(&mpt->m_mutex) != 0) {
3099      -                        rval = mptsas_accept_txwq_and_pkt(mpt, cmd);
3100      -                        mutex_exit(&mpt->m_mutex);
3101      -                } else if (cmd->cmd_pkt_flags & FLAG_NOINTR) {
3102      -                        mutex_enter(&mpt->m_mutex);
3103      -                        rval = mptsas_accept_txwq_and_pkt(mpt, cmd);
3104      -                        mutex_exit(&mpt->m_mutex);
3105      -                } else {
3106      -                        mutex_enter(&mpt->m_tx_waitq_mutex);
3107      -                        /*
3108      -                         * ptgt->m_dr_flag is protected by m_mutex or
3109      -                         * m_tx_waitq_mutex. In this case, m_tx_waitq_mutex
3110      -                         * is acquired.
3111      -                         */
3112      -                        if (ptgt->m_dr_flag == MPTSAS_DR_INTRANSITION) {
3113      -                                if (cmd->cmd_pkt_flags & FLAG_NOQUEUE) {
3114      -                                        /*
3115      -                                         * The command should be allowed to
3116      -                                         * retry by returning TRAN_BUSY to
3117      -                                         * to stall the I/O's which come from
3118      -                                         * scsi_vhci since the device/path is
3119      -                                         * in unstable state now.
3120      -                                         */
3121      -                                        mutex_exit(&mpt->m_tx_waitq_mutex);
3122      -                                        return (TRAN_BUSY);
3123      -                                } else {
3124      -                                        /*
3125      -                                         * The device is offline, just fail the
3126      -                                         * command by returning
3127      -                                         * TRAN_FATAL_ERROR.
3128      -                                         */
3129      -                                        mutex_exit(&mpt->m_tx_waitq_mutex);
3130      -                                        return (TRAN_FATAL_ERROR);
     3434 +        if (mpt->m_txwq_enabled == TRUE) {
     3435 +                int gotmtx = 0;
     3436 +
     3437 +                if (mpt->m_txwq_allow_q_jumping) {
     3438 +                        gotmtx = mutex_tryenter(&mpt->m_mutex);
     3439 +                }
     3440 +                if (gotmtx == 0) {
     3441 +                        /* We didn't get the mutex or didn't try */
     3442 +                        if (cmd->cmd_pkt_flags & FLAG_NOINTR) {
     3443 +                                mutex_enter(&mpt->m_mutex);
     3444 +                                /* Polled commands queue jump */
     3445 +                                mptsas_accept_tx_waitqs(mpt);
     3446 +                        } else {
     3447 +                                rval = mptsas_check_targ_intxtion(
     3448 +                                    cmd->cmd_tgt_addr,
     3449 +                                    cmd->cmd_pkt_flags);
     3450 +                                if (rval != TRAN_ACCEPT) {
     3451 +                                        return (rval);
3131 3452                                  }
3132      -                        }
3133      -                        if (mpt->m_tx_draining) {
     3453 +
3134 3454                                  cmd->cmd_flags |= CFLAG_TXQ;
3135      -                                *mpt->m_tx_waitqtail = cmd;
3136      -                                mpt->m_tx_waitqtail = &cmd->cmd_linkp;
3137      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
3138      -                        } else { /* drain the queue */
3139      -                                mpt->m_tx_draining = 1;
3140      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
3141      -                                mutex_enter(&mpt->m_mutex);
3142      -                                rval = mptsas_accept_txwq_and_pkt(mpt, cmd);
3143      -                                mutex_exit(&mpt->m_mutex);
     3455 +                                pref = mpt->m_pref_tx_waitq;
     3456 +                                txwq = &mpt->m_tx_waitq[pref];
     3457 +
     3458 +                                if (mutex_tryenter(&txwq->txwq_mutex) == 0) {
     3459 +                                        txwq = &mpt->m_tx_waitq[pref^1];
     3460 +                                        mutex_enter(&txwq->txwq_mutex);
     3461 +                                } else {
     3462 +                                        pref ^= 1;
     3463 +                                        mpt->m_pref_tx_waitq = pref;
     3464 +                                }
     3465 +
     3466 +                                *txwq->txwq_qtail = cmd;
     3467 +                                txwq->txwq_qtail = &cmd->cmd_linkp;
     3468 +                                txwq->txwq_len++;
     3469 +                                if (!txwq->txwq_draining) {
     3470 +                                        cv_signal(&txwq->txwq_cv);
     3471 +                                }
     3472 +                                mutex_exit(&txwq->txwq_mutex);
     3473 +                                return (rval);
3144 3474                          }
3145 3475                  }
3146 3476          } else {
3147 3477                  mutex_enter(&mpt->m_mutex);
3148      -                /*
3149      -                 * ptgt->m_dr_flag is protected by m_mutex or m_tx_waitq_mutex
3150      -                 * in this case, m_mutex is acquired.
3151      -                 */
3152      -                if (ptgt->m_dr_flag == MPTSAS_DR_INTRANSITION) {
3153      -                        if (cmd->cmd_pkt_flags & FLAG_NOQUEUE) {
3154      -                                /*
3155      -                                 * commands should be allowed to retry by
3156      -                                 * returning TRAN_BUSY to stall the I/O's
3157      -                                 * which come from scsi_vhci since the device/
3158      -                                 * path is in unstable state now.
3159      -                                 */
3160      -                                mutex_exit(&mpt->m_mutex);
3161      -                                return (TRAN_BUSY);
3162      -                        } else {
3163      -                                /*
3164      -                                 * The device is offline, just fail the
3165      -                                 * command by returning TRAN_FATAL_ERROR.
3166      -                                 */
3167      -                                mutex_exit(&mpt->m_mutex);
3168      -                                return (TRAN_FATAL_ERROR);
3169      -                        }
3170      -                }
3171      -                rval = mptsas_accept_pkt(mpt, cmd);
     3478 +        }
     3479 +        rval = mptsas_check_targ_intxtion(cmd->cmd_tgt_addr,
     3480 +            cmd->cmd_pkt_flags);
     3481 +        if (rval != TRAN_ACCEPT) {
3172 3482                  mutex_exit(&mpt->m_mutex);
     3483 +                return (rval);
     3484 +        }
     3485 +
     3486 +        start = mptsas_accept_pkt(mpt, cmd, &rval);
     3487 +        mutex_exit(&mpt->m_mutex);
     3488 +        if (start) {
     3489 +                (void) mptsas_start_cmd(mpt, cmd);
3173 3490          }
3174 3491  
3175 3492          return (rval);
3176 3493  }
3177 3494  
3178      -/*
3179      - * Accept all the queued cmds(if any) before accept the current one.
3180      - */
3181 3495  static int
3182      -mptsas_accept_txwq_and_pkt(mptsas_t *mpt, mptsas_cmd_t *cmd)
     3496 +mptsas_check_targ_intxtion(mptsas_target_t *ptgt, int cmd_pkt_flags)
3183 3497  {
3184      -        int rval;
3185      -        mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
3186      -
3187      -        ASSERT(mutex_owned(&mpt->m_mutex));
3188 3498          /*
3189      -         * The call to mptsas_accept_tx_waitq() must always be performed
3190      -         * because that is where mpt->m_tx_draining is cleared.
3191      -         */
3192      -        mutex_enter(&mpt->m_tx_waitq_mutex);
3193      -        mptsas_accept_tx_waitq(mpt);
3194      -        mutex_exit(&mpt->m_tx_waitq_mutex);
3195      -        /*
3196      -         * ptgt->m_dr_flag is protected by m_mutex or m_tx_waitq_mutex
3197      -         * in this case, m_mutex is acquired.
     3499 +         * ptgt->m_dr_flag is a variable that is only ever changed by
     3500 +         * direct write under the main m_mutex.
     3501 +         * It doesn't need a mutex hold to protect this read.
3198 3502           */
     3503 +
3199 3504          if (ptgt->m_dr_flag == MPTSAS_DR_INTRANSITION) {
3200      -                if (cmd->cmd_pkt_flags & FLAG_NOQUEUE) {
     3505 +                if (cmd_pkt_flags & FLAG_NOQUEUE) {
3201 3506                          /*
3202 3507                           * The command should be allowed to retry by returning
3203 3508                           * TRAN_BUSY to stall the I/O's which come from
3204 3509                           * scsi_vhci since the device/path is in unstable state
3205 3510                           * now.
3206 3511                           */
3207 3512                          return (TRAN_BUSY);
3208 3513                  } else {
3209 3514                          /*
3210 3515                           * The device is offline, just fail the command by
3211 3516                           * return TRAN_FATAL_ERROR.
3212 3517                           */
3213 3518                          return (TRAN_FATAL_ERROR);
3214 3519                  }
3215 3520          }
3216      -        rval = mptsas_accept_pkt(mpt, cmd);
     3521 +        return (TRAN_ACCEPT);
     3522 +}
3217 3523  
3218      -        return (rval);
     3524 +/*
     3525 + * Note that this function has a side effect of releasing the
     3526 + * per target mutex.
     3527 + */
     3528 +static void
     3529 +mptsas_offline_target_direct(mptsas_t *mpt, mptsas_target_t *ptgt)
     3530 +{
     3531 +        char                            phy_mask_name[MPTSAS_MAX_PHYS];
     3532 +        mptsas_phymask_t                phymask = ptgt->m_addr.mta_phymask;
     3533 +        dev_info_t                      *parent;
     3534 +
     3535 +        ASSERT(mutex_owned(&mpt->m_mutex));
     3536 +
     3537 +        ptgt->m_dr_flag = MPTSAS_DR_INTRANSITION;
     3538 +        bzero(phy_mask_name, MPTSAS_MAX_PHYS);
     3539 +        (void) sprintf(phy_mask_name, "%x", phymask);
     3540 +        parent = scsi_hba_iport_find(mpt->m_dip, phy_mask_name);
     3541 +
     3542 +        if (parent != NULL) {
     3543 +                mptsas_offline_target(mpt, ptgt,
     3544 +                    ptgt->m_deviceinfo & DEVINFO_DIRECT_ATTACHED ?
     3545 +                    MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE :
     3546 +                    MPTSAS_TOPO_FLAG_EXPANDER_ATTACHED_DEVICE,
     3547 +                    parent);
     3548 +        } else {
     3549 +                mptsas_log(mpt, CE_WARN, "Failed to find an "
     3550 +                    "iport for \"%s\", should not happen!",
     3551 +                    phy_mask_name);
     3552 +        }
3219 3553  }
3220 3554  
     3555 +/*
     3556 + * In order to be efficient with the m_mutex (which can be dropped before
     3557 + * calling mptsas_start_cmd()) indicate if start_cmd should be called via the
     3558 + * returned value (FALSE or TRUE). Caller is then responsible for doing the
     3559 + * right thing with the m_mutex.
     3560 + */
3221 3561  static int
3222      -mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *cmd)
     3562 +mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *cmd, int *tran_rval)
3223 3563  {
3224 3564          int             rval = TRAN_ACCEPT;
3225 3565          mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
3226 3566  
3227 3567          NDBG1(("mptsas_accept_pkt: cmd=0x%p", (void *)cmd));
3228 3568  
3229 3569          ASSERT(mutex_owned(&mpt->m_mutex));
3230 3570  
3231 3571          if ((cmd->cmd_flags & CFLAG_PREPARED) == 0) {
3232 3572                  rval = mptsas_prepare_pkt(cmd);
3233 3573                  if (rval != TRAN_ACCEPT) {
3234 3574                          cmd->cmd_flags &= ~CFLAG_TRANFLAG;
3235      -                        return (rval);
     3575 +                        goto set_tranrval;
3236 3576                  }
3237 3577          }
3238 3578  
3239 3579          /*
3240      -         * reset the throttle if we were draining
     3580 +         * If the command came from the tx wait q it may have slipped
     3581 +         * by the check for dr_flag before being added to the queue.
     3582 +         * Fail here with abort status.
3241 3583           */
3242      -        if ((ptgt->m_t_ncmds == 0) &&
3243      -            (ptgt->m_t_throttle == DRAIN_THROTTLE)) {
3244      -                NDBG23(("reset throttle"));
3245      -                ASSERT(ptgt->m_reset_delay == 0);
3246      -                mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     3584 +        if (cmd->cmd_flags & CFLAG_TXQ) {
     3585 +                rval = mptsas_check_targ_intxtion(cmd->cmd_tgt_addr,
     3586 +                    cmd->cmd_pkt_flags);
     3587 +                if (rval != TRAN_ACCEPT) {
     3588 +                        mptsas_set_pkt_reason(mpt, cmd, CMD_ABORTED,
     3589 +                            STAT_ABORTED);
     3590 +                        mptsas_doneq_add(mpt, cmd);
     3591 +                        mptsas_doneq_empty(mpt);
     3592 +                        goto set_tranrval;
     3593 +                }
3247 3594          }
3248      -
3249 3595          /*
3250 3596           * If HBA is being reset, the DevHandles are being re-initialized,
3251 3597           * which means that they could be invalid even if the target is still
3252 3598           * attached.  Check if being reset and if DevHandle is being
3253 3599           * re-initialized.  If this is the case, return BUSY so the I/O can be
3254 3600           * retried later.
3255 3601           */
3256 3602          if ((ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) && mpt->m_in_reset) {
3257 3603                  mptsas_set_pkt_reason(mpt, cmd, CMD_RESET, STAT_BUS_RESET);
3258 3604                  if (cmd->cmd_flags & CFLAG_TXQ) {
3259 3605                          mptsas_doneq_add(mpt, cmd);
3260 3606                          mptsas_doneq_empty(mpt);
3261      -                        return (rval);
3262 3607                  } else {
3263      -                        return (TRAN_BUSY);
     3608 +                        rval = TRAN_BUSY;
3264 3609                  }
     3610 +                goto set_tranrval;
     3611 +        }
     3612 +
     3613 +        mutex_enter(&ptgt->m_t_mutex);
     3614 +        /*
     3615 +         * reset the throttle if we were draining
     3616 +         */
     3617 +        if ((ptgt->m_t_ncmds == 0) &&
     3618 +            (ptgt->m_t_throttle == DRAIN_THROTTLE)) {
     3619 +                NDBG23(("reset throttle"));
     3620 +                ASSERT(ptgt->m_reset_delay == 0);
     3621 +                mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
3265 3622          }
3266 3623  
3267 3624          /*
3268 3625           * If device handle has already been invalidated, just
3269      -         * fail the command. In theory, command from scsi_vhci
3270      -         * client is impossible send down command with invalid
     3626 +         * fail the command. In theory, for a command from scsi_vhci
     3627 +         * client it's impossible to receive a command with an invalid
3271 3628           * devhdl since devhdl is set after path offline, target
3272      -         * driver is not suppose to select a offlined path.
     3629 +         * driver is not supposed to select an offlined path.
3273 3630           */
3274 3631          if (ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) {
3275      -                NDBG20(("rejecting command, it might because invalid devhdl "
     3632 +                NDBG3(("rejecting command, it might because invalid devhdl "
3276 3633                      "request."));
     3634 +                mutex_exit(&ptgt->m_t_mutex);
3277 3635                  mptsas_set_pkt_reason(mpt, cmd, CMD_DEV_GONE, STAT_TERMINATED);
3278 3636                  if (cmd->cmd_flags & CFLAG_TXQ) {
3279 3637                          mptsas_doneq_add(mpt, cmd);
3280 3638                          mptsas_doneq_empty(mpt);
3281      -                        return (rval);
3282 3639                  } else {
3283      -                        return (TRAN_FATAL_ERROR);
     3640 +                        rval = TRAN_FATAL_ERROR;
3284 3641                  }
     3642 +                goto set_tranrval;
3285 3643          }
3286 3644          /*
3287 3645           * The first case is the normal case.  mpt gets a command from the
3288 3646           * target driver and starts it.
3289 3647           * Since SMID 0 is reserved and the TM slot is reserved, the actual max
3290 3648           * commands is m_max_requests - 2.
3291 3649           */
3292 3650          if ((mpt->m_ncmds <= (mpt->m_max_requests - 2)) &&
3293 3651              (ptgt->m_t_throttle > HOLD_THROTTLE) &&
3294 3652              (ptgt->m_t_ncmds < ptgt->m_t_throttle) &&
3295      -            (ptgt->m_reset_delay == 0) &&
     3653 +            (ptgt->m_reset_delay == 0) && (mpt->m_polled_intr == 0) &&
3296 3654              (ptgt->m_t_nwait == 0) &&
3297 3655              ((cmd->cmd_pkt_flags & FLAG_NOINTR) == 0)) {
3298      -                if (mptsas_save_cmd(mpt, cmd) == TRUE) {
3299      -                        (void) mptsas_start_cmd(mpt, cmd);
     3656 +                ASSERT((cmd->cmd_flags & CFLAG_CMDIOC) == 0);
     3657 +                if (mptsas_save_cmd_to_slot(mpt, cmd) == TRUE) {
     3658 +                        ptgt->m_t_ncmds++;
     3659 +                        mutex_exit(&ptgt->m_t_mutex);
     3660 +                        cmd->cmd_active_expiration = 0;
     3661 +                        *tran_rval = rval;
     3662 +                        return (TRUE);
3300 3663                  } else {
     3664 +                        mutex_exit(&ptgt->m_t_mutex);
3301 3665                          mptsas_waitq_add(mpt, cmd);
3302 3666                  }
3303 3667          } else {
     3668 +                mutex_exit(&ptgt->m_t_mutex);
3304 3669                  /*
3305 3670                   * Add this pkt to the work queue
3306 3671                   */
3307 3672                  mptsas_waitq_add(mpt, cmd);
3308 3673  
3309 3674                  if (cmd->cmd_pkt_flags & FLAG_NOINTR) {
3310 3675                          (void) mptsas_poll(mpt, cmd, MPTSAS_POLL_TIME);
3311 3676  
3312 3677                          /*
3313 3678                           * Only flush the doneq if this is not a TM
3314 3679                           * cmd.  For TM cmds the flushing of the
3315 3680                           * doneq will be done in those routines.
3316 3681                           */
3317 3682                          if ((cmd->cmd_flags & CFLAG_TM_CMD) == 0) {
3318 3683                                  mptsas_doneq_empty(mpt);
3319 3684                          }
3320 3685                  }
3321 3686          }
3322      -        return (rval);
     3687 +set_tranrval:
     3688 +        *tran_rval = rval;
     3689 +        return (FALSE);
3323 3690  }
3324 3691  
3325      -int
3326      -mptsas_save_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
     3692 +static void
     3693 +mptsas_retry_pkt(mptsas_t *mpt, mptsas_cmd_t *cmd)
3327 3694  {
3328      -        mptsas_slots_t *slots = mpt->m_active;
3329      -        uint_t slot, start_rotor;
3330      -        mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
     3695 +        int             rval;
3331 3696  
3332      -        ASSERT(MUTEX_HELD(&mpt->m_mutex));
     3697 +        cmd->cmd_pkt_flags |= FLAG_HEAD;
     3698 +        cmd->cmd_flags |= CFLAG_RETRY;
     3699 +        cmd->cmd_flags &= ~CFLAG_TXQ;
     3700 +        if (mptsas_accept_pkt(mpt, cmd, &rval)) {
     3701 +                (void) mptsas_start_cmd(mpt, cmd);
     3702 +        }
     3703 +
     3704 +        /*
     3705 +         * If there was a problem clear the retry flag so that the
     3706 +         * command will be completed with error rather than get lost!
     3707 +         */
     3708 +        if (rval != TRAN_ACCEPT)
     3709 +                cmd->cmd_flags &= ~CFLAG_RETRY;
     3710 +}
     3711 +
     3712 +static int
     3713 +mptsas_save_cmd_to_slot(mptsas_t *mpt, mptsas_cmd_t *cmd)
     3714 +{
     3715 +        mptsas_slots_t *slots = mpt->m_active;
     3716 +        uint_t slot, start_rotor, rotor, n_normal;
3333 3717  
3334 3718          /*
3335 3719           * Account for reserved TM request slot and reserved SMID of 0.
3336 3720           */
3337 3721          ASSERT(slots->m_n_normal == (mpt->m_max_requests - 2));
3338 3722  
3339 3723          /*
3340 3724           * Find the next available slot, beginning at m_rotor.  If no slot is
3341 3725           * available, we'll return FALSE to indicate that.  This mechanism
3342 3726           * considers only the normal slots, not the reserved slot 0 nor the
3343 3727           * task management slot m_n_normal + 1.  The rotor is left to point to
3344 3728           * the normal slot after the one we select, unless we select the last
3345 3729           * normal slot in which case it returns to slot 1.
3346 3730           */
3347      -        start_rotor = slots->m_rotor;
     3731 +        start_rotor = rotor = slots->m_rotor;
     3732 +        n_normal = slots->m_n_normal;
3348 3733          do {
3349      -                slot = slots->m_rotor++;
3350      -                if (slots->m_rotor > slots->m_n_normal)
3351      -                        slots->m_rotor = 1;
     3734 +                slot = rotor++;
     3735 +                if (rotor > n_normal)
     3736 +                        rotor = 1;
3352 3737  
3353      -                if (slots->m_rotor == start_rotor)
     3738 +                if (rotor == start_rotor)
3354 3739                          break;
3355 3740          } while (slots->m_slot[slot] != NULL);
     3741 +        slots->m_rotor = rotor;
3356 3742  
3357 3743          if (slots->m_slot[slot] != NULL)
3358 3744                  return (FALSE);
3359 3745  
3360 3746          ASSERT(slot != 0 && slot <= slots->m_n_normal);
3361 3747  
3362 3748          cmd->cmd_slot = slot;
3363 3749          slots->m_slot[slot] = cmd;
3364      -        mpt->m_ncmds++;
     3750 +        atomic_inc_32(&mpt->m_ncmds);
3365 3751  
3366 3752          /*
3367      -         * only increment per target ncmds if this is not a
3368      -         * command that has no target associated with it (i.e. a
3369      -         * event acknoledgment)
     3753 +         * Distribute the commands amongst the reply queues (Interrupt vectors).
     3754 +         * Stick to 0 for polled.
3370 3755           */
3371      -        if ((cmd->cmd_flags & CFLAG_CMDIOC) == 0) {
3372      -                ptgt->m_t_ncmds++;
     3756 +        if (!(cmd->cmd_pkt_flags & FLAG_NOINTR) &&
     3757 +            !(cmd->cmd_flags & (CFLAG_PASSTHRU|CFLAG_CONFIG|CFLAG_FW_DIAG)) &&
     3758 +            (mpt->m_post_reply_qcount > 1)) {
     3759 +                cmd->cmd_rpqidx = slot % mpt->m_post_reply_qcount;
     3760 +        }
     3761 +        atomic_inc_32(&mpt->m_rep_post_queues[cmd->cmd_rpqidx].rpq_ncmds);
     3762 +        return (TRUE);
     3763 +}
     3764 +
     3765 +int
     3766 +mptsas_save_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
     3767 +{
     3768 +        mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
     3769 +
     3770 +        ASSERT(MUTEX_HELD(&mpt->m_mutex));
     3771 +
     3772 +        if (!mptsas_save_cmd_to_slot(mpt, cmd)) {
     3773 +                return (FALSE);
3373 3774          }
3374      -        cmd->cmd_active_timeout = cmd->cmd_pkt->pkt_time;
3375 3775  
3376 3776          /*
3377      -         * If initial timout is less than or equal to one tick, bump
3378      -         * the timeout by a tick so that command doesn't timeout before
3379      -         * its allotted time.
     3777 +         * only increment per target ncmds if this is not a
     3778 +         * command that has no target associated with it (i.e. a
     3779 +         * event acknoledgement)
3380 3780           */
3381      -        if (cmd->cmd_active_timeout <= mptsas_scsi_watchdog_tick) {
3382      -                cmd->cmd_active_timeout += mptsas_scsi_watchdog_tick;
     3781 +        if ((cmd->cmd_flags & CFLAG_CMDIOC) == 0) {
     3782 +                /*
     3783 +                 * Expiration time is set in mptsas_start_cmd
     3784 +                 */
     3785 +                mutex_enter(&ptgt->m_t_mutex);
     3786 +                ptgt->m_t_ncmds++;
     3787 +                mutex_exit(&ptgt->m_t_mutex);
     3788 +                cmd->cmd_active_expiration = 0;
     3789 +        } else {
     3790 +                /*
     3791 +                 * Initialize expiration time for passthrough commands,
     3792 +                 */
     3793 +                cmd->cmd_active_expiration = gethrtime() +
     3794 +                    (hrtime_t)cmd->cmd_pkt->pkt_time * NANOSEC;
3383 3795          }
3384 3796          return (TRUE);
3385 3797  }
3386 3798  
3387 3799  /*
3388 3800   * prepare the pkt:
3389 3801   * the pkt may have been resubmitted or just reused so
3390 3802   * initialize some fields and do some checks.
3391 3803   */
3392 3804  static int
↓ open down ↓ 84 lines elided ↑ open up ↑
3477 3889          NDBG3(("mptsas_scsi_init_pkt:\n"
3478 3890              "\ttgt=%d in=0x%p bp=0x%p clen=%d slen=%d tlen=%d flags=%x",
3479 3891              ap->a_target, (void *)pkt, (void *)bp,
3480 3892              cmdlen, statuslen, tgtlen, flags));
3481 3893  
3482 3894          /*
3483 3895           * Allocate the new packet.
3484 3896           */
3485 3897          if (pkt == NULL) {
3486 3898                  ddi_dma_handle_t        save_dma_handle;
3487      -                ddi_dma_handle_t        save_arq_dma_handle;
3488      -                struct buf              *save_arq_bp;
3489      -                ddi_dma_cookie_t        save_arqcookie;
3490 3899  
3491 3900                  cmd = kmem_cache_alloc(mpt->m_kmem_cache, kf);
3492 3901  
3493 3902                  if (cmd) {
3494 3903                          save_dma_handle = cmd->cmd_dmahandle;
3495      -                        save_arq_dma_handle = cmd->cmd_arqhandle;
3496      -                        save_arq_bp = cmd->cmd_arq_buf;
3497      -                        save_arqcookie = cmd->cmd_arqcookie;
3498 3904                          bzero(cmd, sizeof (*cmd) + scsi_pkt_size());
3499 3905                          cmd->cmd_dmahandle = save_dma_handle;
3500      -                        cmd->cmd_arqhandle = save_arq_dma_handle;
3501      -                        cmd->cmd_arq_buf = save_arq_bp;
3502      -                        cmd->cmd_arqcookie = save_arqcookie;
3503 3906  
3504 3907                          pkt = (void *)((uchar_t *)cmd +
3505 3908                              sizeof (struct mptsas_cmd));
3506 3909                          pkt->pkt_ha_private = (opaque_t)cmd;
3507 3910                          pkt->pkt_address = *ap;
3508 3911                          pkt->pkt_private = (opaque_t)cmd->cmd_pkt_private;
3509 3912                          pkt->pkt_scbp = (opaque_t)&cmd->cmd_scb;
3510 3913                          pkt->pkt_cdbp = (opaque_t)&cmd->cmd_cdb;
3511 3914                          cmd->cmd_pkt = (struct scsi_pkt *)pkt;
3512 3915                          cmd->cmd_cdblen = (uchar_t)cmdlen;
↓ open down ↓ 259 lines elided ↑ open up ↑
3772 4175                          dmap->addr.address64.High = (uint32_t)
3773 4176                              (cmd->cmd_cookie.dmac_laddress >> 32);
3774 4177                  }
3775 4178  
3776 4179                  /*
3777 4180                   * If this was partially allocated we set the resid
3778 4181                   * the amount of data NOT transferred in this window
3779 4182                   * If there is only one window, the resid will be 0
3780 4183                   */
3781 4184                  pkt->pkt_resid = (bp->b_bcount - cmd->cmd_totaldmacount);
3782      -                NDBG16(("mptsas_dmaget: cmd_dmacount=%d.", cmd->cmd_dmacount));
     4185 +                NDBG3(("mptsas_scsi_init_pkt: cmd_dmacount=%d.",
     4186 +                    cmd->cmd_dmacount));
3783 4187          }
3784 4188          return (pkt);
3785 4189  }
3786 4190  
3787 4191  /*
3788 4192   * tran_destroy_pkt(9E) - scsi_pkt(9s) deallocation
3789 4193   *
3790 4194   * Notes:
3791 4195   *      - also frees DMA resources if allocated
3792 4196   *      - implicit DMA synchonization
↓ open down ↓ 32 lines elided ↑ open up ↑
3825 4229  /*
3826 4230   * kmem cache constructor and destructor:
3827 4231   * When constructing, we bzero the cmd and allocate the dma handle
3828 4232   * When destructing, just free the dma handle
3829 4233   */
3830 4234  static int
3831 4235  mptsas_kmem_cache_constructor(void *buf, void *cdrarg, int kmflags)
3832 4236  {
3833 4237          mptsas_cmd_t            *cmd = buf;
3834 4238          mptsas_t                *mpt  = cdrarg;
3835      -        struct scsi_address     ap;
3836      -        uint_t                  cookiec;
3837      -        ddi_dma_attr_t          arq_dma_attr;
3838 4239          int                     (*callback)(caddr_t);
3839 4240  
3840 4241          callback = (kmflags == KM_SLEEP)? DDI_DMA_SLEEP: DDI_DMA_DONTWAIT;
3841 4242  
3842 4243          NDBG4(("mptsas_kmem_cache_constructor"));
3843 4244  
3844      -        ap.a_hba_tran = mpt->m_tran;
3845      -        ap.a_target = 0;
3846      -        ap.a_lun = 0;
3847      -
3848 4245          /*
3849 4246           * allocate a dma handle
3850 4247           */
3851 4248          if ((ddi_dma_alloc_handle(mpt->m_dip, &mpt->m_io_dma_attr, callback,
3852 4249              NULL, &cmd->cmd_dmahandle)) != DDI_SUCCESS) {
3853 4250                  cmd->cmd_dmahandle = NULL;
3854 4251                  return (-1);
3855 4252          }
3856      -
3857      -        cmd->cmd_arq_buf = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL,
3858      -            SENSE_LENGTH, B_READ, callback, NULL);
3859      -        if (cmd->cmd_arq_buf == NULL) {
3860      -                ddi_dma_free_handle(&cmd->cmd_dmahandle);
3861      -                cmd->cmd_dmahandle = NULL;
3862      -                return (-1);
3863      -        }
3864      -
3865      -        /*
3866      -         * allocate a arq handle
3867      -         */
3868      -        arq_dma_attr = mpt->m_msg_dma_attr;
3869      -        arq_dma_attr.dma_attr_sgllen = 1;
3870      -        if ((ddi_dma_alloc_handle(mpt->m_dip, &arq_dma_attr, callback,
3871      -            NULL, &cmd->cmd_arqhandle)) != DDI_SUCCESS) {
3872      -                ddi_dma_free_handle(&cmd->cmd_dmahandle);
3873      -                scsi_free_consistent_buf(cmd->cmd_arq_buf);
3874      -                cmd->cmd_dmahandle = NULL;
3875      -                cmd->cmd_arqhandle = NULL;
3876      -                return (-1);
3877      -        }
3878      -
3879      -        if (ddi_dma_buf_bind_handle(cmd->cmd_arqhandle,
3880      -            cmd->cmd_arq_buf, (DDI_DMA_READ | DDI_DMA_CONSISTENT),
3881      -            callback, NULL, &cmd->cmd_arqcookie, &cookiec) != DDI_SUCCESS) {
3882      -                ddi_dma_free_handle(&cmd->cmd_dmahandle);
3883      -                ddi_dma_free_handle(&cmd->cmd_arqhandle);
3884      -                scsi_free_consistent_buf(cmd->cmd_arq_buf);
3885      -                cmd->cmd_dmahandle = NULL;
3886      -                cmd->cmd_arqhandle = NULL;
3887      -                cmd->cmd_arq_buf = NULL;
3888      -                return (-1);
3889      -        }
3890      -
3891 4253          return (0);
3892 4254  }
3893 4255  
3894 4256  static void
3895 4257  mptsas_kmem_cache_destructor(void *buf, void *cdrarg)
3896 4258  {
3897 4259  #ifndef __lock_lint
3898 4260          _NOTE(ARGUNUSED(cdrarg))
3899 4261  #endif
3900 4262          mptsas_cmd_t    *cmd = buf;
3901 4263  
3902 4264          NDBG4(("mptsas_kmem_cache_destructor"));
3903 4265  
3904      -        if (cmd->cmd_arqhandle) {
3905      -                (void) ddi_dma_unbind_handle(cmd->cmd_arqhandle);
3906      -                ddi_dma_free_handle(&cmd->cmd_arqhandle);
3907      -                cmd->cmd_arqhandle = NULL;
3908      -        }
3909      -        if (cmd->cmd_arq_buf) {
3910      -                scsi_free_consistent_buf(cmd->cmd_arq_buf);
3911      -                cmd->cmd_arq_buf = NULL;
3912      -        }
3913 4266          if (cmd->cmd_dmahandle) {
3914 4267                  ddi_dma_free_handle(&cmd->cmd_dmahandle);
3915 4268                  cmd->cmd_dmahandle = NULL;
3916 4269          }
3917 4270  }
3918 4271  
3919 4272  static int
3920 4273  mptsas_cache_frames_constructor(void *buf, void *cdrarg, int kmflags)
3921 4274  {
3922 4275          mptsas_cache_frames_t   *p = buf;
↓ open down ↓ 37 lines elided ↑ open up ↑
3960 4313                  mptsas_log(mpt, CE_WARN, "Unable to bind DMA resources for"
3961 4314                      " extra SGL");
3962 4315                  return (DDI_FAILURE);
3963 4316          }
3964 4317  
3965 4318          /*
3966 4319           * Store the SGL memory address.  This chip uses this
3967 4320           * address to dma to and from the driver.  The second
3968 4321           * address is the address mpt uses to fill in the SGL.
3969 4322           */
3970      -        p->m_phys_addr = cookie.dmac_address;
     4323 +        p->m_phys_addr = cookie.dmac_laddress;
3971 4324  
3972 4325          return (DDI_SUCCESS);
3973 4326  }
3974 4327  
3975 4328  static void
3976 4329  mptsas_cache_frames_destructor(void *buf, void *cdrarg)
3977 4330  {
3978 4331  #ifndef __lock_lint
3979 4332          _NOTE(ARGUNUSED(cdrarg))
3980 4333  #endif
↓ open down ↓ 14 lines elided ↑ open up ↑
3995 4348   * allocate and deallocate external pkt space (ie. not part of mptsas_cmd)
3996 4349   * for non-standard length cdb, pkt_private, status areas
3997 4350   * if allocation fails, then deallocate all external space and the pkt
3998 4351   */
3999 4352  /* ARGSUSED */
4000 4353  static int
4001 4354  mptsas_pkt_alloc_extern(mptsas_t *mpt, mptsas_cmd_t *cmd,
4002 4355      int cmdlen, int tgtlen, int statuslen, int kf)
4003 4356  {
4004 4357          caddr_t                 cdbp, scbp, tgt;
4005      -        int                     (*callback)(caddr_t) = (kf == KM_SLEEP) ?
4006      -            DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
4007      -        struct scsi_address     ap;
4008 4358          size_t                  senselength;
4009      -        ddi_dma_attr_t          ext_arq_dma_attr;
4010      -        uint_t                  cookiec;
4011 4359  
4012 4360          NDBG3(("mptsas_pkt_alloc_extern: "
4013 4361              "cmd=0x%p cmdlen=%d tgtlen=%d statuslen=%d kf=%x",
4014 4362              (void *)cmd, cmdlen, tgtlen, statuslen, kf));
4015 4363  
4016 4364          tgt = cdbp = scbp = NULL;
4017 4365          cmd->cmd_scblen         = statuslen;
4018 4366          cmd->cmd_privlen        = (uchar_t)tgtlen;
4019 4367  
4020 4368          if (cmdlen > sizeof (cmd->cmd_cdb)) {
↓ open down ↓ 14 lines elided ↑ open up ↑
4035 4383                  if ((scbp = kmem_zalloc((size_t)statuslen, kf)) == NULL) {
4036 4384                          goto fail;
4037 4385                  }
4038 4386                  cmd->cmd_flags |= CFLAG_SCBEXTERN;
4039 4387                  cmd->cmd_pkt->pkt_scbp = (opaque_t)scbp;
4040 4388  
4041 4389                  /* allocate sense data buf for DMA */
4042 4390  
4043 4391                  senselength = statuslen - MPTSAS_GET_ITEM_OFF(
4044 4392                      struct scsi_arq_status, sts_sensedata);
4045      -                cmd->cmd_rqslen = (uchar_t)senselength;
4046      -
4047      -                ap.a_hba_tran = mpt->m_tran;
4048      -                ap.a_target = 0;
4049      -                ap.a_lun = 0;
4050      -
4051      -                cmd->cmd_ext_arq_buf = scsi_alloc_consistent_buf(&ap,
4052      -                    (struct buf *)NULL, senselength, B_READ,
4053      -                    callback, NULL);
4054      -
4055      -                if (cmd->cmd_ext_arq_buf == NULL) {
4056      -                        goto fail;
4057      -                }
4058      -                /*
4059      -                 * allocate a extern arq handle and bind the buf
4060      -                 */
4061      -                ext_arq_dma_attr = mpt->m_msg_dma_attr;
4062      -                ext_arq_dma_attr.dma_attr_sgllen = 1;
4063      -                if ((ddi_dma_alloc_handle(mpt->m_dip,
4064      -                    &ext_arq_dma_attr, callback,
4065      -                    NULL, &cmd->cmd_ext_arqhandle)) != DDI_SUCCESS) {
4066      -                        goto fail;
4067      -                }
4068      -
4069      -                if (ddi_dma_buf_bind_handle(cmd->cmd_ext_arqhandle,
4070      -                    cmd->cmd_ext_arq_buf, (DDI_DMA_READ | DDI_DMA_CONSISTENT),
4071      -                    callback, NULL, &cmd->cmd_ext_arqcookie,
4072      -                    &cookiec)
4073      -                    != DDI_SUCCESS) {
4074      -                        goto fail;
     4393 +                if (senselength > mpt->m_req_sense_size) {
     4394 +                        unsigned long i;
     4395 +                        cmd->cmd_extrqslen = (uint16_t)senselength;
     4396 +                        cmd->cmd_extrqschunks = (senselength +
     4397 +                            (mpt->m_req_sense_size - 1))/mpt->m_req_sense_size;
     4398 +                        i = rmalloc_wait(mpt->m_erqsense_map,
     4399 +                            cmd->cmd_extrqschunks);
     4400 +                        ASSERT(i != 0);
     4401 +                        cmd->cmd_extrqsidx = i - 1;
     4402 +                        cmd->cmd_arq_buf = mpt->m_extreq_sense +
     4403 +                            (cmd->cmd_extrqsidx * mpt->m_req_sense_size);
     4404 +                } else {
     4405 +                        cmd->cmd_rqslen = (uchar_t)senselength;
4075 4406                  }
4076      -                cmd->cmd_flags |= CFLAG_EXTARQBUFVALID;
4077 4407          }
4078 4408          return (0);
4079 4409  fail:
4080 4410          mptsas_pkt_destroy_extern(mpt, cmd);
4081 4411          return (1);
4082 4412  }
4083 4413  
4084 4414  /*
4085 4415   * deallocate external pkt space and deallocate the pkt
4086 4416   */
↓ open down ↓ 1 lines elided ↑ open up ↑
4088 4418  mptsas_pkt_destroy_extern(mptsas_t *mpt, mptsas_cmd_t *cmd)
4089 4419  {
4090 4420          NDBG3(("mptsas_pkt_destroy_extern: cmd=0x%p", (void *)cmd));
4091 4421  
4092 4422          if (cmd->cmd_flags & CFLAG_FREE) {
4093 4423                  mptsas_log(mpt, CE_PANIC,
4094 4424                      "mptsas_pkt_destroy_extern: freeing free packet");
4095 4425                  _NOTE(NOT_REACHED)
4096 4426                  /* NOTREACHED */
4097 4427          }
     4428 +        if (cmd->cmd_extrqslen != 0) {
     4429 +                rmfree(mpt->m_erqsense_map, cmd->cmd_extrqschunks,
     4430 +                    cmd->cmd_extrqsidx + 1);
     4431 +        }
4098 4432          if (cmd->cmd_flags & CFLAG_CDBEXTERN) {
4099 4433                  kmem_free(cmd->cmd_pkt->pkt_cdbp, (size_t)cmd->cmd_cdblen);
4100 4434          }
4101 4435          if (cmd->cmd_flags & CFLAG_SCBEXTERN) {
4102 4436                  kmem_free(cmd->cmd_pkt->pkt_scbp, (size_t)cmd->cmd_scblen);
4103      -                if (cmd->cmd_flags & CFLAG_EXTARQBUFVALID) {
4104      -                        (void) ddi_dma_unbind_handle(cmd->cmd_ext_arqhandle);
4105      -                }
4106      -                if (cmd->cmd_ext_arqhandle) {
4107      -                        ddi_dma_free_handle(&cmd->cmd_ext_arqhandle);
4108      -                        cmd->cmd_ext_arqhandle = NULL;
4109      -                }
4110      -                if (cmd->cmd_ext_arq_buf)
4111      -                        scsi_free_consistent_buf(cmd->cmd_ext_arq_buf);
4112 4437          }
4113 4438          if (cmd->cmd_flags & CFLAG_PRIVEXTERN) {
4114 4439                  kmem_free(cmd->cmd_pkt->pkt_private, (size_t)cmd->cmd_privlen);
4115 4440          }
4116 4441          cmd->cmd_flags = CFLAG_FREE;
4117 4442          kmem_cache_free(mpt->m_kmem_cache, (void *)cmd);
4118 4443  }
4119 4444  
4120 4445  /*
4121 4446   * tran_sync_pkt(9E) - explicit DMA synchronization
↓ open down ↓ 25 lines elided ↑ open up ↑
4147 4472          mptsas_t        *mpt = ADDR2MPT(ap);
4148 4473  
4149 4474          NDBG3(("mptsas_scsi_dmafree: target=%d pkt=0x%p",
4150 4475              ap->a_target, (void *)pkt));
4151 4476  
4152 4477          if (cmd->cmd_flags & CFLAG_DMAVALID) {
4153 4478                  (void) ddi_dma_unbind_handle(cmd->cmd_dmahandle);
4154 4479                  cmd->cmd_flags &= ~CFLAG_DMAVALID;
4155 4480          }
4156 4481  
4157      -        if (cmd->cmd_flags & CFLAG_EXTARQBUFVALID) {
4158      -                (void) ddi_dma_unbind_handle(cmd->cmd_ext_arqhandle);
4159      -                cmd->cmd_flags &= ~CFLAG_EXTARQBUFVALID;
4160      -        }
4161      -
4162 4482          mptsas_free_extra_sgl_frame(mpt, cmd);
4163 4483  }
4164 4484  
4165 4485  static void
4166 4486  mptsas_pkt_comp(struct scsi_pkt *pkt, mptsas_cmd_t *cmd)
4167 4487  {
4168 4488          if ((cmd->cmd_flags & CFLAG_CMDIOPB) &&
4169 4489              (!(cmd->cmd_flags & CFLAG_DMASEND))) {
4170 4490                  (void) ddi_dma_sync(cmd->cmd_dmahandle, 0, 0,
4171 4491                      DDI_DMA_SYNC_FORCPU);
4172 4492          }
4173 4493          (*pkt->pkt_comp)(pkt);
4174 4494  }
4175 4495  
4176 4496  static void
4177      -mptsas_sge_setup(mptsas_t *mpt, mptsas_cmd_t *cmd, uint32_t *control,
4178      -        pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl)
     4497 +mptsas_sge_mainframe(mptsas_cmd_t *cmd, pMpi2SCSIIORequest_t frame,
     4498 +                ddi_acc_handle_t acc_hdl, uint_t cookiec,
     4499 +                uint32_t end_flags)
4179 4500  {
4180      -        uint_t                  cookiec;
     4501 +        pMpi2SGESimple64_t      sge;
4181 4502          mptti_t                 *dmap;
4182 4503          uint32_t                flags;
     4504 +
     4505 +        dmap = cmd->cmd_sg;
     4506 +
     4507 +        sge = (pMpi2SGESimple64_t)(&frame->SGL);
     4508 +        while (cookiec--) {
     4509 +                ddi_put32(acc_hdl, &sge->Address.Low,
     4510 +                    dmap->addr.address64.Low);
     4511 +                ddi_put32(acc_hdl, &sge->Address.High,
     4512 +                    dmap->addr.address64.High);
     4513 +                ddi_put32(acc_hdl, &sge->FlagsLength, dmap->count);
     4514 +                flags = ddi_get32(acc_hdl, &sge->FlagsLength);
     4515 +                flags |= ((uint32_t)
     4516 +                    (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
     4517 +                    MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
     4518 +                    MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
     4519 +                    MPI2_SGE_FLAGS_SHIFT);
     4520 +
     4521 +                /*
     4522 +                 * If this is the last cookie, we set the flags
     4523 +                 * to indicate so
     4524 +                 */
     4525 +                if (cookiec == 0) {
     4526 +                        flags |= end_flags;
     4527 +                }
     4528 +                if (cmd->cmd_flags & CFLAG_DMASEND) {
     4529 +                        flags |= (MPI2_SGE_FLAGS_HOST_TO_IOC <<
     4530 +                            MPI2_SGE_FLAGS_SHIFT);
     4531 +                } else {
     4532 +                        flags |= (MPI2_SGE_FLAGS_IOC_TO_HOST <<
     4533 +                            MPI2_SGE_FLAGS_SHIFT);
     4534 +                }
     4535 +                ddi_put32(acc_hdl, &sge->FlagsLength, flags);
     4536 +                dmap++;
     4537 +                sge++;
     4538 +        }
     4539 +}
     4540 +
     4541 +static void
     4542 +mptsas_sge_chain(mptsas_t *mpt, mptsas_cmd_t *cmd,
     4543 +    pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl)
     4544 +{
4183 4545          pMpi2SGESimple64_t      sge;
4184 4546          pMpi2SGEChain64_t       sgechain;
4185      -        ASSERT(cmd->cmd_flags & CFLAG_DMAVALID);
     4547 +        uint64_t                nframe_phys_addr;
     4548 +        uint_t                  cookiec;
     4549 +        mptti_t                 *dmap;
     4550 +        uint32_t                flags;
     4551 +        int                     i, j, k, l, frames, sgemax;
     4552 +        int                     temp, maxframe_sges;
     4553 +        uint8_t                 chainflags;
     4554 +        uint16_t                chainlength;
     4555 +        mptsas_cache_frames_t   *p;
     4556 +
     4557 +        cookiec = cmd->cmd_cookiec;
4186 4558  
4187 4559          /*
4188      -         * Save the number of entries in the DMA
4189      -         * Scatter/Gather list
     4560 +         * Hereby we start to deal with multiple frames.
     4561 +         * The process is as follows:
     4562 +         * 1. Determine how many frames are needed for SGL element
     4563 +         *    storage; Note that all frames are stored in contiguous
     4564 +         *    memory space and in 64-bit DMA mode each element is
     4565 +         *    3 double-words (12 bytes) long.
     4566 +         * 2. Fill up the main frame. We need to do this separately
     4567 +         *    since it contains the SCSI IO request header and needs
     4568 +         *    dedicated processing. Note that the last 4 double-words
     4569 +         *    of the SCSI IO header is for SGL element storage
     4570 +         *    (MPI2_SGE_IO_UNION).
     4571 +         * 3. Fill the chain element in the main frame, so the DMA
     4572 +         *    engine can use the following frames.
     4573 +         * 4. Enter a loop to fill the remaining frames. Note that the
     4574 +         *    last frame contains no chain element.  The remaining
     4575 +         *    frames go into the mpt SGL buffer allocated on the fly,
     4576 +         *    not immediately following the main message frame, as in
     4577 +         *    Gen1.
     4578 +         * Some restrictions:
     4579 +         * 1. For 64-bit DMA, the simple element and chain element
     4580 +         *    are both of 3 double-words (12 bytes) in size, even
     4581 +         *    though all frames are stored in the first 4G of mem
     4582 +         *    range and the higher 32-bits of the address are always 0.
     4583 +         * 2. On some controllers (like the 1064/1068), a frame can
     4584 +         *    hold SGL elements with the last 1 or 2 double-words
     4585 +         *    (4 or 8 bytes) un-used. On these controllers, we should
     4586 +         *    recognize that there's not enough room for another SGL
     4587 +         *    element and move the sge pointer to the next frame.
4190 4588           */
4191      -        cookiec = cmd->cmd_cookiec;
4192 4589  
4193      -        NDBG1(("mptsas_sge_setup: cookiec=%d", cookiec));
     4590 +        /*
     4591 +         * Sgemax is the number of SGE's that will fit
     4592 +         * each extra frame and frames is total
     4593 +         * number of frames we'll need.  1 sge entry per
     4594 +         * frame is reseverd for the chain element thus the -1 below.
     4595 +         */
     4596 +        sgemax = ((mpt->m_req_frame_size / sizeof (MPI2_SGE_SIMPLE64)) - 1);
     4597 +        maxframe_sges = MPTSAS_MAX_FRAME_SGES64(mpt);
     4598 +        temp = (cookiec - (maxframe_sges - 1)) / sgemax;
4194 4599  
4195 4600          /*
4196      -         * Set read/write bit in control.
     4601 +         * A little check to see if we need to round up the number
     4602 +         * of frames we need
4197 4603           */
4198      -        if (cmd->cmd_flags & CFLAG_DMASEND) {
4199      -                *control |= MPI2_SCSIIO_CONTROL_WRITE;
     4604 +        if ((cookiec - (maxframe_sges - 1)) - (temp * sgemax) > 1) {
     4605 +                frames = (temp + 1);
4200 4606          } else {
4201      -                *control |= MPI2_SCSIIO_CONTROL_READ;
     4607 +                frames = temp;
4202 4608          }
     4609 +        dmap = cmd->cmd_sg;
     4610 +        sge = (pMpi2SGESimple64_t)(&frame->SGL);
4203 4611  
4204      -        ddi_put32(acc_hdl, &frame->DataLength, cmd->cmd_dmacount);
     4612 +        /*
     4613 +         * First fill in the main frame
     4614 +         */
     4615 +        j = maxframe_sges - 1;
     4616 +        mptsas_sge_mainframe(cmd, frame, acc_hdl, j,
     4617 +            ((uint32_t)(MPI2_SGE_FLAGS_LAST_ELEMENT) <<
     4618 +            MPI2_SGE_FLAGS_SHIFT));
     4619 +        dmap += j;
     4620 +        sge += j;
     4621 +        j++;
     4622 +
     4623 +        /*
     4624 +         * Fill in the chain element in the main frame.
     4625 +         * About calculation on ChainOffset:
     4626 +         * 1. Struct msg_scsi_io_request has 4 double-words (16 bytes)
     4627 +         *    in the end reserved for SGL element storage
     4628 +         *    (MPI2_SGE_IO_UNION); we should count it in our
     4629 +         *    calculation.  See its definition in the header file.
     4630 +         * 2. Constant j is the counter of the current SGL element
     4631 +         *    that will be processed, and (j - 1) is the number of
     4632 +         *    SGL elements that have been processed (stored in the
     4633 +         *    main frame).
     4634 +         * 3. ChainOffset value should be in units of double-words (4
     4635 +         *    bytes) so the last value should be divided by 4.
     4636 +         */
     4637 +        ddi_put8(acc_hdl, &frame->ChainOffset,
     4638 +            (sizeof (MPI2_SCSI_IO_REQUEST) -
     4639 +            sizeof (MPI2_SGE_IO_UNION) +
     4640 +            (j - 1) * sizeof (MPI2_SGE_SIMPLE64)) >> 2);
     4641 +        sgechain = (pMpi2SGEChain64_t)sge;
     4642 +        chainflags = (MPI2_SGE_FLAGS_CHAIN_ELEMENT |
     4643 +            MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
     4644 +            MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
     4645 +        ddi_put8(acc_hdl, &sgechain->Flags, chainflags);
     4646 +
     4647 +        /*
     4648 +         * The size of the next frame is the accurate size of space
     4649 +         * (in bytes) used to store the SGL elements. j is the counter
     4650 +         * of SGL elements. (j - 1) is the number of SGL elements that
     4651 +         * have been processed (stored in frames).
     4652 +         */
     4653 +        if (frames >= 2) {
     4654 +                chainlength = mpt->m_req_frame_size /
     4655 +                    sizeof (MPI2_SGE_SIMPLE64) *
     4656 +                    sizeof (MPI2_SGE_SIMPLE64);
     4657 +        } else {
     4658 +                chainlength = ((cookiec - (j - 1)) *
     4659 +                    sizeof (MPI2_SGE_SIMPLE64));
     4660 +        }
     4661 +
     4662 +        p = cmd->cmd_extra_frames;
     4663 +
     4664 +        ddi_put16(acc_hdl, &sgechain->Length, chainlength);
     4665 +        ddi_put32(acc_hdl, &sgechain->Address.Low,
     4666 +            (p->m_phys_addr&0xffffffffull));
     4667 +        ddi_put32(acc_hdl, &sgechain->Address.High, p->m_phys_addr>>32);
4205 4668  
4206 4669          /*
4207      -         * We have 2 cases here.  First where we can fit all the
4208      -         * SG elements into the main frame, and the case
4209      -         * where we can't.
4210      -         * If we have more cookies than we can attach to a frame
4211      -         * we will need to use a chain element to point
4212      -         * a location of memory where the rest of the S/G
4213      -         * elements reside.
     4670 +         * If there are more than 2 frames left we have to
     4671 +         * fill in the next chain offset to the location of
     4672 +         * the chain element in the next frame.
     4673 +         * sgemax is the number of simple elements in an extra
     4674 +         * frame. Note that the value NextChainOffset should be
     4675 +         * in double-words (4 bytes).
4214 4676           */
4215      -        if (cookiec <= MPTSAS_MAX_FRAME_SGES64(mpt)) {
4216      -                dmap = cmd->cmd_sg;
4217      -                sge = (pMpi2SGESimple64_t)(&frame->SGL);
4218      -                while (cookiec--) {
4219      -                        ddi_put32(acc_hdl,
4220      -                            &sge->Address.Low, dmap->addr.address64.Low);
4221      -                        ddi_put32(acc_hdl,
4222      -                            &sge->Address.High, dmap->addr.address64.High);
4223      -                        ddi_put32(acc_hdl, &sge->FlagsLength,
4224      -                            dmap->count);
4225      -                        flags = ddi_get32(acc_hdl, &sge->FlagsLength);
4226      -                        flags |= ((uint32_t)
4227      -                            (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
4228      -                            MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
4229      -                            MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
4230      -                            MPI2_SGE_FLAGS_SHIFT);
     4677 +        if (frames >= 2) {
     4678 +                ddi_put8(acc_hdl, &sgechain->NextChainOffset,
     4679 +                    (sgemax * sizeof (MPI2_SGE_SIMPLE64)) >> 2);
     4680 +        } else {
     4681 +                ddi_put8(acc_hdl, &sgechain->NextChainOffset, 0);
     4682 +        }
     4683 +
     4684 +        /*
     4685 +         * Jump to next frame;
     4686 +         * Starting here, chain buffers go into the per command SGL.
     4687 +         * This buffer is allocated when chain buffers are needed.
     4688 +         */
     4689 +        sge = (pMpi2SGESimple64_t)p->m_frames_addr;
     4690 +        i = cookiec;
     4691 +
     4692 +        /*
     4693 +         * Start filling in frames with SGE's.  If we
     4694 +         * reach the end of frame and still have SGE's
     4695 +         * to fill we need to add a chain element and
     4696 +         * use another frame.  j will be our counter
     4697 +         * for what cookie we are at and i will be
     4698 +         * the total cookiec. k is the current frame
     4699 +         */
     4700 +        for (k = 1; k <= frames; k++) {
     4701 +                for (l = 1; (l <= (sgemax + 1)) && (j <= i); j++, l++) {
4231 4702  
4232 4703                          /*
4233      -                         * If this is the last cookie, we set the flags
4234      -                         * to indicate so
     4704 +                         * If we have reached the end of frame
     4705 +                         * and we have more SGE's to fill in
     4706 +                         * we have to fill the final entry
     4707 +                         * with a chain element and then
     4708 +                         * continue to the next frame
4235 4709                           */
4236      -                        if (cookiec == 0) {
4237      -                                flags |=
4238      -                                    ((uint32_t)(MPI2_SGE_FLAGS_LAST_ELEMENT
4239      -                                    | MPI2_SGE_FLAGS_END_OF_BUFFER
4240      -                                    | MPI2_SGE_FLAGS_END_OF_LIST) <<
4241      -                                    MPI2_SGE_FLAGS_SHIFT);
4242      -                        }
4243      -                        if (cmd->cmd_flags & CFLAG_DMASEND) {
4244      -                                flags |= (MPI2_SGE_FLAGS_HOST_TO_IOC <<
4245      -                                    MPI2_SGE_FLAGS_SHIFT);
4246      -                        } else {
4247      -                                flags |= (MPI2_SGE_FLAGS_IOC_TO_HOST <<
4248      -                                    MPI2_SGE_FLAGS_SHIFT);
     4710 +                        if ((l == (sgemax + 1)) && (k != frames)) {
     4711 +                                sgechain = (pMpi2SGEChain64_t)sge;
     4712 +                                j--;
     4713 +                                chainflags = (
     4714 +                                    MPI2_SGE_FLAGS_CHAIN_ELEMENT |
     4715 +                                    MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
     4716 +                                    MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
     4717 +                                ddi_put8(p->m_acc_hdl,
     4718 +                                    &sgechain->Flags, chainflags);
     4719 +                                /*
     4720 +                                 * k is the frame counter and (k + 1)
     4721 +                                 * is the number of the next frame.
     4722 +                                 * Note that frames are in contiguous
     4723 +                                 * memory space.
     4724 +                                 */
     4725 +                                nframe_phys_addr = p->m_phys_addr +
     4726 +                                    (mpt->m_req_frame_size * k);
     4727 +                                ddi_put32(p->m_acc_hdl,
     4728 +                                    &sgechain->Address.Low,
     4729 +                                    nframe_phys_addr&0xffffffffull);
     4730 +                                ddi_put32(p->m_acc_hdl,
     4731 +                                    &sgechain->Address.High,
     4732 +                                    nframe_phys_addr>>32);
     4733 +
     4734 +                                /*
     4735 +                                 * If there are more than 2 frames left
     4736 +                                 * we have to next chain offset to
     4737 +                                 * the location of the chain element
     4738 +                                 * in the next frame and fill in the
     4739 +                                 * length of the next chain
     4740 +                                 */
     4741 +                                if ((frames - k) >= 2) {
     4742 +                                        ddi_put8(p->m_acc_hdl,
     4743 +                                            &sgechain->NextChainOffset,
     4744 +                                            (sgemax *
     4745 +                                            sizeof (MPI2_SGE_SIMPLE64))
     4746 +                                            >> 2);
     4747 +                                        ddi_put16(p->m_acc_hdl,
     4748 +                                            &sgechain->Length,
     4749 +                                            mpt->m_req_frame_size /
     4750 +                                            sizeof (MPI2_SGE_SIMPLE64) *
     4751 +                                            sizeof (MPI2_SGE_SIMPLE64));
     4752 +                                } else {
     4753 +                                        /*
     4754 +                                         * This is the last frame. Set
     4755 +                                         * the NextChainOffset to 0 and
     4756 +                                         * Length is the total size of
     4757 +                                         * all remaining simple elements
     4758 +                                         */
     4759 +                                        ddi_put8(p->m_acc_hdl,
     4760 +                                            &sgechain->NextChainOffset,
     4761 +                                            0);
     4762 +                                        ddi_put16(p->m_acc_hdl,
     4763 +                                            &sgechain->Length,
     4764 +                                            (cookiec - j) *
     4765 +                                            sizeof (MPI2_SGE_SIMPLE64));
     4766 +                                }
     4767 +
     4768 +                                /* Jump to the next frame */
     4769 +                                sge = (pMpi2SGESimple64_t)
     4770 +                                    ((char *)p->m_frames_addr +
     4771 +                                    (int)mpt->m_req_frame_size * k);
     4772 +
     4773 +                                continue;
4249 4774                          }
4250      -                        ddi_put32(acc_hdl, &sge->FlagsLength, flags);
4251      -                        dmap++;
4252      -                        sge++;
4253      -                }
4254      -        } else {
4255      -                /*
4256      -                 * Hereby we start to deal with multiple frames.
4257      -                 * The process is as follows:
4258      -                 * 1. Determine how many frames are needed for SGL element
4259      -                 *    storage; Note that all frames are stored in contiguous
4260      -                 *    memory space and in 64-bit DMA mode each element is
4261      -                 *    3 double-words (12 bytes) long.
4262      -                 * 2. Fill up the main frame. We need to do this separately
4263      -                 *    since it contains the SCSI IO request header and needs
4264      -                 *    dedicated processing. Note that the last 4 double-words
4265      -                 *    of the SCSI IO header is for SGL element storage
4266      -                 *    (MPI2_SGE_IO_UNION).
4267      -                 * 3. Fill the chain element in the main frame, so the DMA
4268      -                 *    engine can use the following frames.
4269      -                 * 4. Enter a loop to fill the remaining frames. Note that the
4270      -                 *    last frame contains no chain element.  The remaining
4271      -                 *    frames go into the mpt SGL buffer allocated on the fly,
4272      -                 *    not immediately following the main message frame, as in
4273      -                 *    Gen1.
4274      -                 * Some restrictions:
4275      -                 * 1. For 64-bit DMA, the simple element and chain element
4276      -                 *    are both of 3 double-words (12 bytes) in size, even
4277      -                 *    though all frames are stored in the first 4G of mem
4278      -                 *    range and the higher 32-bits of the address are always 0.
4279      -                 * 2. On some controllers (like the 1064/1068), a frame can
4280      -                 *    hold SGL elements with the last 1 or 2 double-words
4281      -                 *    (4 or 8 bytes) un-used. On these controllers, we should
4282      -                 *    recognize that there's not enough room for another SGL
4283      -                 *    element and move the sge pointer to the next frame.
4284      -                 */
4285      -                int             i, j, k, l, frames, sgemax;
4286      -                int             temp;
4287      -                uint8_t         chainflags;
4288      -                uint16_t        chainlength;
4289      -                mptsas_cache_frames_t *p;
4290      -
4291      -                /*
4292      -                 * Sgemax is the number of SGE's that will fit
4293      -                 * each extra frame and frames is total
4294      -                 * number of frames we'll need.  1 sge entry per
4295      -                 * frame is reseverd for the chain element thus the -1 below.
4296      -                 */
4297      -                sgemax = ((mpt->m_req_frame_size / sizeof (MPI2_SGE_SIMPLE64))
4298      -                    - 1);
4299      -                temp = (cookiec - (MPTSAS_MAX_FRAME_SGES64(mpt) - 1)) / sgemax;
4300      -
4301      -                /*
4302      -                 * A little check to see if we need to round up the number
4303      -                 * of frames we need
4304      -                 */
4305      -                if ((cookiec - (MPTSAS_MAX_FRAME_SGES64(mpt) - 1)) - (temp *
4306      -                    sgemax) > 1) {
4307      -                        frames = (temp + 1);
4308      -                } else {
4309      -                        frames = temp;
4310      -                }
4311      -                dmap = cmd->cmd_sg;
4312      -                sge = (pMpi2SGESimple64_t)(&frame->SGL);
4313 4775  
4314      -                /*
4315      -                 * First fill in the main frame
4316      -                 */
4317      -                for (j = 1; j < MPTSAS_MAX_FRAME_SGES64(mpt); j++) {
4318      -                        ddi_put32(acc_hdl, &sge->Address.Low,
     4776 +                        ddi_put32(p->m_acc_hdl,
     4777 +                            &sge->Address.Low,
4319 4778                              dmap->addr.address64.Low);
4320      -                        ddi_put32(acc_hdl, &sge->Address.High,
     4779 +                        ddi_put32(p->m_acc_hdl,
     4780 +                            &sge->Address.High,
4321 4781                              dmap->addr.address64.High);
4322      -                        ddi_put32(acc_hdl, &sge->FlagsLength, dmap->count);
4323      -                        flags = ddi_get32(acc_hdl, &sge->FlagsLength);
4324      -                        flags |= ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
     4782 +                        ddi_put32(p->m_acc_hdl,
     4783 +                            &sge->FlagsLength, dmap->count);
     4784 +                        flags = ddi_get32(p->m_acc_hdl,
     4785 +                            &sge->FlagsLength);
     4786 +                        flags |= ((uint32_t)(
     4787 +                            MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
4325 4788                              MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
4326 4789                              MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
4327 4790                              MPI2_SGE_FLAGS_SHIFT);
4328 4791  
4329 4792                          /*
4330      -                         * If this is the last SGE of this frame
4331      -                         * we set the end of list flag
     4793 +                         * If we are at the end of the frame and
     4794 +                         * there is another frame to fill in
     4795 +                         * we set the last simple element as last
     4796 +                         * element
4332 4797                           */
4333      -                        if (j == (MPTSAS_MAX_FRAME_SGES64(mpt) - 1)) {
     4798 +                        if ((l == sgemax) && (k != frames)) {
4334 4799                                  flags |= ((uint32_t)
4335 4800                                      (MPI2_SGE_FLAGS_LAST_ELEMENT) <<
4336 4801                                      MPI2_SGE_FLAGS_SHIFT);
4337 4802                          }
     4803 +
     4804 +                        /*
     4805 +                         * If this is the final cookie we
     4806 +                         * indicate it by setting the flags
     4807 +                         */
     4808 +                        if (j == i) {
     4809 +                                flags |= ((uint32_t)
     4810 +                                    (MPI2_SGE_FLAGS_LAST_ELEMENT |
     4811 +                                    MPI2_SGE_FLAGS_END_OF_BUFFER |
     4812 +                                    MPI2_SGE_FLAGS_END_OF_LIST) <<
     4813 +                                    MPI2_SGE_FLAGS_SHIFT);
     4814 +                        }
4338 4815                          if (cmd->cmd_flags & CFLAG_DMASEND) {
4339 4816                                  flags |=
4340 4817                                      (MPI2_SGE_FLAGS_HOST_TO_IOC <<
4341 4818                                      MPI2_SGE_FLAGS_SHIFT);
4342 4819                          } else {
4343 4820                                  flags |=
4344 4821                                      (MPI2_SGE_FLAGS_IOC_TO_HOST <<
4345 4822                                      MPI2_SGE_FLAGS_SHIFT);
4346 4823                          }
4347      -                        ddi_put32(acc_hdl, &sge->FlagsLength, flags);
     4824 +                        ddi_put32(p->m_acc_hdl,
     4825 +                            &sge->FlagsLength, flags);
4348 4826                          dmap++;
4349 4827                          sge++;
4350 4828                  }
     4829 +        }
4351 4830  
4352      -                /*
4353      -                 * Fill in the chain element in the main frame.
4354      -                 * About calculation on ChainOffset:
4355      -                 * 1. Struct msg_scsi_io_request has 4 double-words (16 bytes)
4356      -                 *    in the end reserved for SGL element storage
4357      -                 *    (MPI2_SGE_IO_UNION); we should count it in our
4358      -                 *    calculation.  See its definition in the header file.
4359      -                 * 2. Constant j is the counter of the current SGL element
4360      -                 *    that will be processed, and (j - 1) is the number of
4361      -                 *    SGL elements that have been processed (stored in the
4362      -                 *    main frame).
4363      -                 * 3. ChainOffset value should be in units of double-words (4
4364      -                 *    bytes) so the last value should be divided by 4.
4365      -                 */
4366      -                ddi_put8(acc_hdl, &frame->ChainOffset,
4367      -                    (sizeof (MPI2_SCSI_IO_REQUEST) -
4368      -                    sizeof (MPI2_SGE_IO_UNION) +
4369      -                    (j - 1) * sizeof (MPI2_SGE_SIMPLE64)) >> 2);
4370      -                sgechain = (pMpi2SGEChain64_t)sge;
4371      -                chainflags = (MPI2_SGE_FLAGS_CHAIN_ELEMENT |
4372      -                    MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
4373      -                    MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
4374      -                ddi_put8(acc_hdl, &sgechain->Flags, chainflags);
     4831 +        /*
     4832 +         * Sync DMA with the chain buffers that were just created
     4833 +         */
     4834 +        (void) ddi_dma_sync(p->m_dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
     4835 +}
4375 4836  
4376      -                /*
4377      -                 * The size of the next frame is the accurate size of space
4378      -                 * (in bytes) used to store the SGL elements. j is the counter
4379      -                 * of SGL elements. (j - 1) is the number of SGL elements that
4380      -                 * have been processed (stored in frames).
4381      -                 */
4382      -                if (frames >= 2) {
4383      -                        chainlength = mpt->m_req_frame_size /
4384      -                            sizeof (MPI2_SGE_SIMPLE64) *
4385      -                            sizeof (MPI2_SGE_SIMPLE64);
4386      -                } else {
4387      -                        chainlength = ((cookiec - (j - 1)) *
4388      -                            sizeof (MPI2_SGE_SIMPLE64));
4389      -                }
     4837 +static void
     4838 +mptsas_ieee_sge_mainframe(mptsas_cmd_t *cmd, pMpi2SCSIIORequest_t frame,
     4839 +    ddi_acc_handle_t acc_hdl, uint_t cookiec,
     4840 +    uint8_t end_flag)
     4841 +{
     4842 +        pMpi2IeeeSgeSimple64_t  ieeesge;
     4843 +        mptti_t                 *dmap;
     4844 +        uint8_t                 flags;
     4845 +
     4846 +        dmap = cmd->cmd_sg;
4390 4847  
4391      -                p = cmd->cmd_extra_frames;
     4848 +        NDBG1(("mptsas_ieee_sge_mainframe: cookiec=%d, %s", cookiec,
     4849 +            cmd->cmd_flags & CFLAG_DMASEND?"Out":"In"));
4392 4850  
4393      -                ddi_put16(acc_hdl, &sgechain->Length, chainlength);
4394      -                ddi_put32(acc_hdl, &sgechain->Address.Low,
4395      -                    p->m_phys_addr);
4396      -                /* SGL is allocated in the first 4G mem range */
4397      -                ddi_put32(acc_hdl, &sgechain->Address.High, 0);
     4851 +        ieeesge = (pMpi2IeeeSgeSimple64_t)(&frame->SGL);
     4852 +        while (cookiec--) {
     4853 +                ddi_put32(acc_hdl, &ieeesge->Address.Low,
     4854 +                    dmap->addr.address64.Low);
     4855 +                ddi_put32(acc_hdl, &ieeesge->Address.High,
     4856 +                    dmap->addr.address64.High);
     4857 +                ddi_put32(acc_hdl, &ieeesge->Length, dmap->count);
     4858 +                NDBG1(("mptsas_ieee_sge_mainframe: len=%d, high=0x%x",
     4859 +                    dmap->count, dmap->addr.address64.High));
     4860 +                flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
     4861 +                    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
4398 4862  
4399 4863                  /*
4400      -                 * If there are more than 2 frames left we have to
4401      -                 * fill in the next chain offset to the location of
4402      -                 * the chain element in the next frame.
4403      -                 * sgemax is the number of simple elements in an extra
4404      -                 * frame. Note that the value NextChainOffset should be
4405      -                 * in double-words (4 bytes).
     4864 +                 * If this is the last cookie, we set the flags
     4865 +                 * to indicate so
4406 4866                   */
4407      -                if (frames >= 2) {
4408      -                        ddi_put8(acc_hdl, &sgechain->NextChainOffset,
4409      -                            (sgemax * sizeof (MPI2_SGE_SIMPLE64)) >> 2);
4410      -                } else {
4411      -                        ddi_put8(acc_hdl, &sgechain->NextChainOffset, 0);
     4867 +                if (cookiec == 0) {
     4868 +                        flags |= end_flag;
4412 4869                  }
4413 4870  
4414 4871                  /*
4415      -                 * Jump to next frame;
4416      -                 * Starting here, chain buffers go into the per command SGL.
4417      -                 * This buffer is allocated when chain buffers are needed.
     4872 +                 * XXX: Hmmm, what about the direction based on
     4873 +                 * cmd->cmd_flags & CFLAG_DMASEND?
4418 4874                   */
4419      -                sge = (pMpi2SGESimple64_t)p->m_frames_addr;
4420      -                i = cookiec;
     4875 +                ddi_put8(acc_hdl, &ieeesge->Flags, flags);
     4876 +                dmap++;
     4877 +                ieeesge++;
     4878 +        }
     4879 +}
4421 4880  
4422      -                /*
4423      -                 * Start filling in frames with SGE's.  If we
4424      -                 * reach the end of frame and still have SGE's
4425      -                 * to fill we need to add a chain element and
4426      -                 * use another frame.  j will be our counter
4427      -                 * for what cookie we are at and i will be
4428      -                 * the total cookiec. k is the current frame
4429      -                 */
4430      -                for (k = 1; k <= frames; k++) {
4431      -                        for (l = 1; (l <= (sgemax + 1)) && (j <= i); j++, l++) {
     4881 +static void
     4882 +mptsas_ieee_sge_chain(mptsas_t *mpt, mptsas_cmd_t *cmd,
     4883 +    pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl)
     4884 +{
     4885 +        pMpi2IeeeSgeSimple64_t  ieeesge;
     4886 +        pMpi25IeeeSgeChain64_t  ieeesgechain;
     4887 +        uint64_t                nframe_phys_addr;
     4888 +        uint_t                  cookiec;
     4889 +        mptti_t                 *dmap;
     4890 +        uint8_t                 flags;
     4891 +        int                     i, j, k, l, frames, sgemax;
     4892 +        int                     temp, maxframe_sges;
     4893 +        uint8_t                 chainflags;
     4894 +        uint32_t                chainlength;
     4895 +        mptsas_cache_frames_t   *p;
4432 4896  
4433      -                                /*
4434      -                                 * If we have reached the end of frame
4435      -                                 * and we have more SGE's to fill in
4436      -                                 * we have to fill the final entry
4437      -                                 * with a chain element and then
4438      -                                 * continue to the next frame
4439      -                                 */
4440      -                                if ((l == (sgemax + 1)) && (k != frames)) {
4441      -                                        sgechain = (pMpi2SGEChain64_t)sge;
4442      -                                        j--;
4443      -                                        chainflags = (
4444      -                                            MPI2_SGE_FLAGS_CHAIN_ELEMENT |
4445      -                                            MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
4446      -                                            MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
4447      -                                        ddi_put8(p->m_acc_hdl,
4448      -                                            &sgechain->Flags, chainflags);
4449      -                                        /*
4450      -                                         * k is the frame counter and (k + 1)
4451      -                                         * is the number of the next frame.
4452      -                                         * Note that frames are in contiguous
4453      -                                         * memory space.
4454      -                                         */
4455      -                                        ddi_put32(p->m_acc_hdl,
4456      -                                            &sgechain->Address.Low,
4457      -                                            (p->m_phys_addr +
4458      -                                            (mpt->m_req_frame_size * k)));
4459      -                                        ddi_put32(p->m_acc_hdl,
4460      -                                            &sgechain->Address.High, 0);
     4897 +        cookiec = cmd->cmd_cookiec;
4461 4898  
4462      -                                        /*
4463      -                                         * If there are more than 2 frames left
4464      -                                         * we have to next chain offset to
4465      -                                         * the location of the chain element
4466      -                                         * in the next frame and fill in the
4467      -                                         * length of the next chain
4468      -                                         */
4469      -                                        if ((frames - k) >= 2) {
4470      -                                                ddi_put8(p->m_acc_hdl,
4471      -                                                    &sgechain->NextChainOffset,
4472      -                                                    (sgemax *
4473      -                                                    sizeof (MPI2_SGE_SIMPLE64))
4474      -                                                    >> 2);
4475      -                                                ddi_put16(p->m_acc_hdl,
4476      -                                                    &sgechain->Length,
4477      -                                                    mpt->m_req_frame_size /
4478      -                                                    sizeof (MPI2_SGE_SIMPLE64) *
4479      -                                                    sizeof (MPI2_SGE_SIMPLE64));
4480      -                                        } else {
4481      -                                                /*
4482      -                                                 * This is the last frame. Set
4483      -                                                 * the NextChainOffset to 0 and
4484      -                                                 * Length is the total size of
4485      -                                                 * all remaining simple elements
4486      -                                                 */
4487      -                                                ddi_put8(p->m_acc_hdl,
4488      -                                                    &sgechain->NextChainOffset,
4489      -                                                    0);
4490      -                                                ddi_put16(p->m_acc_hdl,
4491      -                                                    &sgechain->Length,
4492      -                                                    (cookiec - j) *
4493      -                                                    sizeof (MPI2_SGE_SIMPLE64));
4494      -                                        }
     4899 +        NDBG1(("mptsas_ieee_sge_chain: cookiec=%d", cookiec));
4495 4900  
4496      -                                        /* Jump to the next frame */
4497      -                                        sge = (pMpi2SGESimple64_t)
4498      -                                            ((char *)p->m_frames_addr +
4499      -                                            (int)mpt->m_req_frame_size * k);
     4901 +        /*
     4902 +         * Hereby we start to deal with multiple frames.
     4903 +         * The process is as follows:
     4904 +         * 1. Determine how many frames are needed for SGL element
     4905 +         *    storage; Note that all frames are stored in contiguous
     4906 +         *    memory space and in 64-bit DMA mode each element is
     4907 +         *    4 double-words (16 bytes) long.
     4908 +         * 2. Fill up the main frame. We need to do this separately
     4909 +         *    since it contains the SCSI IO request header and needs
     4910 +         *    dedicated processing. Note that the last 4 double-words
     4911 +         *    of the SCSI IO header is for SGL element storage
     4912 +         *    (MPI2_SGE_IO_UNION).
     4913 +         * 3. Fill the chain element in the main frame, so the DMA
     4914 +         *    engine can use the following frames.
     4915 +         * 4. Enter a loop to fill the remaining frames. Note that the
     4916 +         *    last frame contains no chain element.  The remaining
     4917 +         *    frames go into the mpt SGL buffer allocated on the fly,
     4918 +         *    not immediately following the main message frame, as in
     4919 +         *    Gen1.
     4920 +         * Some restrictions:
     4921 +         * 1. For 64-bit DMA, the simple element and chain element
     4922 +         *    are both of 4 double-words (16 bytes) in size, even
     4923 +         *    though all frames are stored in the first 4G of mem
     4924 +         *    range and the higher 32-bits of the address are always 0.
     4925 +         * 2. On some controllers (like the 1064/1068), a frame can
     4926 +         *    hold SGL elements with the last 1 or 2 double-words
     4927 +         *    (4 or 8 bytes) un-used. On these controllers, we should
     4928 +         *    recognize that there's not enough room for another SGL
     4929 +         *    element and move the sge pointer to the next frame.
     4930 +         */
4500 4931  
4501      -                                        continue;
4502      -                                }
     4932 +        /*
     4933 +         * Sgemax is the number of SGE's that will fit
     4934 +         * each extra frame and frames is total
     4935 +         * number of frames we'll need.  1 sge entry per
     4936 +         * frame is reseverd for the chain element thus the -1 below.
     4937 +         */
     4938 +        sgemax = ((mpt->m_req_frame_size / sizeof (MPI2_IEEE_SGE_SIMPLE64))
     4939 +            - 1);
     4940 +        maxframe_sges = MPTSAS_MAX_FRAME_SGES64(mpt);
     4941 +        temp = (cookiec - (maxframe_sges - 1)) / sgemax;
4503 4942  
4504      -                                ddi_put32(p->m_acc_hdl,
4505      -                                    &sge->Address.Low,
4506      -                                    dmap->addr.address64.Low);
4507      -                                ddi_put32(p->m_acc_hdl,
4508      -                                    &sge->Address.High,
4509      -                                    dmap->addr.address64.High);
4510      -                                ddi_put32(p->m_acc_hdl,
4511      -                                    &sge->FlagsLength, dmap->count);
4512      -                                flags = ddi_get32(p->m_acc_hdl,
4513      -                                    &sge->FlagsLength);
4514      -                                flags |= ((uint32_t)(
4515      -                                    MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
4516      -                                    MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
4517      -                                    MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
4518      -                                    MPI2_SGE_FLAGS_SHIFT);
     4943 +        /*
     4944 +         * A little check to see if we need to round up the number
     4945 +         * of frames we need
     4946 +         */
     4947 +        if ((cookiec - (maxframe_sges - 1)) - (temp * sgemax) > 1) {
     4948 +                frames = (temp + 1);
     4949 +        } else {
     4950 +                frames = temp;
     4951 +        }
     4952 +        NDBG1(("mptsas_ieee_sge_chain: temp=%d, frames=%d", temp, frames));
     4953 +        dmap = cmd->cmd_sg;
     4954 +        ieeesge = (pMpi2IeeeSgeSimple64_t)(&frame->SGL);
     4955 +
     4956 +        /*
     4957 +         * First fill in the main frame
     4958 +         */
     4959 +        j = maxframe_sges - 1;
     4960 +        mptsas_ieee_sge_mainframe(cmd, frame, acc_hdl, j, 0);
     4961 +        dmap += j;
     4962 +        ieeesge += j;
     4963 +        j++;
     4964 +
     4965 +        /*
     4966 +         * Fill in the chain element in the main frame.
     4967 +         * About calculation on ChainOffset:
     4968 +         * 1. Struct msg_scsi_io_request has 4 double-words (16 bytes)
     4969 +         *    in the end reserved for SGL element storage
     4970 +         *    (MPI2_SGE_IO_UNION); we should count it in our
     4971 +         *    calculation.  See its definition in the header file.
     4972 +         * 2. Constant j is the counter of the current SGL element
     4973 +         *    that will be processed, and (j - 1) is the number of
     4974 +         *    SGL elements that have been processed (stored in the
     4975 +         *    main frame).
     4976 +         * 3. ChainOffset value should be in units of quad-words (16
     4977 +         *    bytes) so the last value should be divided by 16.
     4978 +         */
     4979 +        ddi_put8(acc_hdl, &frame->ChainOffset,
     4980 +            (sizeof (MPI2_SCSI_IO_REQUEST) -
     4981 +            sizeof (MPI2_SGE_IO_UNION) +
     4982 +            (j - 1) * sizeof (MPI2_IEEE_SGE_SIMPLE64)) >> 4);
     4983 +        ieeesgechain = (pMpi25IeeeSgeChain64_t)ieeesge;
     4984 +        chainflags = (MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
     4985 +            MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
     4986 +        ddi_put8(acc_hdl, &ieeesgechain->Flags, chainflags);
     4987 +
     4988 +        /*
     4989 +         * The size of the next frame is the accurate size of space
     4990 +         * (in bytes) used to store the SGL elements. j is the counter
     4991 +         * of SGL elements. (j - 1) is the number of SGL elements that
     4992 +         * have been processed (stored in frames).
     4993 +         */
     4994 +        if (frames >= 2) {
     4995 +                chainlength = mpt->m_req_frame_size /
     4996 +                    sizeof (MPI2_IEEE_SGE_SIMPLE64) *
     4997 +                    sizeof (MPI2_IEEE_SGE_SIMPLE64);
     4998 +        } else {
     4999 +                chainlength = ((cookiec - (j - 1)) *
     5000 +                    sizeof (MPI2_IEEE_SGE_SIMPLE64));
     5001 +        }
     5002 +
     5003 +        p = cmd->cmd_extra_frames;
     5004 +
     5005 +        ddi_put32(acc_hdl, &ieeesgechain->Length, chainlength);
     5006 +        ddi_put32(acc_hdl, &ieeesgechain->Address.Low,
     5007 +            p->m_phys_addr&0xffffffffull);
     5008 +        ddi_put32(acc_hdl, &ieeesgechain->Address.High, p->m_phys_addr>>32);
     5009 +
     5010 +        /*
     5011 +         * If there are more than 2 frames left we have to
     5012 +         * fill in the next chain offset to the location of
     5013 +         * the chain element in the next frame.
     5014 +         * sgemax is the number of simple elements in an extra
     5015 +         * frame. Note that the value NextChainOffset should be
     5016 +         * in double-words (4 bytes).
     5017 +         */
     5018 +        if (frames >= 2) {
     5019 +                ddi_put8(acc_hdl, &ieeesgechain->NextChainOffset,
     5020 +                    (sgemax * sizeof (MPI2_IEEE_SGE_SIMPLE64)) >> 4);
     5021 +        } else {
     5022 +                ddi_put8(acc_hdl, &ieeesgechain->NextChainOffset, 0);
     5023 +        }
4519 5024  
     5025 +        /*
     5026 +         * Jump to next frame;
     5027 +         * Starting here, chain buffers go into the per command SGL.
     5028 +         * This buffer is allocated when chain buffers are needed.
     5029 +         */
     5030 +        ieeesge = (pMpi2IeeeSgeSimple64_t)p->m_frames_addr;
     5031 +        i = cookiec;
     5032 +
     5033 +        /*
     5034 +         * Start filling in frames with SGE's.  If we
     5035 +         * reach the end of frame and still have SGE's
     5036 +         * to fill we need to add a chain element and
     5037 +         * use another frame.  j will be our counter
     5038 +         * for what cookie we are at and i will be
     5039 +         * the total cookiec. k is the current frame
     5040 +         */
     5041 +        for (k = 1; k <= frames; k++) {
     5042 +                for (l = 1; (l <= (sgemax + 1)) && (j <= i); j++, l++) {
     5043 +
     5044 +                        /*
     5045 +                         * If we have reached the end of frame
     5046 +                         * and we have more SGE's to fill in
     5047 +                         * we have to fill the final entry
     5048 +                         * with a chain element and then
     5049 +                         * continue to the next frame
     5050 +                         */
     5051 +                        if ((l == (sgemax + 1)) && (k != frames)) {
     5052 +                                ieeesgechain = (pMpi25IeeeSgeChain64_t)ieeesge;
     5053 +                                j--;
     5054 +                                chainflags =
     5055 +                                    MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
     5056 +                                    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
     5057 +                                ddi_put8(p->m_acc_hdl,
     5058 +                                    &ieeesgechain->Flags, chainflags);
4520 5059                                  /*
4521      -                                 * If we are at the end of the frame and
4522      -                                 * there is another frame to fill in
4523      -                                 * we set the last simple element as last
4524      -                                 * element
     5060 +                                 * k is the frame counter and (k + 1)
     5061 +                                 * is the number of the next frame.
     5062 +                                 * Note that frames are in contiguous
     5063 +                                 * memory space.
4525 5064                                   */
4526      -                                if ((l == sgemax) && (k != frames)) {
4527      -                                        flags |= ((uint32_t)
4528      -                                            (MPI2_SGE_FLAGS_LAST_ELEMENT) <<
4529      -                                            MPI2_SGE_FLAGS_SHIFT);
4530      -                                }
     5065 +                                nframe_phys_addr = p->m_phys_addr +
     5066 +                                    (mpt->m_req_frame_size * k);
     5067 +                                ddi_put32(p->m_acc_hdl,
     5068 +                                    &ieeesgechain->Address.Low,
     5069 +                                    nframe_phys_addr&0xffffffffull);
     5070 +                                ddi_put32(p->m_acc_hdl,
     5071 +                                    &ieeesgechain->Address.High,
     5072 +                                    nframe_phys_addr>>32);
4531 5073  
4532 5074                                  /*
4533      -                                 * If this is the final cookie we
4534      -                                 * indicate it by setting the flags
     5075 +                                 * If there are more than 2 frames left
     5076 +                                 * we have to next chain offset to
     5077 +                                 * the location of the chain element
     5078 +                                 * in the next frame and fill in the
     5079 +                                 * length of the next chain
4535 5080                                   */
4536      -                                if (j == i) {
4537      -                                        flags |= ((uint32_t)
4538      -                                            (MPI2_SGE_FLAGS_LAST_ELEMENT |
4539      -                                            MPI2_SGE_FLAGS_END_OF_BUFFER |
4540      -                                            MPI2_SGE_FLAGS_END_OF_LIST) <<
4541      -                                            MPI2_SGE_FLAGS_SHIFT);
4542      -                                }
4543      -                                if (cmd->cmd_flags & CFLAG_DMASEND) {
4544      -                                        flags |=
4545      -                                            (MPI2_SGE_FLAGS_HOST_TO_IOC <<
4546      -                                            MPI2_SGE_FLAGS_SHIFT);
     5081 +                                if ((frames - k) >= 2) {
     5082 +                                        ddi_put8(p->m_acc_hdl,
     5083 +                                            &ieeesgechain->NextChainOffset,
     5084 +                                            (sgemax *
     5085 +                                            sizeof (MPI2_IEEE_SGE_SIMPLE64))
     5086 +                                            >> 4);
     5087 +                                        ddi_put32(p->m_acc_hdl,
     5088 +                                            &ieeesgechain->Length,
     5089 +                                            mpt->m_req_frame_size /
     5090 +                                            sizeof (MPI2_IEEE_SGE_SIMPLE64) *
     5091 +                                            sizeof (MPI2_IEEE_SGE_SIMPLE64));
4547 5092                                  } else {
4548      -                                        flags |=
4549      -                                            (MPI2_SGE_FLAGS_IOC_TO_HOST <<
4550      -                                            MPI2_SGE_FLAGS_SHIFT);
     5093 +                                        /*
     5094 +                                         * This is the last frame. Set
     5095 +                                         * the NextChainOffset to 0 and
     5096 +                                         * Length is the total size of
     5097 +                                         * all remaining simple elements
     5098 +                                         */
     5099 +                                        ddi_put8(p->m_acc_hdl,
     5100 +                                            &ieeesgechain->NextChainOffset,
     5101 +                                            0);
     5102 +                                        ddi_put32(p->m_acc_hdl,
     5103 +                                            &ieeesgechain->Length,
     5104 +                                            (cookiec - j) *
     5105 +                                            sizeof (MPI2_IEEE_SGE_SIMPLE64));
4551 5106                                  }
4552      -                                ddi_put32(p->m_acc_hdl,
4553      -                                    &sge->FlagsLength, flags);
4554      -                                dmap++;
4555      -                                sge++;
     5107 +
     5108 +                                /* Jump to the next frame */
     5109 +                                ieeesge = (pMpi2IeeeSgeSimple64_t)
     5110 +                                    ((char *)p->m_frames_addr +
     5111 +                                    (int)mpt->m_req_frame_size * k);
     5112 +
     5113 +                                continue;
     5114 +                        }
     5115 +
     5116 +                        ddi_put32(p->m_acc_hdl,
     5117 +                            &ieeesge->Address.Low,
     5118 +                            dmap->addr.address64.Low);
     5119 +                        ddi_put32(p->m_acc_hdl,
     5120 +                            &ieeesge->Address.High,
     5121 +                            dmap->addr.address64.High);
     5122 +                        ddi_put32(p->m_acc_hdl,
     5123 +                            &ieeesge->Length, dmap->count);
     5124 +                        flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
     5125 +                            MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
     5126 +
     5127 +                        /*
     5128 +                         * If we are at the end of the frame and
     5129 +                         * there is another frame to fill in
     5130 +                         * do we need to do anything?
     5131 +                         * if ((l == sgemax) && (k != frames)) {
     5132 +                         * }
     5133 +                         */
     5134 +
     5135 +                        /*
     5136 +                         * If this is the final cookie set end of list.
     5137 +                         */
     5138 +                        if (j == i) {
     5139 +                                flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
4556 5140                          }
     5141 +
     5142 +                        ddi_put8(p->m_acc_hdl, &ieeesge->Flags, flags);
     5143 +                        dmap++;
     5144 +                        ieeesge++;
4557 5145                  }
     5146 +        }
4558 5147  
4559      -                /*
4560      -                 * Sync DMA with the chain buffers that were just created
4561      -                 */
4562      -                (void) ddi_dma_sync(p->m_dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
     5148 +        /*
     5149 +         * Sync DMA with the chain buffers that were just created
     5150 +         */
     5151 +        (void) ddi_dma_sync(p->m_dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
     5152 +}
     5153 +
     5154 +static void
     5155 +mptsas_sge_setup(mptsas_t *mpt, mptsas_cmd_t *cmd, uint32_t *control,
     5156 +    pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl)
     5157 +{
     5158 +        ASSERT(cmd->cmd_flags & CFLAG_DMAVALID);
     5159 +
     5160 +        NDBG1(("mptsas_sge_setup: cookiec=%d", cmd->cmd_cookiec));
     5161 +
     5162 +        /*
     5163 +         * Set read/write bit in control.
     5164 +         */
     5165 +        if (cmd->cmd_flags & CFLAG_DMASEND) {
     5166 +                *control |= MPI2_SCSIIO_CONTROL_WRITE;
     5167 +        } else {
     5168 +                *control |= MPI2_SCSIIO_CONTROL_READ;
     5169 +        }
     5170 +
     5171 +        ddi_put32(acc_hdl, &frame->DataLength, cmd->cmd_dmacount);
     5172 +
     5173 +        /*
     5174 +         * We have 4 cases here.  First where we can fit all the
     5175 +         * SG elements into the main frame, and the case
     5176 +         * where we can't. The SG element is also different when using
     5177 +         * MPI2.5 interface.
     5178 +         * If we have more cookies than we can attach to a frame
     5179 +         * we will need to use a chain element to point
     5180 +         * a location of memory where the rest of the S/G
     5181 +         * elements reside.
     5182 +         */
     5183 +        if (cmd->cmd_cookiec <= MPTSAS_MAX_FRAME_SGES64(mpt)) {
     5184 +                if (mpt->m_MPI25) {
     5185 +                        mptsas_ieee_sge_mainframe(cmd, frame, acc_hdl,
     5186 +                            cmd->cmd_cookiec,
     5187 +                            MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
     5188 +                } else {
     5189 +                        mptsas_sge_mainframe(cmd, frame, acc_hdl,
     5190 +                            cmd->cmd_cookiec,
     5191 +                            ((uint32_t)(MPI2_SGE_FLAGS_LAST_ELEMENT
     5192 +                            | MPI2_SGE_FLAGS_END_OF_BUFFER
     5193 +                            | MPI2_SGE_FLAGS_END_OF_LIST) <<
     5194 +                            MPI2_SGE_FLAGS_SHIFT));
     5195 +                }
     5196 +        } else {
     5197 +                if (mpt->m_MPI25) {
     5198 +                        mptsas_ieee_sge_chain(mpt, cmd, frame, acc_hdl);
     5199 +                } else {
     5200 +                        mptsas_sge_chain(mpt, cmd, frame, acc_hdl);
     5201 +                }
4563 5202          }
4564 5203  }
4565 5204  
4566 5205  /*
4567 5206   * Interrupt handling
4568 5207   * Utility routine.  Poll for status of a command sent to HBA
4569 5208   * without interrupts (a FLAG_NOINTR command).
4570 5209   */
4571 5210  int
4572 5211  mptsas_poll(mptsas_t *mpt, mptsas_cmd_t *poll_cmd, int polltime)
4573 5212  {
4574      -        int     rval = TRUE;
     5213 +        int             rval = TRUE;
     5214 +        uint32_t        int_mask;
4575 5215  
4576      -        NDBG5(("mptsas_poll: cmd=0x%p", (void *)poll_cmd));
     5216 +        NDBG5(("mptsas_poll: cmd=0x%p, flags 0x%x", (void *)poll_cmd,
     5217 +            poll_cmd->cmd_flags));
     5218 +
     5219 +        /*
     5220 +         * Get the current interrupt mask and disable interrupts.  When
     5221 +         * re-enabling ints, set mask to saved value.
     5222 +         */
     5223 +        int_mask = ddi_get32(mpt->m_datap, &mpt->m_reg->HostInterruptMask);
     5224 +        MPTSAS_DISABLE_INTR(mpt);
     5225 +
     5226 +        mpt->m_polled_intr = 1;
4577 5227  
4578 5228          if ((poll_cmd->cmd_flags & CFLAG_TM_CMD) == 0) {
4579 5229                  mptsas_restart_hba(mpt);
4580 5230          }
4581 5231  
4582 5232          /*
4583 5233           * Wait, using drv_usecwait(), long enough for the command to
4584 5234           * reasonably return from the target if the target isn't
4585 5235           * "dead".  A polled command may well be sent from scsi_poll, and
4586 5236           * there are retries built in to scsi_poll if the transport
↓ open down ↓ 31 lines elided ↑ open up ↑
4618 5268                              (STATE_GOT_BUS|STATE_GOT_TARGET|STATE_SENT_CMD);
4619 5269                  } else {
4620 5270  
4621 5271                          /* find and remove it from the waitq */
4622 5272                          NDBG5(("mptsas_poll: delete from waitq"));
4623 5273                          mptsas_waitq_delete(mpt, poll_cmd);
4624 5274                  }
4625 5275  
4626 5276          }
4627 5277          mptsas_fma_check(mpt, poll_cmd);
     5278 +
     5279 +        /*
     5280 +         * Clear polling flag, re-enable interrupts.
     5281 +         */
     5282 +        mpt->m_polled_intr = 0;
     5283 +        ddi_put32(mpt->m_datap, &mpt->m_reg->HostInterruptMask, int_mask);
     5284 +
     5285 +        /*
     5286 +         * If there are queued cmd, start them now.
     5287 +         */
     5288 +        if (mpt->m_waitq != NULL) {
     5289 +                mptsas_restart_waitq(mpt);
     5290 +        }
     5291 +
4628 5292          NDBG5(("mptsas_poll: done"));
4629 5293          return (rval);
4630 5294  }
4631 5295  
4632 5296  /*
4633 5297   * Used for polling cmds and TM function
4634 5298   */
4635 5299  static int
4636 5300  mptsas_wait_intr(mptsas_t *mpt, int polltime)
4637 5301  {
4638      -        int                             cnt;
     5302 +        int                             cnt, rval = FALSE;
4639 5303          pMpi2ReplyDescriptorsUnion_t    reply_desc_union;
4640      -        uint32_t                        int_mask;
     5304 +        mptsas_reply_pqueue_t           *rpqp;
4641 5305  
4642 5306          NDBG5(("mptsas_wait_intr"));
4643      -
4644      -        mpt->m_polled_intr = 1;
     5307 +        ASSERT(mutex_owned(&mpt->m_mutex));
4645 5308  
4646 5309          /*
4647      -         * Get the current interrupt mask and disable interrupts.  When
4648      -         * re-enabling ints, set mask to saved value.
     5310 +         * Keep polling for at least (polltime * 1000) seconds
4649 5311           */
4650      -        int_mask = ddi_get32(mpt->m_datap, &mpt->m_reg->HostInterruptMask);
4651      -        MPTSAS_DISABLE_INTR(mpt);
     5312 +        rpqp = mpt->m_rep_post_queues;
4652 5313  
4653 5314          /*
4654      -         * Keep polling for at least (polltime * 1000) seconds
     5315 +         * Drop the main mutex and grab the mutex for reply queue 0
4655 5316           */
     5317 +        mutex_exit(&mpt->m_mutex);
     5318 +        mutex_enter(&rpqp->rpq_mutex);
4656 5319          for (cnt = 0; cnt < polltime; cnt++) {
4657 5320                  (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
4658 5321                      DDI_DMA_SYNC_FORCPU);
4659 5322  
     5323 +                /*
     5324 +                 * Polled requests should only come back through
     5325 +                 * the first interrupt.
     5326 +                 */
4660 5327                  reply_desc_union = (pMpi2ReplyDescriptorsUnion_t)
4661      -                    MPTSAS_GET_NEXT_REPLY(mpt, mpt->m_post_index);
     5328 +                    MPTSAS_GET_NEXT_REPLY(rpqp, rpqp->rpq_index);
4662 5329  
4663 5330                  if (ddi_get32(mpt->m_acc_post_queue_hdl,
4664 5331                      &reply_desc_union->Words.Low) == 0xFFFFFFFF ||
4665 5332                      ddi_get32(mpt->m_acc_post_queue_hdl,
4666 5333                      &reply_desc_union->Words.High) == 0xFFFFFFFF) {
4667 5334                          drv_usecwait(1000);
4668 5335                          continue;
4669 5336                  }
4670 5337  
4671 5338                  /*
4672 5339                   * The reply is valid, process it according to its
4673 5340                   * type.
4674 5341                   */
4675      -                mptsas_process_intr(mpt, reply_desc_union);
4676      -
4677      -                if (++mpt->m_post_index == mpt->m_post_queue_depth) {
4678      -                        mpt->m_post_index = 0;
4679      -                }
     5342 +                mptsas_process_intr(mpt, rpqp, reply_desc_union);
4680 5343  
4681 5344                  /*
4682      -                 * Update the global reply index
     5345 +                 * Clear the reply descriptor for re-use.
4683 5346                   */
4684      -                ddi_put32(mpt->m_datap,
4685      -                    &mpt->m_reg->ReplyPostHostIndex, mpt->m_post_index);
4686      -                mpt->m_polled_intr = 0;
     5347 +                ddi_put64(mpt->m_acc_post_queue_hdl,
     5348 +                    &((uint64_t *)(void *)rpqp->rpq_queue)[rpqp->rpq_index],
     5349 +                    0xFFFFFFFFFFFFFFFF);
     5350 +                (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
     5351 +                    DDI_DMA_SYNC_FORDEV);
     5352 +
     5353 +                if (++rpqp->rpq_index == mpt->m_post_queue_depth) {
     5354 +                        rpqp->rpq_index = 0;
     5355 +                }
4687 5356  
4688 5357                  /*
4689      -                 * Re-enable interrupts and quit.
     5358 +                 * Update the reply index
4690 5359                   */
4691      -                ddi_put32(mpt->m_datap, &mpt->m_reg->HostInterruptMask,
4692      -                    int_mask);
4693      -                return (TRUE);
4694      -
     5360 +                ddi_put32(mpt->m_datap,
     5361 +                    &mpt->m_reg->ReplyPostHostIndex, rpqp->rpq_index);
     5362 +                rval = TRUE;
     5363 +                break;
4695 5364          }
4696 5365  
4697      -        /*
4698      -         * Clear polling flag, re-enable interrupts and quit.
4699      -         */
4700      -        mpt->m_polled_intr = 0;
4701      -        ddi_put32(mpt->m_datap, &mpt->m_reg->HostInterruptMask, int_mask);
4702      -        return (FALSE);
     5366 +        mutex_exit(&rpqp->rpq_mutex);
     5367 +        mutex_enter(&mpt->m_mutex);
     5368 +
     5369 +        return (rval);
4703 5370  }
4704 5371  
4705 5372  static void
4706 5373  mptsas_handle_scsi_io_success(mptsas_t *mpt,
     5374 +    mptsas_reply_pqueue_t *rpqp,
4707 5375      pMpi2ReplyDescriptorsUnion_t reply_desc)
4708 5376  {
4709 5377          pMpi2SCSIIOSuccessReplyDescriptor_t     scsi_io_success;
4710 5378          uint16_t                                SMID;
4711 5379          mptsas_slots_t                          *slots = mpt->m_active;
4712 5380          mptsas_cmd_t                            *cmd = NULL;
4713 5381          struct scsi_pkt                         *pkt;
4714 5382  
4715      -        ASSERT(mutex_owned(&mpt->m_mutex));
4716      -
4717 5383          scsi_io_success = (pMpi2SCSIIOSuccessReplyDescriptor_t)reply_desc;
4718 5384          SMID = ddi_get16(mpt->m_acc_post_queue_hdl, &scsi_io_success->SMID);
4719 5385  
4720 5386          /*
4721 5387           * This is a success reply so just complete the IO.  First, do a sanity
4722 5388           * check on the SMID.  The final slot is used for TM requests, which
4723 5389           * would not come into this reply handler.
4724 5390           */
4725 5391          if ((SMID == 0) || (SMID > slots->m_n_normal)) {
4726 5392                  mptsas_log(mpt, CE_WARN, "?Received invalid SMID of %d\n",
↓ open down ↓ 5 lines elided ↑ open up ↑
4732 5398          cmd = slots->m_slot[SMID];
4733 5399  
4734 5400          /*
4735 5401           * print warning and return if the slot is empty
4736 5402           */
4737 5403          if (cmd == NULL) {
4738 5404                  mptsas_log(mpt, CE_WARN, "?NULL command for successful SCSI IO "
4739 5405                      "in slot %d", SMID);
4740 5406                  return;
4741 5407          }
     5408 +        ASSERT(cmd->cmd_rpqidx == rpqp->rpq_num);
4742 5409  
4743 5410          pkt = CMD2PKT(cmd);
4744 5411          pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
4745 5412              STATE_GOT_STATUS);
4746 5413          if (cmd->cmd_flags & CFLAG_DMAVALID) {
4747 5414                  pkt->pkt_state |= STATE_XFERRED_DATA;
4748 5415          }
4749 5416          pkt->pkt_resid = 0;
4750 5417  
4751 5418          if (cmd->cmd_flags & CFLAG_PASSTHRU) {
4752 5419                  cmd->cmd_flags |= CFLAG_FINISHED;
4753 5420                  cv_broadcast(&mpt->m_passthru_cv);
4754 5421                  return;
4755      -        } else {
4756      -                mptsas_remove_cmd(mpt, cmd);
     5422 +        }
     5423 +        if (!(cmd->cmd_flags & CFLAG_TM_CMD)) {
     5424 +                if (cmd->cmd_flags & CFLAG_CMDIOC) {
     5425 +                        mutex_enter(&mpt->m_mutex);
     5426 +                        mptsas_remove_cmd(mpt, cmd);
     5427 +                        mutex_exit(&mpt->m_mutex);
     5428 +                } else {
     5429 +#ifdef MPTSAS_DEBUG
     5430 +                        /*
     5431 +                         * In order to test timeout for a command set
     5432 +                         * mptsas_test_timeout via mdb to avoid completion
     5433 +                         * processing here.
     5434 +                         */
     5435 +                        if (mptsas_test_timeout) {
     5436 +                                mptsas_test_timeout = 0;
     5437 +                                return;
     5438 +                        }
     5439 +#endif
     5440 +                        /*
     5441 +                         * This is the normal path, avoid grabbing
     5442 +                         * the m_mutex.
     5443 +                         */
     5444 +                        mptsas_remove_cmd_nomtx(mpt, cmd);
     5445 +                }
4757 5446          }
4758 5447  
4759 5448          if (cmd->cmd_flags & CFLAG_RETRY) {
4760 5449                  /*
4761 5450                   * The target returned QFULL or busy, do not add tihs
4762 5451                   * pkt to the doneq since the hba will retry
4763 5452                   * this cmd.
4764 5453                   *
4765 5454                   * The pkt has already been resubmitted in
4766 5455                   * mptsas_handle_qfull() or in mptsas_check_scsi_io_error().
4767 5456                   * Remove this cmd_flag here.
4768 5457                   */
4769 5458                  cmd->cmd_flags &= ~CFLAG_RETRY;
4770 5459          } else {
4771      -                mptsas_doneq_add(mpt, cmd);
     5460 +                mptsas_rpdoneq_add(mpt, rpqp, cmd);
4772 5461          }
4773 5462  }
4774 5463  
4775 5464  static void
4776 5465  mptsas_handle_address_reply(mptsas_t *mpt,
4777 5466      pMpi2ReplyDescriptorsUnion_t reply_desc)
4778 5467  {
4779 5468          pMpi2AddressReplyDescriptor_t   address_reply;
4780 5469          pMPI2DefaultReply_t             reply;
4781 5470          mptsas_fw_diagnostic_buffer_t   *pBuffer;
4782      -        uint32_t                        reply_addr;
     5471 +        uint32_t                        reply_addr, reply_frame_dma_baseaddr;
4783 5472          uint16_t                        SMID, iocstatus;
4784 5473          mptsas_slots_t                  *slots = mpt->m_active;
4785 5474          mptsas_cmd_t                    *cmd = NULL;
4786 5475          uint8_t                         function, buffer_type;
4787 5476          m_replyh_arg_t                  *args;
4788 5477          int                             reply_frame_no;
4789 5478  
4790 5479          ASSERT(mutex_owned(&mpt->m_mutex));
4791 5480  
4792 5481          address_reply = (pMpi2AddressReplyDescriptor_t)reply_desc;
4793 5482          reply_addr = ddi_get32(mpt->m_acc_post_queue_hdl,
4794 5483              &address_reply->ReplyFrameAddress);
4795 5484          SMID = ddi_get16(mpt->m_acc_post_queue_hdl, &address_reply->SMID);
4796 5485  
4797 5486          /*
4798 5487           * If reply frame is not in the proper range we should ignore this
4799 5488           * message and exit the interrupt handler.
4800 5489           */
4801      -        if ((reply_addr < mpt->m_reply_frame_dma_addr) ||
4802      -            (reply_addr >= (mpt->m_reply_frame_dma_addr +
     5490 +        reply_frame_dma_baseaddr = mpt->m_reply_frame_dma_addr & 0xfffffffful;
     5491 +        if ((reply_addr < reply_frame_dma_baseaddr) ||
     5492 +            (reply_addr >= (reply_frame_dma_baseaddr +
4803 5493              (mpt->m_reply_frame_size * mpt->m_max_replies))) ||
4804      -            ((reply_addr - mpt->m_reply_frame_dma_addr) %
     5494 +            ((reply_addr - reply_frame_dma_baseaddr) %
4805 5495              mpt->m_reply_frame_size != 0)) {
4806 5496                  mptsas_log(mpt, CE_WARN, "?Received invalid reply frame "
4807 5497                      "address 0x%x\n", reply_addr);
4808 5498                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
4809 5499                  return;
4810 5500          }
4811 5501  
4812 5502          (void) ddi_dma_sync(mpt->m_dma_reply_frame_hdl, 0, 0,
4813 5503              DDI_DMA_SYNC_FORCPU);
4814 5504          reply = (pMPI2DefaultReply_t)(mpt->m_reply_frame + (reply_addr -
4815      -            mpt->m_reply_frame_dma_addr));
     5505 +            reply_frame_dma_baseaddr));
4816 5506          function = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->Function);
4817 5507  
     5508 +        NDBG31(("mptsas_handle_address_reply: function 0x%x, reply_addr=0x%x",
     5509 +            function, reply_addr));
     5510 +
4818 5511          /*
4819 5512           * don't get slot information and command for events since these values
4820 5513           * don't exist
4821 5514           */
4822 5515          if ((function != MPI2_FUNCTION_EVENT_NOTIFICATION) &&
4823 5516              (function != MPI2_FUNCTION_DIAG_BUFFER_POST)) {
4824 5517                  /*
4825 5518                   * This could be a TM reply, which use the last allocated SMID,
4826 5519                   * so allow for that.
4827 5520                   */
↓ open down ↓ 8 lines elided ↑ open up ↑
4836 5529                  cmd = slots->m_slot[SMID];
4837 5530  
4838 5531                  /*
4839 5532                   * print warning and return if the slot is empty
4840 5533                   */
4841 5534                  if (cmd == NULL) {
4842 5535                          mptsas_log(mpt, CE_WARN, "?NULL command for address "
4843 5536                              "reply in slot %d", SMID);
4844 5537                          return;
4845 5538                  }
4846      -                if ((cmd->cmd_flags & CFLAG_PASSTHRU) ||
4847      -                    (cmd->cmd_flags & CFLAG_CONFIG) ||
4848      -                    (cmd->cmd_flags & CFLAG_FW_DIAG)) {
     5539 +                if ((cmd->cmd_flags &
     5540 +                    (CFLAG_PASSTHRU | CFLAG_CONFIG | CFLAG_FW_DIAG))) {
4849 5541                          cmd->cmd_rfm = reply_addr;
4850 5542                          cmd->cmd_flags |= CFLAG_FINISHED;
4851 5543                          cv_broadcast(&mpt->m_passthru_cv);
4852 5544                          cv_broadcast(&mpt->m_config_cv);
4853 5545                          cv_broadcast(&mpt->m_fw_diag_cv);
4854 5546                          return;
4855 5547                  } else if (!(cmd->cmd_flags & CFLAG_FW_CMD)) {
4856 5548                          mptsas_remove_cmd(mpt, cmd);
4857 5549                  }
4858 5550                  NDBG31(("\t\tmptsas_process_intr: slot=%d", SMID));
↓ open down ↓ 9 lines elided ↑ open up ↑
4868 5560          case MPI2_FUNCTION_SCSI_TASK_MGMT:
4869 5561                  cmd->cmd_rfm = reply_addr;
4870 5562                  mptsas_check_task_mgt(mpt, (pMpi2SCSIManagementReply_t)reply,
4871 5563                      cmd);
4872 5564                  break;
4873 5565          case MPI2_FUNCTION_FW_DOWNLOAD:
4874 5566                  cmd->cmd_flags |= CFLAG_FINISHED;
4875 5567                  cv_signal(&mpt->m_fw_cv);
4876 5568                  break;
4877 5569          case MPI2_FUNCTION_EVENT_NOTIFICATION:
4878      -                reply_frame_no = (reply_addr - mpt->m_reply_frame_dma_addr) /
     5570 +                reply_frame_no = (reply_addr - reply_frame_dma_baseaddr) /
4879 5571                      mpt->m_reply_frame_size;
4880 5572                  args = &mpt->m_replyh_args[reply_frame_no];
4881 5573                  args->mpt = (void *)mpt;
4882 5574                  args->rfm = reply_addr;
4883 5575  
4884 5576                  /*
4885 5577                   * Record the event if its type is enabled in
4886 5578                   * this mpt instance by ioctl.
4887 5579                   */
4888 5580                  mptsas_record_event(args);
↓ open down ↓ 90 lines elided ↑ open up ↑
4979 5671                  mpt->m_free_index = 0;
4980 5672          }
4981 5673          ddi_put32(mpt->m_datap, &mpt->m_reg->ReplyFreeHostIndex,
4982 5674              mpt->m_free_index);
4983 5675  
4984 5676          if (cmd->cmd_flags & CFLAG_FW_CMD)
4985 5677                  return;
4986 5678  
4987 5679          if (cmd->cmd_flags & CFLAG_RETRY) {
4988 5680                  /*
4989      -                 * The target returned QFULL or busy, do not add tihs
     5681 +                 * The target returned QFULL or busy, do not add this
4990 5682                   * pkt to the doneq since the hba will retry
4991 5683                   * this cmd.
4992 5684                   *
4993 5685                   * The pkt has already been resubmitted in
4994 5686                   * mptsas_handle_qfull() or in mptsas_check_scsi_io_error().
4995 5687                   * Remove this cmd_flag here.
4996 5688                   */
4997 5689                  cmd->cmd_flags &= ~CFLAG_RETRY;
4998 5690          } else {
4999 5691                  mptsas_doneq_add(mpt, cmd);
5000 5692          }
5001 5693  }
5002 5694  
     5695 +#ifdef MPTSAS_DEBUG
     5696 +static uint8_t mptsas_last_sense[256];
     5697 +#endif
     5698 +
5003 5699  static void
5004 5700  mptsas_check_scsi_io_error(mptsas_t *mpt, pMpi2SCSIIOReply_t reply,
5005 5701      mptsas_cmd_t *cmd)
5006 5702  {
5007 5703          uint8_t                 scsi_status, scsi_state;
5008      -        uint16_t                ioc_status;
     5704 +        uint16_t                ioc_status, cmd_rqs_len;
5009 5705          uint32_t                xferred, sensecount, responsedata, loginfo = 0;
5010 5706          struct scsi_pkt         *pkt;
5011 5707          struct scsi_arq_status  *arqstat;
5012      -        struct buf              *bp;
5013 5708          mptsas_target_t         *ptgt = cmd->cmd_tgt_addr;
5014 5709          uint8_t                 *sensedata = NULL;
5015      -
5016      -        if ((cmd->cmd_flags & (CFLAG_SCBEXTERN | CFLAG_EXTARQBUFVALID)) ==
5017      -            (CFLAG_SCBEXTERN | CFLAG_EXTARQBUFVALID)) {
5018      -                bp = cmd->cmd_ext_arq_buf;
5019      -        } else {
5020      -                bp = cmd->cmd_arq_buf;
5021      -        }
     5710 +        uint64_t                sas_wwn;
     5711 +        uint8_t                 phy;
     5712 +        char                    wwn_str[MPTSAS_WWN_STRLEN];
5022 5713  
5023 5714          scsi_status = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->SCSIStatus);
5024 5715          ioc_status = ddi_get16(mpt->m_acc_reply_frame_hdl, &reply->IOCStatus);
5025 5716          scsi_state = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->SCSIState);
5026 5717          xferred = ddi_get32(mpt->m_acc_reply_frame_hdl, &reply->TransferCount);
5027 5718          sensecount = ddi_get32(mpt->m_acc_reply_frame_hdl, &reply->SenseCount);
5028 5719          responsedata = ddi_get32(mpt->m_acc_reply_frame_hdl,
5029 5720              &reply->ResponseInfo);
5030 5721  
5031 5722          if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
     5723 +                sas_wwn = ptgt->m_addr.mta_wwn;
     5724 +                phy = ptgt->m_phynum;
     5725 +                if (sas_wwn == 0) {
     5726 +                        (void) sprintf(wwn_str, "p%x", phy);
     5727 +                } else {
     5728 +                        (void) sprintf(wwn_str, "w%016"PRIx64, sas_wwn);
     5729 +                }
5032 5730                  loginfo = ddi_get32(mpt->m_acc_reply_frame_hdl,
5033 5731                      &reply->IOCLogInfo);
5034 5732                  mptsas_log(mpt, CE_NOTE,
5035      -                    "?Log info 0x%x received for target %d.\n"
     5733 +                    "?Log info 0x%x received for target %d %s.\n"
5036 5734                      "\tscsi_status=0x%x, ioc_status=0x%x, scsi_state=0x%x",
5037      -                    loginfo, Tgt(cmd), scsi_status, ioc_status,
     5735 +                    loginfo, Tgt(cmd), wwn_str, scsi_status, ioc_status,
5038 5736                      scsi_state);
5039 5737          }
5040 5738  
5041 5739          NDBG31(("\t\tscsi_status=0x%x, ioc_status=0x%x, scsi_state=0x%x",
5042 5740              scsi_status, ioc_status, scsi_state));
5043 5741  
5044 5742          pkt = CMD2PKT(cmd);
5045 5743          *(pkt->pkt_scbp) = scsi_status;
5046 5744  
5047 5745          if (loginfo == 0x31170000) {
↓ open down ↓ 4 lines elided ↑ open up ↑
5052 5750                   */
5053 5751                  *(pkt->pkt_scbp) = STATUS_BUSY;
5054 5752                  return;
5055 5753          }
5056 5754  
5057 5755          if ((scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS) &&
5058 5756              ((ioc_status & MPI2_IOCSTATUS_MASK) ==
5059 5757              MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE)) {
5060 5758                  pkt->pkt_reason = CMD_INCOMPLETE;
5061 5759                  pkt->pkt_state |= STATE_GOT_BUS;
     5760 +                mutex_enter(&ptgt->m_t_mutex);
5062 5761                  if (ptgt->m_reset_delay == 0) {
5063 5762                          mptsas_set_throttle(mpt, ptgt,
5064 5763                              DRAIN_THROTTLE);
5065 5764                  }
     5765 +                mutex_exit(&ptgt->m_t_mutex);
5066 5766                  return;
5067 5767          }
5068 5768  
5069 5769          if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
5070 5770                  responsedata &= 0x000000FF;
5071 5771                  if (responsedata & MPTSAS_SCSI_RESPONSE_CODE_TLR_OFF) {
5072 5772                          mptsas_log(mpt, CE_NOTE, "Do not support the TLR\n");
5073 5773                          pkt->pkt_reason = CMD_TLR_OFF;
5074 5774                          return;
5075 5775                  }
5076 5776          }
5077 5777  
5078 5778  
5079 5779          switch (scsi_status) {
5080 5780          case MPI2_SCSI_STATUS_CHECK_CONDITION:
     5781 +                (void) ddi_dma_sync(mpt->m_dma_req_sense_hdl, 0, 0,
     5782 +                    DDI_DMA_SYNC_FORCPU);
5081 5783                  pkt->pkt_resid = (cmd->cmd_dmacount - xferred);
5082 5784                  arqstat = (void*)(pkt->pkt_scbp);
5083 5785                  arqstat->sts_rqpkt_status = *((struct scsi_status *)
5084 5786                      (pkt->pkt_scbp));
5085 5787                  pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
5086 5788                      STATE_SENT_CMD | STATE_GOT_STATUS | STATE_ARQ_DONE);
5087 5789                  if (cmd->cmd_flags & CFLAG_XARQ) {
5088 5790                          pkt->pkt_state |= STATE_XARQ_DONE;
5089 5791                  }
5090 5792                  if (pkt->pkt_resid != cmd->cmd_dmacount) {
5091 5793                          pkt->pkt_state |= STATE_XFERRED_DATA;
5092 5794                  }
5093 5795                  arqstat->sts_rqpkt_reason = pkt->pkt_reason;
5094 5796                  arqstat->sts_rqpkt_state  = pkt->pkt_state;
5095 5797                  arqstat->sts_rqpkt_state |= STATE_XFERRED_DATA;
5096 5798                  arqstat->sts_rqpkt_statistics = pkt->pkt_statistics;
5097 5799                  sensedata = (uint8_t *)&arqstat->sts_sensedata;
5098      -
5099      -                bcopy((uchar_t *)bp->b_un.b_addr, sensedata,
     5800 +#ifdef MPTSAS_DEBUG
     5801 +                bcopy((uchar_t *)cmd->cmd_arq_buf, mptsas_last_sense,
     5802 +                    cmd->cmd_rqslen);
     5803 +#endif
     5804 +                if (cmd->cmd_extrqslen != 0) {
     5805 +                        cmd_rqs_len = cmd->cmd_extrqslen;
     5806 +                } else {
     5807 +                        cmd_rqs_len = cmd->cmd_rqslen;
     5808 +                }
     5809 +                bcopy((uchar_t *)cmd->cmd_arq_buf, sensedata,
5100 5810                      ((cmd->cmd_rqslen >= sensecount) ? sensecount :
5101      -                    cmd->cmd_rqslen));
5102      -                arqstat->sts_rqpkt_resid = (cmd->cmd_rqslen - sensecount);
     5811 +                    cmd_rqs_len));
     5812 +                arqstat->sts_rqpkt_resid = (cmd_rqs_len - sensecount);
5103 5813                  cmd->cmd_flags |= CFLAG_CMDARQ;
5104 5814                  /*
5105 5815                   * Set proper status for pkt if autosense was valid
5106 5816                   */
5107 5817                  if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5108 5818                          struct scsi_status zero_status = { 0 };
5109 5819                          arqstat->sts_rqpkt_status = zero_status;
5110 5820                  }
5111 5821  
5112 5822                  /*
↓ open down ↓ 31 lines elided ↑ open up ↑
5144 5854                          topo_node->event = MPTSAS_DR_EVENT_RECONFIG_TARGET;
5145 5855                          topo_node->un.phymask = ptgt->m_addr.mta_phymask;
5146 5856                          topo_node->devhdl = ptgt->m_devhdl;
5147 5857                          topo_node->object = (void *)ptgt;
5148 5858                          topo_node->flags = MPTSAS_TOPO_FLAG_LUN_ASSOCIATED;
5149 5859  
5150 5860                          if ((ddi_taskq_dispatch(mpt->m_dr_taskq,
5151 5861                              mptsas_handle_dr,
5152 5862                              (void *)topo_node,
5153 5863                              DDI_NOSLEEP)) != DDI_SUCCESS) {
     5864 +                                kmem_free(topo_node,
     5865 +                                    sizeof (mptsas_topo_change_list_t));
5154 5866                                  mptsas_log(mpt, CE_NOTE, "mptsas start taskq"
5155 5867                                      "for handle SAS dynamic reconfigure"
5156 5868                                      "failed. \n");
5157 5869                          }
5158 5870                  }
5159 5871                  break;
5160 5872          case MPI2_SCSI_STATUS_GOOD:
5161 5873                  switch (ioc_status & MPI2_IOCSTATUS_MASK) {
5162 5874                  case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5163 5875                          pkt->pkt_reason = CMD_DEV_GONE;
5164 5876                          pkt->pkt_state |= STATE_GOT_BUS;
     5877 +                        mutex_enter(&ptgt->m_t_mutex);
5165 5878                          if (ptgt->m_reset_delay == 0) {
5166 5879                                  mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
5167 5880                          }
     5881 +                        mutex_exit(&ptgt->m_t_mutex);
5168 5882                          NDBG31(("lost disk for target%d, command:%x",
5169 5883                              Tgt(cmd), pkt->pkt_cdbp[0]));
5170 5884                          break;
5171 5885                  case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5172 5886                          NDBG31(("data overrun: xferred=%d", xferred));
5173 5887                          NDBG31(("dmacount=%d", cmd->cmd_dmacount));
5174 5888                          pkt->pkt_reason = CMD_DATA_OVR;
5175 5889                          pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET
5176 5890                              | STATE_SENT_CMD | STATE_GOT_STATUS
5177 5891                              | STATE_XFERRED_DATA);
↓ open down ↓ 4 lines elided ↑ open up ↑
5182 5896                          NDBG31(("data underrun: xferred=%d", xferred));
5183 5897                          NDBG31(("dmacount=%d", cmd->cmd_dmacount));
5184 5898                          pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET
5185 5899                              | STATE_SENT_CMD | STATE_GOT_STATUS);
5186 5900                          pkt->pkt_resid = (cmd->cmd_dmacount - xferred);
5187 5901                          if (pkt->pkt_resid != cmd->cmd_dmacount) {
5188 5902                                  pkt->pkt_state |= STATE_XFERRED_DATA;
5189 5903                          }
5190 5904                          break;
5191 5905                  case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5192      -                        mptsas_set_pkt_reason(mpt,
5193      -                            cmd, CMD_RESET, STAT_BUS_RESET);
     5906 +                        if (cmd->cmd_active_expiration <= gethrtime()) {
     5907 +                                /*
     5908 +                                 * When timeout requested, propagate
     5909 +                                 * proper reason and statistics to
     5910 +                                 * target drivers.
     5911 +                                 */
     5912 +                                mptsas_set_pkt_reason(mpt, cmd, CMD_TIMEOUT,
     5913 +                                    STAT_BUS_RESET | STAT_TIMEOUT);
     5914 +                        } else {
     5915 +                                mptsas_set_pkt_reason(mpt, cmd, CMD_RESET,
     5916 +                                    STAT_BUS_RESET);
     5917 +                        }
5194 5918                          break;
5195 5919                  case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5196 5920                  case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5197 5921                          mptsas_set_pkt_reason(mpt,
5198 5922                              cmd, CMD_RESET, STAT_DEV_RESET);
5199 5923                          break;
5200 5924                  case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5201 5925                  case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5202 5926                          pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET);
5203 5927                          mptsas_set_pkt_reason(mpt,
5204 5928                              cmd, CMD_TERMINATED, STAT_TERMINATED);
5205 5929                          break;
5206 5930                  case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
5207 5931                  case MPI2_IOCSTATUS_BUSY:
5208 5932                          /*
5209 5933                           * set throttles to drain
5210 5934                           */
5211 5935                          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
5212 5936                              ptgt = refhash_next(mpt->m_targets, ptgt)) {
5213      -                                mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
     5937 +                                mptsas_set_throttle_mtx(mpt, ptgt,
     5938 +                                    DRAIN_THROTTLE);
5214 5939                          }
5215 5940  
5216 5941                          /*
5217 5942                           * retry command
5218 5943                           */
5219      -                        cmd->cmd_flags |= CFLAG_RETRY;
5220      -                        cmd->cmd_pkt_flags |= FLAG_HEAD;
5221      -
5222      -                        (void) mptsas_accept_pkt(mpt, cmd);
     5944 +                        mptsas_retry_pkt(mpt, cmd);
5223 5945                          break;
5224 5946                  default:
5225 5947                          mptsas_log(mpt, CE_WARN,
5226 5948                              "unknown ioc_status = %x\n", ioc_status);
5227 5949                          mptsas_log(mpt, CE_CONT, "scsi_state = %x, transfer "
5228 5950                              "count = %x, scsi_status = %x", scsi_state,
5229 5951                              xferred, scsi_status);
5230 5952                          break;
5231 5953                  }
5232 5954                  break;
↓ open down ↓ 63 lines elided ↑ open up ↑
5296 6018                  break;
5297 6019          default:
5298 6020                  mptsas_log(mpt, CE_WARN, "Unknown task management type %d.",
5299 6021                      task_type);
5300 6022                  mptsas_log(mpt, CE_WARN, "ioc status = %x", ioc_status);
5301 6023                  break;
5302 6024          }
5303 6025  }
5304 6026  
5305 6027  static void
5306      -mptsas_doneq_thread(mptsas_doneq_thread_arg_t *arg)
     6028 +mptsas_doneq_thread(mptsas_thread_arg_t *arg)
5307 6029  {
5308 6030          mptsas_t                        *mpt = arg->mpt;
5309      -        uint64_t                        t = arg->t;
     6031 +        uint32_t                        t = arg->t;
5310 6032          mptsas_cmd_t                    *cmd;
5311 6033          struct scsi_pkt                 *pkt;
5312 6034          mptsas_doneq_thread_list_t      *item = &mpt->m_doneq_thread_id[t];
5313 6035  
5314 6036          mutex_enter(&item->mutex);
5315 6037          while (item->flag & MPTSAS_DONEQ_THREAD_ACTIVE) {
5316      -                if (!item->doneq) {
     6038 +                if (!item->dlist.dl_q) {
5317 6039                          cv_wait(&item->cv, &item->mutex);
5318 6040                  }
5319 6041                  pkt = NULL;
5320 6042                  if ((cmd = mptsas_doneq_thread_rm(mpt, t)) != NULL) {
5321 6043                          cmd->cmd_flags |= CFLAG_COMPLETED;
5322 6044                          pkt = CMD2PKT(cmd);
5323 6045                  }
5324 6046                  mutex_exit(&item->mutex);
5325 6047                  if (pkt) {
5326 6048                          mptsas_pkt_comp(pkt, cmd);
5327 6049                  }
5328 6050                  mutex_enter(&item->mutex);
5329 6051          }
5330 6052          mutex_exit(&item->mutex);
5331      -        mutex_enter(&mpt->m_doneq_mutex);
     6053 +        mutex_enter(&mpt->m_qthread_mutex);
5332 6054          mpt->m_doneq_thread_n--;
5333      -        cv_broadcast(&mpt->m_doneq_thread_cv);
5334      -        mutex_exit(&mpt->m_doneq_mutex);
     6055 +        cv_broadcast(&mpt->m_qthread_cv);
     6056 +        mutex_exit(&mpt->m_qthread_mutex);
5335 6057  }
5336 6058  
5337 6059  
5338 6060  /*
5339 6061   * mpt interrupt handler.
5340 6062   */
5341 6063  static uint_t
5342 6064  mptsas_intr(caddr_t arg1, caddr_t arg2)
5343 6065  {
5344 6066          mptsas_t                        *mpt = (void *)arg1;
     6067 +        mptsas_reply_pqueue_t           *rpqp;
     6068 +        int                             reply_q = (int)(uintptr_t)arg2;
5345 6069          pMpi2ReplyDescriptorsUnion_t    reply_desc_union;
5346      -        uchar_t                         did_reply = FALSE;
     6070 +        int                             found = 0, i, rpqidx;
     6071 +        size_t                          dma_sync_len;
     6072 +        off_t                           dma_sync_offset;
     6073 +        uint32_t                        istat;
5347 6074  
5348      -        NDBG1(("mptsas_intr: arg1 0x%p arg2 0x%p", (void *)arg1, (void *)arg2));
     6075 +        NDBG18(("mptsas_intr: arg1 0x%p reply_q 0x%d", (void *)arg1, reply_q));
5349 6076  
5350      -        mutex_enter(&mpt->m_mutex);
     6077 +        rpqp = &mpt->m_rep_post_queues[reply_q];
5351 6078  
5352 6079          /*
5353 6080           * If interrupts are shared by two channels then check whether this
5354 6081           * interrupt is genuinely for this channel by making sure first the
5355 6082           * chip is in high power state.
5356 6083           */
5357 6084          if ((mpt->m_options & MPTSAS_OPT_PM) &&
5358 6085              (mpt->m_power_level != PM_LEVEL_D0)) {
5359      -                mutex_exit(&mpt->m_mutex);
     6086 +                mpt->m_unclaimed_pm_interrupt_count++;
     6087 +                return (DDI_INTR_UNCLAIMED);
     6088 +        }
     6089 +
     6090 +        istat = MPTSAS_GET_ISTAT(mpt);
     6091 +        if (!(istat & MPI2_HIS_REPLY_DESCRIPTOR_INTERRUPT)) {
     6092 +                NDBG18(("Interrupt bit not set, istat 0x%x", istat));
     6093 +                mpt->m_unclaimed_no_interrupt_count++;
     6094 +                /*
     6095 +                 * Really need a good definition of when this is valid.
     6096 +                 * It appears not to be if you have multiple reply post
     6097 +                 * queues, there may be a better way - need LSI info.
     6098 +                 * For now just count them.
     6099 +                 */
     6100 +#if 0
5360 6101                  return (DDI_INTR_UNCLAIMED);
     6102 +#endif
5361 6103          }
5362 6104  
5363 6105          /*
5364 6106           * If polling, interrupt was triggered by some shared interrupt because
5365 6107           * IOC interrupts are disabled during polling, so polling routine will
5366 6108           * handle any replies.  Considering this, if polling is happening,
5367 6109           * return with interrupt unclaimed.
5368 6110           */
5369 6111          if (mpt->m_polled_intr) {
5370      -                mutex_exit(&mpt->m_mutex);
5371      -                mptsas_log(mpt, CE_WARN, "mpt_sas: Unclaimed interrupt");
     6112 +                mptsas_log(mpt, CE_WARN,
     6113 +                    "Unclaimed interrupt, rpq %d (Polling), istat 0x%x",
     6114 +                    reply_q, istat);
     6115 +                mpt->m_unclaimed_polled_interrupt_count++;
5372 6116                  return (DDI_INTR_UNCLAIMED);
5373 6117          }
5374 6118  
5375 6119          /*
5376      -         * Read the istat register.
     6120 +         * At the moment this is the only place the mutex is grabbed.
     6121 +         * So it should never fail!
5377 6122           */
5378      -        if ((INTPENDING(mpt)) != 0) {
5379      -                /*
5380      -                 * read fifo until empty.
5381      -                 */
     6123 +        if (mutex_tryenter(&rpqp->rpq_mutex) == 0) {
     6124 +                mutex_enter(&rpqp->rpq_mutex);
     6125 +                rpqp->rpq_intr_mutexbusy++;
     6126 +        }
     6127 +
     6128 +        dma_sync_len = mpt->m_post_queue_depth * 8;
     6129 +        dma_sync_offset = dma_sync_len * reply_q;
     6130 +        (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl,
     6131 +            dma_sync_offset, dma_sync_len, DDI_DMA_SYNC_FORCPU);
     6132 +
     6133 +        /*
     6134 +         * Go around the reply queue and process each descriptor until
     6135 +         * we get to the next unused one.
     6136 +         * It seems to be an occupational hazard that we get interrupts
     6137 +         * with nothing to do. These are counted below.
     6138 +         */
     6139 +        rpqidx = rpqp->rpq_index;
5382 6140  #ifndef __lock_lint
5383      -                _NOTE(CONSTCOND)
     6141 +        _NOTE(CONSTCOND)
5384 6142  #endif
5385      -                while (TRUE) {
5386      -                        (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
5387      -                            DDI_DMA_SYNC_FORCPU);
5388      -                        reply_desc_union = (pMpi2ReplyDescriptorsUnion_t)
5389      -                            MPTSAS_GET_NEXT_REPLY(mpt, mpt->m_post_index);
     6143 +        while (TRUE) {
     6144 +                reply_desc_union = (pMpi2ReplyDescriptorsUnion_t)
     6145 +                    MPTSAS_GET_NEXT_REPLY(rpqp, rpqidx);
5390 6146  
5391      -                        if (ddi_get32(mpt->m_acc_post_queue_hdl,
5392      -                            &reply_desc_union->Words.Low) == 0xFFFFFFFF ||
5393      -                            ddi_get32(mpt->m_acc_post_queue_hdl,
5394      -                            &reply_desc_union->Words.High) == 0xFFFFFFFF) {
5395      -                                break;
5396      -                        }
     6147 +                if (ddi_get32(mpt->m_acc_post_queue_hdl,
     6148 +                    &reply_desc_union->Words.Low) == 0xFFFFFFFF ||
     6149 +                    ddi_get32(mpt->m_acc_post_queue_hdl,
     6150 +                    &reply_desc_union->Words.High) == 0xFFFFFFFF) {
     6151 +                        break;
     6152 +                }
5397 6153  
5398      -                        /*
5399      -                         * The reply is valid, process it according to its
5400      -                         * type.  Also, set a flag for updating the reply index
5401      -                         * after they've all been processed.
5402      -                         */
5403      -                        did_reply = TRUE;
     6154 +                found++;
5404 6155  
5405      -                        mptsas_process_intr(mpt, reply_desc_union);
     6156 +                ASSERT(ddi_get8(mpt->m_acc_post_queue_hdl,
     6157 +                    &reply_desc_union->Default.MSIxIndex) == reply_q);
5406 6158  
5407      -                        /*
5408      -                         * Increment post index and roll over if needed.
5409      -                         */
5410      -                        if (++mpt->m_post_index == mpt->m_post_queue_depth) {
5411      -                                mpt->m_post_index = 0;
5412      -                        }
5413      -                }
     6159 +                /*
     6160 +                 * Process it according to its type.
     6161 +                 */
     6162 +                mptsas_process_intr(mpt, rpqp, reply_desc_union);
5414 6163  
5415 6164                  /*
5416      -                 * Update the global reply index if at least one reply was
5417      -                 * processed.
     6165 +                 * Clear the reply descriptor for re-use.
5418 6166                   */
5419      -                if (did_reply) {
5420      -                        ddi_put32(mpt->m_datap,
5421      -                            &mpt->m_reg->ReplyPostHostIndex, mpt->m_post_index);
     6167 +                ddi_put64(mpt->m_acc_post_queue_hdl,
     6168 +                    &((uint64_t *)(void *)rpqp->rpq_queue)[rpqidx],
     6169 +                    0xFFFFFFFFFFFFFFFF);
     6170 +
     6171 +                /*
     6172 +                 * Increment post index and roll over if needed.
     6173 +                 */
     6174 +                if (++rpqidx == mpt->m_post_queue_depth) {
     6175 +                        rpqidx = 0;
5422 6176                  }
5423      -        } else {
5424      -                mutex_exit(&mpt->m_mutex);
     6177 +        }
     6178 +
     6179 +        if (found == 0) {
     6180 +                rpqp->rpq_intr_unclaimed++;
     6181 +                mutex_exit(&rpqp->rpq_mutex);
     6182 +                mpt->m_unclaimed_nocmd_interrupt_count++;
5425 6183                  return (DDI_INTR_UNCLAIMED);
5426 6184          }
5427      -        NDBG1(("mptsas_intr complete"));
     6185 +        rpqp->rpq_index = rpqidx;
     6186 +
     6187 +        rpqp->rpq_intr_count++;
     6188 +        NDBG18(("mptsas_intr complete(%d), did %d loops", reply_q, found));
     6189 +
     6190 +        (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl,
     6191 +            dma_sync_offset, dma_sync_len, DDI_DMA_SYNC_FORDEV);
     6192 +
     6193 +        mpt->m_interrupt_count++;
     6194 +
     6195 +        /*
     6196 +         * Update the reply index if at least one reply was processed.
     6197 +         * For more than 8 reply queues on SAS3 controllers we have to do
     6198 +         * things a little different. See Chapter 20 in the MPI 2.5 spec.
     6199 +         */
     6200 +        if (mpt->m_post_reply_qcount > 8) {
     6201 +                /*
     6202 +                 * The offsets from the base are multiples of 0x10.
     6203 +                 * We are indexing into 32 bit quantities so calculate
     6204 +                 * the index for that.
     6205 +                 */
     6206 +                i = (reply_q&~0x7) >> 1;
     6207 +                ddi_put32(mpt->m_datap,
     6208 +                    &mpt->m_reg->SuppReplyPostHostIndex[i],
     6209 +                    rpqp->rpq_index |
     6210 +                    ((reply_q&0x7)<<MPI2_RPHI_MSIX_INDEX_SHIFT));
     6211 +                (void) ddi_get32(mpt->m_datap,
     6212 +                    &mpt->m_reg->SuppReplyPostHostIndex[i]);
     6213 +        } else {
     6214 +                ddi_put32(mpt->m_datap,
     6215 +                    &mpt->m_reg->ReplyPostHostIndex,
     6216 +                    rpqp->rpq_index | (reply_q<<MPI2_RPHI_MSIX_INDEX_SHIFT));
     6217 +                (void) ddi_get32(mpt->m_datap,
     6218 +                    &mpt->m_reg->ReplyPostHostIndex);
     6219 +        }
5428 6220  
5429 6221          /*
5430 6222           * If no helper threads are created, process the doneq in ISR. If
5431 6223           * helpers are created, use the doneq length as a metric to measure the
5432 6224           * load on the interrupt CPU. If it is long enough, which indicates the
5433 6225           * load is heavy, then we deliver the IO completions to the helpers.
5434 6226           * This measurement has some limitations, although it is simple and
5435 6227           * straightforward and works well for most of the cases at present.
     6228 +         * To always use the threads set mptsas_doneq_length_threshold_prop
     6229 +         * to zero in the mpt_sas3.conf file.
     6230 +         *
     6231 +         * Check the current reply queue done queue.
5436 6232           */
5437      -        if (!mpt->m_doneq_thread_n ||
5438      -            (mpt->m_doneq_len <= mpt->m_doneq_length_threshold)) {
5439      -                mptsas_doneq_empty(mpt);
5440      -        } else {
5441      -                mptsas_deliver_doneq_thread(mpt);
     6233 +        if (rpqp->rpq_dlist.dl_len) {
     6234 +                if (!mpt->m_doneq_thread_n ||
     6235 +                    (rpqp->rpq_dlist.dl_len <= mpt->m_doneq_length_threshold)) {
     6236 +                        mptsas_rpdoneq_empty(rpqp);
     6237 +                } else {
     6238 +                        mptsas_deliver_doneq_thread(mpt, &rpqp->rpq_dlist);
     6239 +                }
     6240 +        }
     6241 +
     6242 +        mutex_exit(&rpqp->rpq_mutex);
     6243 +
     6244 +        /*
     6245 +         * Check the main done queue. If we find something
     6246 +         * grab the mutex and check again before processing.
     6247 +         */
     6248 +        if (mpt->m_dlist.dl_len) {
     6249 +                mutex_enter(&mpt->m_mutex);
     6250 +                if (mpt->m_dlist.dl_len) {
     6251 +                        if (!mpt->m_doneq_thread_n ||
     6252 +                            (mpt->m_dlist.dl_len <=
     6253 +                            mpt->m_doneq_length_threshold)) {
     6254 +                                mptsas_doneq_empty(mpt);
     6255 +                        } else {
     6256 +                                mptsas_deliver_doneq_thread(mpt, &mpt->m_dlist);
     6257 +                        }
     6258 +                }
     6259 +                mutex_exit(&mpt->m_mutex);
5442 6260          }
5443 6261  
5444 6262          /*
5445 6263           * If there are queued cmd, start them now.
5446 6264           */
5447 6265          if (mpt->m_waitq != NULL) {
5448      -                mptsas_restart_waitq(mpt);
     6266 +                mutex_enter(&mpt->m_mutex);
     6267 +                if (mpt->m_waitq != NULL && mpt->m_polled_intr == 0) {
     6268 +                        mptsas_restart_waitq(mpt);
     6269 +                }
     6270 +                mutex_exit(&mpt->m_mutex);
5449 6271          }
5450      -
5451      -        mutex_exit(&mpt->m_mutex);
5452 6272          return (DDI_INTR_CLAIMED);
5453 6273  }
5454 6274  
5455 6275  static void
5456      -mptsas_process_intr(mptsas_t *mpt,
     6276 +mptsas_process_intr(mptsas_t *mpt, mptsas_reply_pqueue_t *rpqp,
5457 6277      pMpi2ReplyDescriptorsUnion_t reply_desc_union)
5458 6278  {
5459 6279          uint8_t reply_type;
5460 6280  
5461      -        ASSERT(mutex_owned(&mpt->m_mutex));
     6281 +        /*
     6282 +         * Should get here with the reply queue mutex held, but not
     6283 +         * the main mpt mutex. Want to avoid grabbing that during
     6284 +         * normal operations if possible.
     6285 +         */
     6286 +        ASSERT(mutex_owned(&rpqp->rpq_mutex));
5462 6287  
5463 6288          /*
5464 6289           * The reply is valid, process it according to its
5465 6290           * type.  Also, set a flag for updated the reply index
5466 6291           * after they've all been processed.
5467 6292           */
5468 6293          reply_type = ddi_get8(mpt->m_acc_post_queue_hdl,
5469 6294              &reply_desc_union->Default.ReplyFlags);
     6295 +        NDBG18(("mptsas_process_intr(rpq %d) reply_type 0x%x", rpqp->rpq_num,
     6296 +            reply_type));
5470 6297          reply_type &= MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
5471      -        if (reply_type == MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS) {
5472      -                mptsas_handle_scsi_io_success(mpt, reply_desc_union);
     6298 +        if (reply_type == MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
     6299 +            reply_type == MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS) {
     6300 +                mptsas_handle_scsi_io_success(mpt, rpqp, reply_desc_union);
5473 6301          } else if (reply_type == MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
     6302 +                mutex_enter(&mpt->m_mutex);
5474 6303                  mptsas_handle_address_reply(mpt, reply_desc_union);
     6304 +                mutex_exit(&mpt->m_mutex);
5475 6305          } else {
5476 6306                  mptsas_log(mpt, CE_WARN, "?Bad reply type %x", reply_type);
5477 6307                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
5478 6308          }
5479      -
5480      -        /*
5481      -         * Clear the reply descriptor for re-use and increment
5482      -         * index.
5483      -         */
5484      -        ddi_put64(mpt->m_acc_post_queue_hdl,
5485      -            &((uint64_t *)(void *)mpt->m_post_queue)[mpt->m_post_index],
5486      -            0xFFFFFFFFFFFFFFFF);
5487      -        (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
5488      -            DDI_DMA_SYNC_FORDEV);
5489 6309  }
5490 6310  
5491 6311  /*
5492 6312   * handle qfull condition
5493 6313   */
5494 6314  static void
5495 6315  mptsas_handle_qfull(mptsas_t *mpt, mptsas_cmd_t *cmd)
5496 6316  {
5497 6317          mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
5498 6318  
     6319 +        mutex_enter(&ptgt->m_t_mutex);
5499 6320          if ((++cmd->cmd_qfull_retries > ptgt->m_qfull_retries) ||
5500 6321              (ptgt->m_qfull_retries == 0)) {
5501 6322                  /*
5502 6323                   * We have exhausted the retries on QFULL, or,
5503 6324                   * the target driver has indicated that it
5504 6325                   * wants to handle QFULL itself by setting
5505 6326                   * qfull-retries capability to 0. In either case
5506 6327                   * we want the target driver's QFULL handling
5507 6328                   * to kick in. We do this by having pkt_reason
5508 6329                   * as CMD_CMPLT and pkt_scbp as STATUS_QFULL.
5509 6330                   */
5510 6331                  mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
5511 6332          } else {
5512 6333                  if (ptgt->m_reset_delay == 0) {
5513 6334                          ptgt->m_t_throttle =
5514 6335                              max((ptgt->m_t_ncmds - 2), 0);
5515 6336                  }
     6337 +                mutex_exit(&ptgt->m_t_mutex);
5516 6338  
5517      -                cmd->cmd_pkt_flags |= FLAG_HEAD;
5518 6339                  cmd->cmd_flags &= ~(CFLAG_TRANFLAG);
5519      -                cmd->cmd_flags |= CFLAG_RETRY;
5520 6340  
5521      -                (void) mptsas_accept_pkt(mpt, cmd);
     6341 +                mptsas_retry_pkt(mpt, cmd);
5522 6342  
     6343 +                mutex_enter(&ptgt->m_t_mutex);
5523 6344                  /*
5524 6345                   * when target gives queue full status with no commands
5525 6346                   * outstanding (m_t_ncmds == 0), throttle is set to 0
5526 6347                   * (HOLD_THROTTLE), and the queue full handling start
5527 6348                   * (see psarc/1994/313); if there are commands outstanding,
5528 6349                   * throttle is set to (m_t_ncmds - 2)
5529 6350                   */
5530 6351                  if (ptgt->m_t_throttle == HOLD_THROTTLE) {
5531 6352                          /*
5532 6353                           * By setting throttle to QFULL_THROTTLE, we
↓ open down ↓ 2 lines elided ↑ open up ↑
5535 6356                           * their throttles to be cleared.
5536 6357                           */
5537 6358                          mptsas_set_throttle(mpt, ptgt, QFULL_THROTTLE);
5538 6359                          if (mpt->m_restart_cmd_timeid == 0) {
5539 6360                                  mpt->m_restart_cmd_timeid =
5540 6361                                      timeout(mptsas_restart_cmd, mpt,
5541 6362                                      ptgt->m_qfull_retry_interval);
5542 6363                          }
5543 6364                  }
5544 6365          }
     6366 +        mutex_exit(&ptgt->m_t_mutex);
5545 6367  }
5546 6368  
5547 6369  mptsas_phymask_t
5548 6370  mptsas_physport_to_phymask(mptsas_t *mpt, uint8_t physport)
5549 6371  {
5550 6372          mptsas_phymask_t        phy_mask = 0;
5551 6373          uint8_t                 i = 0;
5552 6374  
5553      -        NDBG20(("mptsas%d physport_to_phymask enter", mpt->m_instance));
     6375 +        NDBG20(("mptsas3%d physport_to_phymask enter", mpt->m_instance));
5554 6376  
5555 6377          ASSERT(mutex_owned(&mpt->m_mutex));
5556 6378  
5557 6379          /*
5558 6380           * If physport is 0xFF, this is a RAID volume.  Use phymask of 0.
5559 6381           */
5560 6382          if (physport == 0xFF) {
5561 6383                  return (0);
5562 6384          }
5563 6385  
5564 6386          for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
5565 6387                  if (mpt->m_phy_info[i].attached_devhdl &&
5566 6388                      (mpt->m_phy_info[i].phy_mask != 0) &&
5567 6389                      (mpt->m_phy_info[i].port_num == physport)) {
5568 6390                          phy_mask = mpt->m_phy_info[i].phy_mask;
5569 6391                          break;
5570 6392                  }
5571 6393          }
5572      -        NDBG20(("mptsas%d physport_to_phymask:physport :%x phymask :%x, ",
     6394 +        NDBG20(("mptsas3%d physport_to_phymask:physport :%x phymask :%x, ",
5573 6395              mpt->m_instance, physport, phy_mask));
5574 6396          return (phy_mask);
5575 6397  }
5576 6398  
5577 6399  /*
5578 6400   * mpt free device handle after device gone, by use of passthrough
5579 6401   */
5580 6402  static int
5581 6403  mptsas_free_devhdl(mptsas_t *mpt, uint16_t devhdl)
5582 6404  {
↓ open down ↓ 33 lines elided ↑ open up ↑
5616 6438  }
5617 6439  
5618 6440  static void
5619 6441  mptsas_update_phymask(mptsas_t *mpt)
5620 6442  {
5621 6443          mptsas_phymask_t mask = 0, phy_mask;
5622 6444          char            *phy_mask_name;
5623 6445          uint8_t         current_port;
5624 6446          int             i, j;
5625 6447  
5626      -        NDBG20(("mptsas%d update phymask ", mpt->m_instance));
     6448 +        NDBG20(("mptsas3%d update phymask ", mpt->m_instance));
5627 6449  
5628 6450          ASSERT(mutex_owned(&mpt->m_mutex));
5629 6451  
5630 6452          (void) mptsas_get_sas_io_unit_page(mpt);
5631 6453  
5632 6454          phy_mask_name = kmem_zalloc(MPTSAS_MAX_PHYS, KM_SLEEP);
5633 6455  
5634 6456          for (i = 0; i < mpt->m_num_phys; i++) {
5635 6457                  phy_mask = 0x00;
5636 6458  
↓ open down ↓ 25 lines elided ↑ open up ↑
5662 6484  
5663 6485                  mutex_exit(&mpt->m_mutex);
5664 6486                  /*
5665 6487                   * register a iport, if the port has already been existed
5666 6488                   * SCSA will do nothing and just return.
5667 6489                   */
5668 6490                  (void) scsi_hba_iport_register(mpt->m_dip, phy_mask_name);
5669 6491                  mutex_enter(&mpt->m_mutex);
5670 6492          }
5671 6493          kmem_free(phy_mask_name, MPTSAS_MAX_PHYS);
5672      -        NDBG20(("mptsas%d update phymask return", mpt->m_instance));
     6494 +        NDBG20(("mptsas3%d update phymask return", mpt->m_instance));
5673 6495  }
5674 6496  
5675 6497  /*
5676 6498   * mptsas_handle_dr is a task handler for DR, the DR action includes:
5677 6499   * 1. Directly attched Device Added/Removed.
5678 6500   * 2. Expander Device Added/Removed.
5679 6501   * 3. Indirectly Attached Device Added/Expander.
5680 6502   * 4. LUNs of a existing device status change.
5681 6503   * 5. RAID volume created/deleted.
5682 6504   * 6. Member of RAID volume is released because of RAID deletion.
5683 6505   * 7. Physical disks are removed because of RAID creation.
5684 6506   */
5685 6507  static void
5686 6508  mptsas_handle_dr(void *args) {
5687 6509          mptsas_topo_change_list_t       *topo_node = NULL;
5688 6510          mptsas_topo_change_list_t       *save_node = NULL;
5689 6511          mptsas_t                        *mpt;
5690 6512          dev_info_t                      *parent = NULL;
5691 6513          mptsas_phymask_t                phymask = 0;
5692      -        char                            *phy_mask_name;
     6514 +        char                            phy_mask_name[MPTSAS_MAX_PHYS];
5693 6515          uint8_t                         flags = 0, physport = 0xff;
5694 6516          uint8_t                         port_update = 0;
5695 6517          uint_t                          event;
5696 6518  
5697 6519          topo_node = (mptsas_topo_change_list_t *)args;
5698 6520  
5699 6521          mpt = topo_node->mpt;
5700 6522          event = topo_node->event;
5701 6523          flags = topo_node->flags;
5702 6524  
5703      -        phy_mask_name = kmem_zalloc(MPTSAS_MAX_PHYS, KM_SLEEP);
5704      -
5705      -        NDBG20(("mptsas%d handle_dr enter", mpt->m_instance));
     6525 +        NDBG20(("mptsas3%d handle_dr enter", mpt->m_instance));
5706 6526  
5707 6527          switch (event) {
5708 6528          case MPTSAS_DR_EVENT_RECONFIG_TARGET:
5709 6529                  if ((flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) ||
5710 6530                      (flags == MPTSAS_TOPO_FLAG_EXPANDER_ATTACHED_DEVICE) ||
5711 6531                      (flags == MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED)) {
5712 6532                          /*
5713 6533                           * Direct attached or expander attached device added
5714 6534                           * into system or a Phys Disk that is being unhidden.
5715 6535                           */
↓ open down ↓ 21 lines elided ↑ open up ↑
5737 6557                   * port have changed to online. So when expander added or
5738 6558                   * directly attached device online event come, we force to
5739 6559                   * update port information by issueing SAS IO Unit Page and
5740 6560                   * update PHYMASKs.
5741 6561                   */
5742 6562                  (void) mptsas_update_phymask(mpt);
5743 6563                  mpt->m_port_chng = 0;
5744 6564  
5745 6565          }
5746 6566          mutex_exit(&mpt->m_mutex);
     6567 +
5747 6568          while (topo_node) {
5748 6569                  phymask = 0;
     6570 +                flags = topo_node->flags;
     6571 +                event = topo_node->event;
     6572 +                if (event == MPTSAS_DR_EVENT_REMOVE_HANDLE) {
     6573 +                        goto handle_topo_change;
     6574 +                }
     6575 +                if ((event == MPTSAS_DR_EVENT_RECONFIG_TARGET) &&
     6576 +                    (flags == MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED)) {
     6577 +                        /*
     6578 +                         * There is no any field in IR_CONFIG_CHANGE
     6579 +                         * event indicate physport/phynum, let's get
     6580 +                         * parent after SAS Device Page0 request.
     6581 +                         */
     6582 +                        goto handle_topo_change;
     6583 +                }
     6584 +
5749 6585                  if (parent == NULL) {
5750 6586                          physport = topo_node->un.physport;
5751      -                        event = topo_node->event;
5752      -                        flags = topo_node->flags;
5753 6587                          if (event & (MPTSAS_DR_EVENT_OFFLINE_TARGET |
5754 6588                              MPTSAS_DR_EVENT_OFFLINE_SMP)) {
5755 6589                                  /*
5756 6590                                   * For all offline events, phymask is known
5757 6591                                   */
5758 6592                                  phymask = topo_node->un.phymask;
5759 6593                                  goto find_parent;
5760 6594                          }
5761      -                        if (event & MPTSAS_TOPO_FLAG_REMOVE_HANDLE) {
5762      -                                goto handle_topo_change;
5763      -                        }
5764 6595                          if (flags & MPTSAS_TOPO_FLAG_LUN_ASSOCIATED) {
5765 6596                                  phymask = topo_node->un.phymask;
5766 6597                                  goto find_parent;
5767 6598                          }
5768 6599  
5769      -                        if ((flags ==
5770      -                            MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED) &&
5771      -                            (event == MPTSAS_DR_EVENT_RECONFIG_TARGET)) {
5772      -                                /*
5773      -                                 * There is no any field in IR_CONFIG_CHANGE
5774      -                                 * event indicate physport/phynum, let's get
5775      -                                 * parent after SAS Device Page0 request.
5776      -                                 */
5777      -                                goto handle_topo_change;
5778      -                        }
5779      -
5780 6600                          mutex_enter(&mpt->m_mutex);
5781 6601                          if (flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) {
5782 6602                                  /*
5783 6603                                   * If the direct attached device added or a
5784 6604                                   * phys disk is being unhidden, argument
5785 6605                                   * physport actually is PHY#, so we have to get
5786 6606                                   * phymask according PHY#.
5787 6607                                   */
5788 6608                                  physport = mpt->m_phy_info[physport].port_num;
5789 6609                          }
5790 6610  
5791 6611                          /*
5792 6612                           * Translate physport to phymask so that we can search
5793 6613                           * parent dip.
5794 6614                           */
5795      -                        phymask = mptsas_physport_to_phymask(mpt,
5796      -                            physport);
     6615 +                        phymask = mptsas_physport_to_phymask(mpt, physport);
5797 6616                          mutex_exit(&mpt->m_mutex);
5798 6617  
5799 6618  find_parent:
5800 6619                          bzero(phy_mask_name, MPTSAS_MAX_PHYS);
5801 6620                          /*
5802 6621                           * For RAID topology change node, write the iport name
5803 6622                           * as v0.
5804 6623                           */
5805 6624                          if (flags & MPTSAS_TOPO_FLAG_RAID_ASSOCIATED) {
5806 6625                                  (void) sprintf(phy_mask_name, "v0");
5807 6626                          } else {
5808 6627                                  /*
5809      -                                 * phymask can bo 0 if the drive has been
     6628 +                                 * phymask can be 0 if the drive has been
5810 6629                                   * pulled by the time an add event is
5811 6630                                   * processed.  If phymask is 0, just skip this
5812 6631                                   * event and continue.
5813 6632                                   */
5814 6633                                  if (phymask == 0) {
5815      -                                        mutex_enter(&mpt->m_mutex);
5816 6634                                          save_node = topo_node;
5817 6635                                          topo_node = topo_node->next;
5818 6636                                          ASSERT(save_node);
5819 6637                                          kmem_free(save_node,
5820 6638                                              sizeof (mptsas_topo_change_list_t));
5821      -                                        mutex_exit(&mpt->m_mutex);
5822      -
5823 6639                                          parent = NULL;
5824 6640                                          continue;
5825 6641                                  }
5826 6642                                  (void) sprintf(phy_mask_name, "%x", phymask);
5827 6643                          }
5828 6644                          parent = scsi_hba_iport_find(mpt->m_dip,
5829 6645                              phy_mask_name);
5830 6646                          if (parent == NULL) {
5831 6647                                  mptsas_log(mpt, CE_WARN, "Failed to find an "
5832      -                                    "iport, should not happen!");
5833      -                                goto out;
     6648 +                                    "iport for \"%s\", should not happen!",
     6649 +                                    phy_mask_name);
     6650 +                                save_node = topo_node;
     6651 +                                topo_node = topo_node->next;
     6652 +                                ASSERT(save_node);
     6653 +                                kmem_free(save_node,
     6654 +                                    sizeof (mptsas_topo_change_list_t));
     6655 +                                continue;
5834 6656                          }
5835 6657  
5836 6658                  }
5837      -                ASSERT(parent);
5838      -handle_topo_change:
     6659 +                ASSERT(parent);
     6660 +handle_topo_change:
     6661 +
     6662 +                mutex_enter(&mpt->m_mutex);
     6663 +                /*
     6664 +                 * If HBA is being reset, don't perform operations depending
     6665 +                 * on the IOC. We must free the topo list, however.
     6666 +                 */
     6667 +                if (!mpt->m_in_reset)
     6668 +                        mptsas_handle_topo_change(topo_node, parent);
     6669 +                else
     6670 +                        NDBG20(("skipping topo change received during reset"));
     6671 +                mutex_exit(&mpt->m_mutex);
     6672 +                save_node = topo_node;
     6673 +                topo_node = topo_node->next;
     6674 +                ASSERT(save_node);
     6675 +                kmem_free(save_node, sizeof (mptsas_topo_change_list_t));
     6676 +
     6677 +                if ((flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) ||
     6678 +                    (flags == MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED) ||
     6679 +                    (flags == MPTSAS_TOPO_FLAG_RAID_ASSOCIATED)) {
     6680 +                        /*
     6681 +                         * If direct attached device associated, make sure
     6682 +                         * reset the parent before start the next one. But
     6683 +                         * all devices associated with expander shares the
     6684 +                         * parent.  Also, reset parent if this is for RAID.
     6685 +                         */
     6686 +                        parent = NULL;
     6687 +                }
     6688 +        }
     6689 +}
     6690 +
     6691 +static void
     6692 +mptsas_offline_target(mptsas_t *mpt, mptsas_target_t *ptgt,
     6693 +    uint8_t topo_flags, dev_info_t *parent)
     6694 +{
     6695 +        uint64_t        sas_wwn = 0;
     6696 +        uint8_t         phy;
     6697 +        char            wwn_str[MPTSAS_WWN_STRLEN];
     6698 +        uint16_t        devhdl;
     6699 +        int             circ = 0, circ1 = 0;
     6700 +        int             rval = 0;
     6701 +
     6702 +        sas_wwn = ptgt->m_addr.mta_wwn;
     6703 +        phy = ptgt->m_phynum;
     6704 +        devhdl = ptgt->m_devhdl;
     6705 +
     6706 +        if (sas_wwn) {
     6707 +                (void) sprintf(wwn_str, "w%016"PRIx64, sas_wwn);
     6708 +        } else {
     6709 +                (void) sprintf(wwn_str, "p%x", phy);
     6710 +        }
     6711 +
     6712 +        /*
     6713 +         * Abort all outstanding command on the device
     6714 +         */
     6715 +        rval = mptsas_do_scsi_reset(mpt, devhdl);
     6716 +        if (rval) {
     6717 +                NDBG20(("mptsas3%d: mptsas_offline_target: reset target "
     6718 +                    "before offline devhdl:%x, phymask:%x, rval:%x",
     6719 +                    mpt->m_instance, ptgt->m_devhdl,
     6720 +                    ptgt->m_addr.mta_phymask, rval));
     6721 +        }
     6722 +
     6723 +        mutex_exit(&mpt->m_mutex);
     6724 +
     6725 +        ndi_devi_enter(scsi_vhci_dip, &circ);
     6726 +        ndi_devi_enter(parent, &circ1);
     6727 +        rval = mptsas_offline_targetdev(parent, wwn_str);
     6728 +        ndi_devi_exit(parent, circ1);
     6729 +        ndi_devi_exit(scsi_vhci_dip, circ);
     6730 +        NDBG20(("mptsas3%d: mptsas_offline_target %s devhdl:%x, "
     6731 +            "phymask:%x, rval:%x", mpt->m_instance, wwn_str,
     6732 +            ptgt->m_devhdl, ptgt->m_addr.mta_phymask, rval));
     6733 +
     6734 +        /*
     6735 +         * Clear parent's props for SMHBA support
     6736 +         */
     6737 +        if (topo_flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) {
     6738 +                if (ddi_prop_update_string(DDI_DEV_T_NONE, parent,
     6739 +                    SCSI_ADDR_PROP_ATTACHED_PORT, "") !=
     6740 +                    DDI_PROP_SUCCESS) {
     6741 +                        (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
     6742 +                            SCSI_ADDR_PROP_ATTACHED_PORT);
     6743 +                        mptsas_log(mpt, CE_WARN, "mptsas attached port "
     6744 +                            "prop update failed");
     6745 +                }
     6746 +                if (ddi_prop_update_int(DDI_DEV_T_NONE, parent,
     6747 +                    MPTSAS_NUM_PHYS, 0) != DDI_PROP_SUCCESS) {
     6748 +                        (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
     6749 +                            MPTSAS_NUM_PHYS);
     6750 +                        mptsas_log(mpt, CE_WARN, "mptsas num phys "
     6751 +                            "prop update failed");
     6752 +                }
     6753 +                if (ddi_prop_update_int(DDI_DEV_T_NONE, parent,
     6754 +                    MPTSAS_VIRTUAL_PORT, 1) != DDI_PROP_SUCCESS) {
     6755 +                        (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
     6756 +                            MPTSAS_VIRTUAL_PORT);
     6757 +                        mptsas_log(mpt, CE_WARN, "mptsas virtual port "
     6758 +                            "prop update failed");
     6759 +                }
     6760 +        }
5839 6761  
5840      -                mutex_enter(&mpt->m_mutex);
     6762 +        mutex_enter(&mpt->m_mutex);
     6763 +        ptgt->m_led_status = 0;
     6764 +        (void) mptsas_flush_led_status(mpt, ptgt);
     6765 +        if (rval == DDI_SUCCESS) {
     6766 +                mutex_destroy(&ptgt->m_t_mutex);
     6767 +                refhash_remove(mpt->m_targets, ptgt);
     6768 +                ptgt = NULL;
     6769 +        } else {
5841 6770                  /*
5842      -                 * If HBA is being reset, don't perform operations depending
5843      -                 * on the IOC. We must free the topo list, however.
5844      -                 */
5845      -                if (!mpt->m_in_reset)
5846      -                        mptsas_handle_topo_change(topo_node, parent);
5847      -                else
5848      -                        NDBG20(("skipping topo change received during reset"));
5849      -                save_node = topo_node;
5850      -                topo_node = topo_node->next;
5851      -                ASSERT(save_node);
5852      -                kmem_free(save_node, sizeof (mptsas_topo_change_list_t));
5853      -                mutex_exit(&mpt->m_mutex);
5854      -
5855      -                if ((flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) ||
5856      -                    (flags == MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED) ||
5857      -                    (flags == MPTSAS_TOPO_FLAG_RAID_ASSOCIATED)) {
5858      -                        /*
5859      -                         * If direct attached device associated, make sure
5860      -                         * reset the parent before start the next one. But
5861      -                         * all devices associated with expander shares the
5862      -                         * parent.  Also, reset parent if this is for RAID.
5863      -                         */
5864      -                        parent = NULL;
5865      -                }
     6771 +                 * clean DR_INTRANSITION flag to allow I/O down to
     6772 +                 * PHCI driver since failover finished.
     6773 +                 * Invalidate the devhdl
     6774 +                 */
     6775 +                ptgt->m_devhdl = MPTSAS_INVALID_DEVHDL;
     6776 +                ptgt->m_tgt_unconfigured = 0;
     6777 +                ptgt->m_dr_flag = MPTSAS_DR_INACTIVE;
5866 6778          }
5867      -out:
5868      -        kmem_free(phy_mask_name, MPTSAS_MAX_PHYS);
5869 6779  }
5870 6780  
5871 6781  static void
5872 6782  mptsas_handle_topo_change(mptsas_topo_change_list_t *topo_node,
5873 6783      dev_info_t *parent)
5874 6784  {
5875 6785          mptsas_target_t *ptgt = NULL;
5876 6786          mptsas_smp_t    *psmp = NULL;
5877 6787          mptsas_t        *mpt = (void *)topo_node->mpt;
5878 6788          uint16_t        devhdl;
5879 6789          uint16_t        attached_devhdl;
5880      -        uint64_t        sas_wwn = 0;
5881 6790          int             rval = 0;
5882 6791          uint32_t        page_address;
5883      -        uint8_t         phy, flags;
5884      -        char            *addr = NULL;
     6792 +        uint8_t         flags;
5885 6793          dev_info_t      *lundip;
5886 6794          int             circ = 0, circ1 = 0;
5887 6795          char            attached_wwnstr[MPTSAS_WWN_STRLEN];
5888 6796  
5889      -        NDBG20(("mptsas%d handle_topo_change enter", mpt->m_instance));
     6797 +        NDBG20(("mptsas3%d handle_topo_change enter, devhdl 0x%x,"
     6798 +            "event 0x%x, flags 0x%x", mpt->m_instance, topo_node->devhdl,
     6799 +            topo_node->event, topo_node->flags));
5890 6800  
5891 6801          ASSERT(mutex_owned(&mpt->m_mutex));
5892 6802  
5893 6803          switch (topo_node->event) {
5894 6804          case MPTSAS_DR_EVENT_RECONFIG_TARGET:
5895 6805          {
5896 6806                  char *phy_mask_name;
5897 6807                  mptsas_phymask_t phymask = 0;
5898 6808  
5899 6809                  if (topo_node->flags == MPTSAS_TOPO_FLAG_RAID_ASSOCIATED) {
↓ open down ↓ 139 lines elided ↑ open up ↑
6039 6949                                              parent, MPTSAS_VIRTUAL_PORT);
6040 6950                                          mptsas_log(mpt, CE_WARN,
6041 6951                                              "mptsas virtual-port"
6042 6952                                              "port prop update failed");
6043 6953                                          return;
6044 6954                                  }
6045 6955                          }
6046 6956                  }
6047 6957                  mutex_enter(&mpt->m_mutex);
6048 6958  
6049      -                NDBG20(("mptsas%d handle_topo_change to online devhdl:%x, "
     6959 +                NDBG20(("mptsas3%d handle_topo_change to online devhdl:%x, "
6050 6960                      "phymask:%x.", mpt->m_instance, ptgt->m_devhdl,
6051 6961                      ptgt->m_addr.mta_phymask));
6052 6962                  break;
6053 6963          }
6054 6964          case MPTSAS_DR_EVENT_OFFLINE_TARGET:
6055 6965          {
6056 6966                  devhdl = topo_node->devhdl;
6057 6967                  ptgt = refhash_linear_search(mpt->m_targets,
6058 6968                      mptsas_target_eval_devhdl, &devhdl);
6059 6969                  if (ptgt == NULL)
6060 6970                          break;
6061 6971  
6062      -                sas_wwn = ptgt->m_addr.mta_wwn;
6063      -                phy = ptgt->m_phynum;
6064      -
6065      -                addr = kmem_zalloc(SCSI_MAXNAMELEN, KM_SLEEP);
6066      -
6067      -                if (sas_wwn) {
6068      -                        (void) sprintf(addr, "w%016"PRIx64, sas_wwn);
6069      -                } else {
6070      -                        (void) sprintf(addr, "p%x", phy);
6071      -                }
6072 6972                  ASSERT(ptgt->m_devhdl == devhdl);
6073 6973  
6074 6974                  if ((topo_node->flags == MPTSAS_TOPO_FLAG_RAID_ASSOCIATED) ||
6075 6975                      (topo_node->flags ==
6076 6976                      MPTSAS_TOPO_FLAG_RAID_PHYSDRV_ASSOCIATED)) {
6077 6977                          /*
6078 6978                           * Get latest RAID info if RAID volume status changes
6079 6979                           * or Phys Disk status changes
6080 6980                           */
6081 6981                          (void) mptsas_get_raid_info(mpt);
6082 6982                  }
6083      -                /*
6084      -                 * Abort all outstanding command on the device
6085      -                 */
6086      -                rval = mptsas_do_scsi_reset(mpt, devhdl);
6087      -                if (rval) {
6088      -                        NDBG20(("mptsas%d handle_topo_change to reset target "
6089      -                            "before offline devhdl:%x, phymask:%x, rval:%x",
6090      -                            mpt->m_instance, ptgt->m_devhdl,
6091      -                            ptgt->m_addr.mta_phymask, rval));
6092      -                }
6093      -
6094      -                mutex_exit(&mpt->m_mutex);
6095      -
6096      -                ndi_devi_enter(scsi_vhci_dip, &circ);
6097      -                ndi_devi_enter(parent, &circ1);
6098      -                rval = mptsas_offline_target(parent, addr);
6099      -                ndi_devi_exit(parent, circ1);
6100      -                ndi_devi_exit(scsi_vhci_dip, circ);
6101      -                NDBG20(("mptsas%d handle_topo_change to offline devhdl:%x, "
6102      -                    "phymask:%x, rval:%x", mpt->m_instance,
6103      -                    ptgt->m_devhdl, ptgt->m_addr.mta_phymask, rval));
6104      -
6105      -                kmem_free(addr, SCSI_MAXNAMELEN);
6106      -
6107      -                /*
6108      -                 * Clear parent's props for SMHBA support
6109      -                 */
6110      -                flags = topo_node->flags;
6111      -                if (flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) {
6112      -                        bzero(attached_wwnstr, sizeof (attached_wwnstr));
6113      -                        if (ddi_prop_update_string(DDI_DEV_T_NONE, parent,
6114      -                            SCSI_ADDR_PROP_ATTACHED_PORT, attached_wwnstr) !=
6115      -                            DDI_PROP_SUCCESS) {
6116      -                                (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
6117      -                                    SCSI_ADDR_PROP_ATTACHED_PORT);
6118      -                                mptsas_log(mpt, CE_WARN, "mptsas attached port "
6119      -                                    "prop update failed");
6120      -                                break;
6121      -                        }
6122      -                        if (ddi_prop_update_int(DDI_DEV_T_NONE, parent,
6123      -                            MPTSAS_NUM_PHYS, 0) !=
6124      -                            DDI_PROP_SUCCESS) {
6125      -                                (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
6126      -                                    MPTSAS_NUM_PHYS);
6127      -                                mptsas_log(mpt, CE_WARN, "mptsas num phys "
6128      -                                    "prop update failed");
6129      -                                break;
6130      -                        }
6131      -                        if (ddi_prop_update_int(DDI_DEV_T_NONE, parent,
6132      -                            MPTSAS_VIRTUAL_PORT, 1) !=
6133      -                            DDI_PROP_SUCCESS) {
6134      -                                (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
6135      -                                    MPTSAS_VIRTUAL_PORT);
6136      -                                mptsas_log(mpt, CE_WARN, "mptsas virtual port "
6137      -                                    "prop update failed");
6138      -                                break;
6139      -                        }
6140      -                }
6141 6983  
6142      -                mutex_enter(&mpt->m_mutex);
6143      -                ptgt->m_led_status = 0;
6144      -                (void) mptsas_flush_led_status(mpt, ptgt);
6145      -                if (rval == DDI_SUCCESS) {
6146      -                        refhash_remove(mpt->m_targets, ptgt);
6147      -                        ptgt = NULL;
6148      -                } else {
6149      -                        /*
6150      -                         * clean DR_INTRANSITION flag to allow I/O down to
6151      -                         * PHCI driver since failover finished.
6152      -                         * Invalidate the devhdl
6153      -                         */
6154      -                        ptgt->m_devhdl = MPTSAS_INVALID_DEVHDL;
6155      -                        ptgt->m_tgt_unconfigured = 0;
6156      -                        mutex_enter(&mpt->m_tx_waitq_mutex);
6157      -                        ptgt->m_dr_flag = MPTSAS_DR_INACTIVE;
6158      -                        mutex_exit(&mpt->m_tx_waitq_mutex);
6159      -                }
     6984 +                mptsas_offline_target(mpt, ptgt, topo_node->flags, parent);
6160 6985  
6161 6986                  /*
6162 6987                   * Send SAS IO Unit Control to free the dev handle
6163 6988                   */
6164 6989                  if ((flags == MPTSAS_TOPO_FLAG_DIRECT_ATTACHED_DEVICE) ||
6165 6990                      (flags == MPTSAS_TOPO_FLAG_EXPANDER_ATTACHED_DEVICE)) {
6166 6991                          rval = mptsas_free_devhdl(mpt, devhdl);
6167 6992  
6168      -                        NDBG20(("mptsas%d handle_topo_change to remove "
     6993 +                        NDBG20(("mptsas3%d handle_topo_change to remove "
6169 6994                              "devhdl:%x, rval:%x", mpt->m_instance, devhdl,
6170 6995                              rval));
6171 6996                  }
6172 6997  
6173 6998                  break;
6174 6999          }
6175      -        case MPTSAS_TOPO_FLAG_REMOVE_HANDLE:
     7000 +        case MPTSAS_DR_EVENT_REMOVE_HANDLE:
6176 7001          {
6177 7002                  devhdl = topo_node->devhdl;
     7003 +
6178 7004                  /*
6179      -                 * If this is the remove handle event, do a reset first.
     7005 +                 * Do a reset first.
6180 7006                   */
6181      -                if (topo_node->event == MPTSAS_TOPO_FLAG_REMOVE_HANDLE) {
6182      -                        rval = mptsas_do_scsi_reset(mpt, devhdl);
6183      -                        if (rval) {
6184      -                                NDBG20(("mpt%d reset target before remove "
6185      -                                    "devhdl:%x, rval:%x", mpt->m_instance,
6186      -                                    devhdl, rval));
6187      -                        }
6188      -                }
     7007 +                rval = mptsas_do_scsi_reset(mpt, devhdl);
     7008 +                NDBG20(("mpt%d reset target before remove "
     7009 +                    "devhdl:%x, rval:%x", mpt->m_instance, devhdl, rval));
6189 7010  
6190 7011                  /*
6191 7012                   * Send SAS IO Unit Control to free the dev handle
6192 7013                   */
6193 7014                  rval = mptsas_free_devhdl(mpt, devhdl);
6194      -                NDBG20(("mptsas%d handle_topo_change to remove "
     7015 +                NDBG20(("mptsas3%d handle_topo_change to remove "
6195 7016                      "devhdl:%x, rval:%x", mpt->m_instance, devhdl,
6196 7017                      rval));
6197 7018                  break;
6198 7019          }
6199 7020          case MPTSAS_DR_EVENT_RECONFIG_SMP:
6200 7021          {
6201 7022                  mptsas_smp_t smp;
6202 7023                  dev_info_t *smpdip;
6203 7024  
6204 7025                  devhdl = topo_node->devhdl;
↓ open down ↓ 72 lines elided ↑ open up ↑
6277 7098                              DDI_PROP_SUCCESS) {
6278 7099                                  (void) ddi_prop_remove(DDI_DEV_T_NONE, parent,
6279 7100                                      SCSI_ADDR_PROP_ATTACHED_PORT);
6280 7101                                  mptsas_log(mpt, CE_WARN, "mptsas attached port "
6281 7102                                      "prop update failed");
6282 7103                                  return;
6283 7104                          }
6284 7105                  }
6285 7106  
6286 7107                  mutex_enter(&mpt->m_mutex);
6287      -                NDBG20(("mptsas%d handle_topo_change to remove devhdl:%x, "
     7108 +                NDBG20(("mptsas3%d handle_topo_change to remove devhdl:%x, "
6288 7109                      "rval:%x", mpt->m_instance, psmp->m_devhdl, rval));
6289 7110                  if (rval == DDI_SUCCESS) {
6290 7111                          refhash_remove(mpt->m_smp_targets, psmp);
6291 7112                  } else {
6292 7113                          psmp->m_devhdl = MPTSAS_INVALID_DEVHDL;
6293 7114                  }
6294 7115  
6295 7116                  bzero(attached_wwnstr, sizeof (attached_wwnstr));
6296 7117  
6297 7118                  break;
↓ open down ↓ 15 lines elided ↑ open up ↑
6313 7134          mptsas_t                        *mpt;
6314 7135          int                             i, j;
6315 7136          uint16_t                        event_data_len;
6316 7137          boolean_t                       sendAEN = FALSE;
6317 7138  
6318 7139          replyh_arg = (m_replyh_arg_t *)args;
6319 7140          rfm = replyh_arg->rfm;
6320 7141          mpt = replyh_arg->mpt;
6321 7142  
6322 7143          eventreply = (pMpi2EventNotificationReply_t)
6323      -            (mpt->m_reply_frame + (rfm - mpt->m_reply_frame_dma_addr));
     7144 +            (mpt->m_reply_frame + (rfm -
     7145 +            (mpt->m_reply_frame_dma_addr&0xfffffffful)));
6324 7146          event = ddi_get16(mpt->m_acc_reply_frame_hdl, &eventreply->Event);
6325 7147  
6326 7148  
6327 7149          /*
6328 7150           * Generate a system event to let anyone who cares know that a
6329 7151           * LOG_ENTRY_ADDED event has occurred.  This is sent no matter what the
6330 7152           * event mask is set to.
6331 7153           */
6332 7154          if (event == MPI2_EVENT_LOG_ENTRY_ADDED) {
6333 7155                  sendAEN = TRUE;
↓ open down ↓ 67 lines elided ↑ open up ↑
6401 7223          mptsas_t                        *mpt;
6402 7224          uint_t                          iocstatus;
6403 7225  
6404 7226          replyh_arg = (m_replyh_arg_t *)args;
6405 7227          rfm = replyh_arg->rfm;
6406 7228          mpt = replyh_arg->mpt;
6407 7229  
6408 7230          ASSERT(mutex_owned(&mpt->m_mutex));
6409 7231  
6410 7232          eventreply = (pMpi2EventNotificationReply_t)
6411      -            (mpt->m_reply_frame + (rfm - mpt->m_reply_frame_dma_addr));
     7233 +            (mpt->m_reply_frame + (rfm -
     7234 +            (mpt->m_reply_frame_dma_addr&0xfffffffful)));
6412 7235          event = ddi_get16(mpt->m_acc_reply_frame_hdl, &eventreply->Event);
6413 7236  
6414 7237          if (iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
6415 7238              &eventreply->IOCStatus)) {
6416 7239                  if (iocstatus == MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
6417 7240                          mptsas_log(mpt, CE_WARN,
6418      -                            "!mptsas_handle_event_sync: IOCStatus=0x%x, "
6419      -                            "IOCLogInfo=0x%x", iocstatus,
     7241 +                            "!mptsas_handle_event_sync: event 0x%x, "
     7242 +                            "IOCStatus=0x%x, "
     7243 +                            "IOCLogInfo=0x%x", event, iocstatus,
6420 7244                              ddi_get32(mpt->m_acc_reply_frame_hdl,
6421 7245                              &eventreply->IOCLogInfo));
6422 7246                  } else {
6423 7247                          mptsas_log(mpt, CE_WARN,
6424      -                            "mptsas_handle_event_sync: IOCStatus=0x%x, "
6425      -                            "IOCLogInfo=0x%x", iocstatus,
     7248 +                            "mptsas_handle_event_sync: event 0x%x, "
     7249 +                            "IOCStatus=0x%x, "
     7250 +                            "(IOCLogInfo=0x%x)", event, iocstatus,
6426 7251                              ddi_get32(mpt->m_acc_reply_frame_hdl,
6427 7252                              &eventreply->IOCLogInfo));
6428 7253                  }
6429 7254          }
6430 7255  
6431 7256          /*
6432 7257           * figure out what kind of event we got and handle accordingly
6433 7258           */
6434 7259          switch (event) {
6435 7260          case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
↓ open down ↓ 114 lines elided ↑ open up ↑
6550 7375                              (reason_code !=
6551 7376                              MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING)) {
6552 7377                                  continue;
6553 7378                          }
6554 7379                          curr[0] = 0;
6555 7380                          prev[0] = 0;
6556 7381                          string[0] = 0;
6557 7382                          switch (reason_code) {
6558 7383                          case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
6559 7384                          {
6560      -                                NDBG20(("mptsas%d phy %d physical_port %d "
     7385 +                                NDBG20(("mptsas3%d phy %d physical_port %d "
6561 7386                                      "dev_handle %d added", mpt->m_instance, phy,
6562 7387                                      physport, dev_handle));
6563 7388                                  link_rate = ddi_get8(mpt->m_acc_reply_frame_hdl,
6564 7389                                      &sas_topo_change_list->PHY[i].LinkRate);
6565 7390                                  state = (link_rate &
6566 7391                                      MPI2_EVENT_SAS_TOPO_LR_CURRENT_MASK) >>
6567 7392                                      MPI2_EVENT_SAS_TOPO_LR_CURRENT_SHIFT;
6568 7393                                  switch (state) {
6569 7394                                  case MPI2_EVENT_SAS_TOPO_LR_PHY_DISABLED:
6570 7395                                          (void) sprintf(curr, "is disabled");
↓ open down ↓ 15 lines elided ↑ open up ↑
6586 7411                                              "1.5 Gbps");
6587 7412                                          break;
6588 7413                                  case MPI2_EVENT_SAS_TOPO_LR_RATE_3_0:
6589 7414                                          (void) sprintf(curr, "is online at 3.0 "
6590 7415                                              "Gbps");
6591 7416                                          break;
6592 7417                                  case MPI2_EVENT_SAS_TOPO_LR_RATE_6_0:
6593 7418                                          (void) sprintf(curr, "is online at 6.0 "
6594 7419                                              "Gbps");
6595 7420                                          break;
     7421 +                                case MPI25_EVENT_SAS_TOPO_LR_RATE_12_0:
     7422 +                                        (void) sprintf(curr,
     7423 +                                            "is online at 12.0 Gbps");
     7424 +                                        break;
6596 7425                                  default:
6597 7426                                          (void) sprintf(curr, "state is "
6598 7427                                              "unknown");
6599 7428                                          break;
6600 7429                                  }
6601 7430                                  /*
6602 7431                                   * New target device added into the system.
6603 7432                                   * Set association flag according to if an
6604 7433                                   * expander is used or not.
6605 7434                                   */
↓ open down ↓ 26 lines elided ↑ open up ↑
6632 7461                                  if (topo_head == NULL) {
6633 7462                                          topo_head = topo_tail = topo_node;
6634 7463                                  } else {
6635 7464                                          topo_tail->next = topo_node;
6636 7465                                          topo_tail = topo_node;
6637 7466                                  }
6638 7467                                  break;
6639 7468                          }
6640 7469                          case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
6641 7470                          {
6642      -                                NDBG20(("mptsas%d phy %d physical_port %d "
     7471 +                                NDBG20(("mptsas3%d phy %d physical_port %d "
6643 7472                                      "dev_handle %d removed", mpt->m_instance,
6644 7473                                      phy, physport, dev_handle));
6645 7474                                  /*
6646 7475                                   * Set association flag according to if an
6647 7476                                   * expander is used or not.
6648 7477                                   */
6649 7478                                  exp_flag =
6650 7479                                      MPTSAS_TOPO_FLAG_EXPANDER_ATTACHED_DEVICE;
6651 7480                                  if (flags ==
6652 7481                                      MPTSAS_TOPO_FLAG_EXPANDER_ASSOCIATED) {
↓ open down ↓ 10 lines elided ↑ open up ↑
6663 7492                                   * If ptgt is NULL here, it means that the
6664 7493                                   * DevHandle is not in the hash table.  This is
6665 7494                                   * reasonable sometimes.  For example, if a
6666 7495                                   * disk was pulled, then added, then pulled
6667 7496                                   * again, the disk will not have been put into
6668 7497                                   * the hash table because the add event will
6669 7498                                   * have an invalid phymask.  BUT, this does not
6670 7499                                   * mean that the DevHandle is invalid.  The
6671 7500                                   * controller will still have a valid DevHandle
6672 7501                                   * that must be removed.  To do this, use the
6673      -                                 * MPTSAS_TOPO_FLAG_REMOVE_HANDLE event.
     7502 +                                 * MPTSAS_DR_EVENT_REMOVE_HANDLE event.
6674 7503                                   */
6675 7504                                  if (ptgt == NULL) {
6676 7505                                          topo_node = kmem_zalloc(
6677 7506                                              sizeof (mptsas_topo_change_list_t),
6678 7507                                              KM_SLEEP);
6679 7508                                          topo_node->mpt = mpt;
6680 7509                                          topo_node->un.phymask = 0;
6681 7510                                          topo_node->event =
6682      -                                            MPTSAS_TOPO_FLAG_REMOVE_HANDLE;
     7511 +                                            MPTSAS_DR_EVENT_REMOVE_HANDLE;
6683 7512                                          topo_node->devhdl = dev_handle;
6684 7513                                          topo_node->flags = flags;
6685 7514                                          topo_node->object = NULL;
6686 7515                                          if (topo_head == NULL) {
6687 7516                                                  topo_head = topo_tail =
6688 7517                                                      topo_node;
6689 7518                                          } else {
6690 7519                                                  topo_tail->next = topo_node;
6691 7520                                                  topo_tail = topo_node;
6692 7521                                          }
6693 7522                                          break;
6694 7523                                  }
6695 7524  
6696 7525                                  /*
6697 7526                                   * Update DR flag immediately avoid I/O failure
6698      -                                 * before failover finish. Pay attention to the
6699      -                                 * mutex protect, we need grab m_tx_waitq_mutex
6700      -                                 * during set m_dr_flag because we won't add
6701      -                                 * the following command into waitq, instead,
     7527 +                                 * before failover finish. We won't add
     7528 +                                 * any following commands into waitq, instead,
6702 7529                                   * we need return TRAN_BUSY in the tran_start
6703 7530                                   * context.
6704 7531                                   */
6705      -                                mutex_enter(&mpt->m_tx_waitq_mutex);
6706 7532                                  ptgt->m_dr_flag = MPTSAS_DR_INTRANSITION;
6707      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
6708 7533  
6709 7534                                  topo_node = kmem_zalloc(
6710 7535                                      sizeof (mptsas_topo_change_list_t),
6711 7536                                      KM_SLEEP);
6712 7537                                  topo_node->mpt = mpt;
6713 7538                                  topo_node->un.phymask =
6714 7539                                      ptgt->m_addr.mta_phymask;
6715 7540                                  topo_node->event =
6716 7541                                      MPTSAS_DR_EVENT_OFFLINE_TARGET;
6717 7542                                  topo_node->devhdl = dev_handle;
↓ open down ↓ 71 lines elided ↑ open up ↑
6789 7614                                              "6.0 Gbps");
6790 7615                                          if ((expd_handle == 0) &&
6791 7616                                              (enc_handle == 1)) {
6792 7617                                                  mpt->m_port_chng = 1;
6793 7618                                          }
6794 7619                                          mptsas_smhba_log_sysevent(mpt,
6795 7620                                              ESC_SAS_PHY_EVENT,
6796 7621                                              SAS_PHY_ONLINE,
6797 7622                                              &mpt->m_phy_info[i].smhba_info);
6798 7623                                          break;
     7624 +                                case MPI25_EVENT_SAS_TOPO_LR_RATE_12_0:
     7625 +                                        (void) sprintf(curr, "is online at "
     7626 +                                            "12.0 Gbps");
     7627 +                                        if ((expd_handle == 0) &&
     7628 +                                            (enc_handle == 1)) {
     7629 +                                                mpt->m_port_chng = 1;
     7630 +                                        }
     7631 +                                        mptsas_smhba_log_sysevent(mpt,
     7632 +                                            ESC_SAS_PHY_EVENT,
     7633 +                                            SAS_PHY_ONLINE,
     7634 +                                            &mpt->m_phy_info[i].smhba_info);
     7635 +                                        break;
6799 7636                                  default:
6800 7637                                          (void) sprintf(curr, "state is "
6801 7638                                              "unknown");
6802 7639                                          break;
6803 7640                                  }
6804 7641  
6805 7642                                  state = (link_rate &
6806 7643                                      MPI2_EVENT_SAS_TOPO_LR_PREV_MASK) >>
6807 7644                                      MPI2_EVENT_SAS_TOPO_LR_PREV_SHIFT;
6808 7645                                  switch (state) {
↓ open down ↓ 17 lines elided ↑ open up ↑
6826 7663                                              "1.5 Gbps");
6827 7664                                          break;
6828 7665                                  case MPI2_EVENT_SAS_TOPO_LR_RATE_3_0:
6829 7666                                          (void) sprintf(prev, ", was online at "
6830 7667                                              "3.0 Gbps");
6831 7668                                          break;
6832 7669                                  case MPI2_EVENT_SAS_TOPO_LR_RATE_6_0:
6833 7670                                          (void) sprintf(prev, ", was online at "
6834 7671                                              "6.0 Gbps");
6835 7672                                          break;
     7673 +                                case MPI25_EVENT_SAS_TOPO_LR_RATE_12_0:
     7674 +                                        (void) sprintf(prev, ", was online at "
     7675 +                                            "12.0 Gbps");
     7676 +                                        break;
6836 7677                                  default:
6837 7678                                  break;
6838 7679                                  }
6839 7680                                  (void) sprintf(&string[strlen(string)], "link "
6840 7681                                      "changed, ");
6841 7682                                  break;
6842 7683                          case MPI2_EVENT_SAS_TOPO_RC_NO_CHANGE:
6843 7684                                  continue;
6844 7685                          case MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING:
6845 7686                                  (void) sprintf(&string[strlen(string)],
6846 7687                                      "target not responding, delaying "
6847 7688                                      "removal");
6848 7689                                  break;
6849 7690                          }
6850      -                        NDBG20(("mptsas%d phy %d DevHandle %x, %s%s%s\n",
     7691 +                        NDBG20(("mptsas3%d phy %d DevHandle %x, %s%s%s\n",
6851 7692                              mpt->m_instance, phy, dev_handle, string, curr,
6852 7693                              prev));
6853 7694                  }
6854 7695                  if (topo_head != NULL) {
6855 7696                          /*
6856 7697                           * Launch DR taskq to handle topology change
6857 7698                           */
6858 7699                          if ((ddi_taskq_dispatch(mpt->m_dr_taskq,
6859 7700                              mptsas_handle_dr, (void *)topo_head,
6860 7701                              DDI_NOSLEEP)) != DDI_SUCCESS) {
     7702 +                                while (topo_head != NULL) {
     7703 +                                        topo_node = topo_head;
     7704 +                                        topo_head = topo_head->next;
     7705 +                                        kmem_free(topo_node,
     7706 +                                            sizeof (mptsas_topo_change_list_t));
     7707 +                                }
6861 7708                                  mptsas_log(mpt, CE_NOTE, "mptsas start taskq "
6862 7709                                      "for handle SAS DR event failed. \n");
6863 7710                          }
6864 7711                  }
6865 7712                  break;
6866 7713          }
6867 7714          case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
6868 7715          {
6869 7716                  Mpi2EventDataIrConfigChangeList_t       *irChangeList;
6870 7717                  mptsas_topo_change_list_t               *topo_head = NULL;
↓ open down ↓ 1 lines elided ↑ open up ↑
6872 7719                  mptsas_topo_change_list_t               *topo_node = NULL;
6873 7720                  mptsas_target_t                         *ptgt;
6874 7721                  uint8_t                                 num_entries, i, reason;
6875 7722                  uint16_t                                volhandle, diskhandle;
6876 7723  
6877 7724                  irChangeList = (pMpi2EventDataIrConfigChangeList_t)
6878 7725                      eventreply->EventData;
6879 7726                  num_entries = ddi_get8(mpt->m_acc_reply_frame_hdl,
6880 7727                      &irChangeList->NumElements);
6881 7728  
6882      -                NDBG20(("mptsas%d IR_CONFIGURATION_CHANGE_LIST event received",
     7729 +                NDBG20(("mptsas3%d IR_CONFIGURATION_CHANGE_LIST event received",
6883 7730                      mpt->m_instance));
6884 7731  
6885 7732                  for (i = 0; i < num_entries; i++) {
6886 7733                          reason = ddi_get8(mpt->m_acc_reply_frame_hdl,
6887 7734                              &irChangeList->ConfigElement[i].ReasonCode);
6888 7735                          volhandle = ddi_get16(mpt->m_acc_reply_frame_hdl,
6889 7736                              &irChangeList->ConfigElement[i].VolDevHandle);
6890 7737                          diskhandle = ddi_get16(mpt->m_acc_reply_frame_hdl,
6891 7738                              &irChangeList->ConfigElement[i].PhysDiskDevHandle);
6892 7739  
↓ open down ↓ 35 lines elided ↑ open up ↑
6928 7775                                          break;
6929 7776  
6930 7777                                  /*
6931 7778                                   * Clear any flags related to volume
6932 7779                                   */
6933 7780                                  (void) mptsas_delete_volume(mpt, volhandle);
6934 7781  
6935 7782                                  /*
6936 7783                                   * Update DR flag immediately avoid I/O failure
6937 7784                                   */
6938      -                                mutex_enter(&mpt->m_tx_waitq_mutex);
6939 7785                                  ptgt->m_dr_flag = MPTSAS_DR_INTRANSITION;
6940      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
6941 7786  
6942 7787                                  topo_node = kmem_zalloc(
6943 7788                                      sizeof (mptsas_topo_change_list_t),
6944 7789                                      KM_SLEEP);
6945 7790                                  topo_node->mpt = mpt;
6946 7791                                  topo_node->un.phymask =
6947 7792                                      ptgt->m_addr.mta_phymask;
6948 7793                                  topo_node->event =
6949 7794                                      MPTSAS_DR_EVENT_OFFLINE_TARGET;
6950 7795                                  topo_node->devhdl = volhandle;
↓ open down ↓ 12 lines elided ↑ open up ↑
6963 7808                          case MPI2_EVENT_IR_CHANGE_RC_HIDE:
6964 7809                          {
6965 7810                                  ptgt = refhash_linear_search(mpt->m_targets,
6966 7811                                      mptsas_target_eval_devhdl, &diskhandle);
6967 7812                                  if (ptgt == NULL)
6968 7813                                          break;
6969 7814  
6970 7815                                  /*
6971 7816                                   * Update DR flag immediately avoid I/O failure
6972 7817                                   */
6973      -                                mutex_enter(&mpt->m_tx_waitq_mutex);
6974 7818                                  ptgt->m_dr_flag = MPTSAS_DR_INTRANSITION;
6975      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
6976 7819  
6977 7820                                  topo_node = kmem_zalloc(
6978 7821                                      sizeof (mptsas_topo_change_list_t),
6979 7822                                      KM_SLEEP);
6980 7823                                  topo_node->mpt = mpt;
6981 7824                                  topo_node->un.phymask =
6982 7825                                      ptgt->m_addr.mta_phymask;
6983 7826                                  topo_node->event =
6984 7827                                      MPTSAS_DR_EVENT_OFFLINE_TARGET;
6985 7828                                  topo_node->devhdl = diskhandle;
↓ open down ↓ 43 lines elided ↑ open up ↑
7029 7872                          }
7030 7873                  }
7031 7874  
7032 7875                  if (topo_head != NULL) {
7033 7876                          /*
7034 7877                           * Launch DR taskq to handle topology change
7035 7878                           */
7036 7879                          if ((ddi_taskq_dispatch(mpt->m_dr_taskq,
7037 7880                              mptsas_handle_dr, (void *)topo_head,
7038 7881                              DDI_NOSLEEP)) != DDI_SUCCESS) {
     7882 +                                while (topo_head != NULL) {
     7883 +                                        topo_node = topo_head;
     7884 +                                        topo_head = topo_head->next;
     7885 +                                        kmem_free(topo_node,
     7886 +                                            sizeof (mptsas_topo_change_list_t));
     7887 +                                }
7039 7888                                  mptsas_log(mpt, CE_NOTE, "mptsas start taskq "
7040 7889                                      "for handle SAS DR event failed. \n");
7041 7890                          }
7042 7891                  }
7043 7892                  break;
7044 7893          }
7045 7894          default:
7046 7895                  return (DDI_FAILURE);
7047 7896          }
7048 7897  
↓ open down ↓ 22 lines elided ↑ open up ↑
7071 7920          /*
7072 7921           * If HBA is being reset, drop incoming event.
7073 7922           */
7074 7923          if (mpt->m_in_reset) {
7075 7924                  NDBG20(("dropping event received prior to reset"));
7076 7925                  mutex_exit(&mpt->m_mutex);
7077 7926                  return;
7078 7927          }
7079 7928  
7080 7929          eventreply = (pMpi2EventNotificationReply_t)
7081      -            (mpt->m_reply_frame + (rfm - mpt->m_reply_frame_dma_addr));
     7930 +            (mpt->m_reply_frame + (rfm -
     7931 +            (mpt->m_reply_frame_dma_addr&0xfffffffful)));
7082 7932          event = ddi_get16(mpt->m_acc_reply_frame_hdl, &eventreply->Event);
7083 7933  
7084 7934          if (iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
7085 7935              &eventreply->IOCStatus)) {
7086 7936                  if (iocstatus == MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
7087 7937                          mptsas_log(mpt, CE_WARN,
7088 7938                              "!mptsas_handle_event: IOCStatus=0x%x, "
7089 7939                              "IOCLogInfo=0x%x", iocstatus,
7090 7940                              ddi_get32(mpt->m_acc_reply_frame_hdl,
7091 7941                              &eventreply->IOCLogInfo));
↓ open down ↓ 12 lines elided ↑ open up ↑
7104 7954          switch (event) {
7105 7955          case MPI2_EVENT_LOG_ENTRY_ADDED:
7106 7956                  break;
7107 7957          case MPI2_EVENT_LOG_DATA:
7108 7958                  iocloginfo = ddi_get32(mpt->m_acc_reply_frame_hdl,
7109 7959                      &eventreply->IOCLogInfo);
7110 7960                  NDBG20(("mptsas %d log info %x received.\n", mpt->m_instance,
7111 7961                      iocloginfo));
7112 7962                  break;
7113 7963          case MPI2_EVENT_STATE_CHANGE:
7114      -                NDBG20(("mptsas%d state change.", mpt->m_instance));
     7964 +                NDBG20(("mptsas3%d state change.", mpt->m_instance));
7115 7965                  break;
7116 7966          case MPI2_EVENT_HARD_RESET_RECEIVED:
7117      -                NDBG20(("mptsas%d event change.", mpt->m_instance));
     7967 +                NDBG20(("mptsas3%d event change.", mpt->m_instance));
7118 7968                  break;
7119 7969          case MPI2_EVENT_SAS_DISCOVERY:
7120 7970          {
7121 7971                  MPI2_EVENT_DATA_SAS_DISCOVERY   *sasdiscovery;
7122 7972                  char                            string[80];
7123 7973                  uint8_t                         rc;
7124 7974  
7125 7975                  sasdiscovery =
7126 7976                      (pMpi2EventDataSasDiscovery_t)eventreply->EventData;
7127 7977  
↓ open down ↓ 16 lines elided ↑ open up ↑
7144 7994                          (void) sprintf(string, "UNKNOWN");
7145 7995                          break;
7146 7996                  }
7147 7997  
7148 7998                  NDBG20(("SAS DISCOVERY is %s for port %d, status %x", string,
7149 7999                      port, status));
7150 8000  
7151 8001                  break;
7152 8002          }
7153 8003          case MPI2_EVENT_EVENT_CHANGE:
7154      -                NDBG20(("mptsas%d event change.", mpt->m_instance));
     8004 +                NDBG20(("mptsas3%d event change.", mpt->m_instance));
7155 8005                  break;
7156 8006          case MPI2_EVENT_TASK_SET_FULL:
7157 8007          {
7158 8008                  pMpi2EventDataTaskSetFull_t     taskfull;
7159 8009  
7160 8010                  taskfull = (pMpi2EventDataTaskSetFull_t)eventreply->EventData;
7161 8011  
7162      -                NDBG20(("TASK_SET_FULL received for mptsas%d, depth %d\n",
     8012 +                NDBG20(("TASK_SET_FULL received for mptsas3%d, depth %d\n",
7163 8013                      mpt->m_instance,  ddi_get16(mpt->m_acc_reply_frame_hdl,
7164 8014                      &taskfull->CurrentDepth)));
7165 8015                  break;
7166 8016          }
7167 8017          case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
7168 8018          {
7169 8019                  /*
7170 8020                   * SAS TOPOLOGY CHANGE LIST Event has already been handled
7171 8021                   * in mptsas_handle_event_sync() of interrupt context
7172 8022                   */
↓ open down ↓ 13 lines elided ↑ open up ↑
7186 8036                  switch (rc) {
7187 8037                  case MPI2_EVENT_SAS_ENCL_RC_ADDED:
7188 8038                          (void) sprintf(string, "added");
7189 8039                          break;
7190 8040                  case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
7191 8041                          (void) sprintf(string, ", not responding");
7192 8042                          break;
7193 8043                  default:
7194 8044                  break;
7195 8045                  }
7196      -                NDBG20(("mptsas%d ENCLOSURE STATUS CHANGE for enclosure %x%s\n",
7197      -                    mpt->m_instance, ddi_get16(mpt->m_acc_reply_frame_hdl,
     8046 +                NDBG20(("mptsas3%d ENCLOSURE STATUS CHANGE for enclosure "
     8047 +                    "%x%s\n", mpt->m_instance,
     8048 +                    ddi_get16(mpt->m_acc_reply_frame_hdl,
7198 8049                      &encstatus->EnclosureHandle), string));
7199 8050                  break;
7200 8051          }
7201 8052  
7202 8053          /*
7203 8054           * MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE is handled by
7204 8055           * mptsas_handle_event_sync,in here just send ack message.
7205 8056           */
7206 8057          case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
7207 8058          {
↓ open down ↓ 104 lines elided ↑ open up ↑
7312 8163                                      "expansion");
7313 8164                                  break;
7314 8165                          case MPI2_EVENT_IR_RAIDOP_CONSISTENCY_CHECK:
7315 8166                                  (void) sprintf(reason_str, "consistency check");
7316 8167                                  break;
7317 8168                          default:
7318 8169                                  (void) sprintf(reason_str, "unknown reason %x",
7319 8170                                      rc);
7320 8171                  }
7321 8172  
7322      -                NDBG20(("mptsas%d raid operational status: (%s)"
     8173 +                NDBG20(("mptsas3%d raid operational status: (%s)"
7323 8174                      "\thandle(0x%04x), percent complete(%d)\n",
7324 8175                      mpt->m_instance, reason_str, handle, percent));
7325 8176                  break;
7326 8177          }
7327 8178          case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
7328 8179          {
7329 8180                  pMpi2EventDataSasBroadcastPrimitive_t   sas_broadcast;
7330 8181                  uint8_t                                 phy_num;
7331 8182                  uint8_t                                 primitive;
7332 8183  
↓ open down ↓ 48 lines elided ↑ open up ↑
7381 8232                              SAS_PORT_BROADCAST_D24_0,
7382 8233                              &mpt->m_phy_info[phy_num].smhba_info);
7383 8234                          break;
7384 8235                  case MPI2_EVENT_PRIMITIVE_CHANGE1_RESERVED:
7385 8236                          mptsas_smhba_log_sysevent(mpt,
7386 8237                              ESC_SAS_HBA_PORT_BROADCAST,
7387 8238                              SAS_PORT_BROADCAST_D27_4,
7388 8239                              &mpt->m_phy_info[phy_num].smhba_info);
7389 8240                          break;
7390 8241                  default:
7391      -                        NDBG20(("mptsas%d: unknown BROADCAST PRIMITIVE"
     8242 +                        NDBG16(("mptsas3%d: unknown BROADCAST PRIMITIVE"
7392 8243                              " %x received",
7393 8244                              mpt->m_instance, primitive));
7394 8245                          break;
7395 8246                  }
7396      -                NDBG20(("mptsas%d sas broadcast primitive: "
     8247 +                NDBG16(("mptsas3%d sas broadcast primitive: "
7397 8248                      "\tprimitive(0x%04x), phy(%d) complete\n",
7398 8249                      mpt->m_instance, primitive, phy_num));
7399 8250                  break;
7400 8251          }
7401 8252          case MPI2_EVENT_IR_VOLUME:
7402 8253          {
7403 8254                  Mpi2EventDataIrVolume_t         *irVolume;
7404 8255                  uint16_t                        devhandle;
7405 8256                  uint32_t                        state;
7406 8257                  int                             config, vol;
↓ open down ↓ 185 lines elided ↑ open up ↑
7592 8443                              state == MPI2_RAID_PD_STATE_NOT_COMPATIBLE
7593 8444                              ? "not compatible" :
7594 8445                              state == MPI2_RAID_PD_STATE_NOT_CONFIGURED
7595 8446                              ? "not configured" :
7596 8447                              "state unknown");
7597 8448                          break;
7598 8449                  }
7599 8450                  break;
7600 8451          }
7601 8452          default:
7602      -                NDBG20(("mptsas%d: unknown event %x received",
     8453 +                NDBG20(("mptsas3%d: unknown event %x received",
7603 8454                      mpt->m_instance, event));
7604 8455                  break;
7605 8456          }
7606 8457  
7607 8458          /*
7608 8459           * Return the reply frame to the free queue.
7609 8460           */
7610 8461          ddi_put32(mpt->m_acc_free_queue_hdl,
7611 8462              &((uint32_t *)(void *)mpt->m_free_queue)[mpt->m_free_index], rfm);
7612 8463          (void) ddi_dma_sync(mpt->m_dma_free_queue_hdl, 0, 0,
↓ open down ↓ 14 lines elided ↑ open up ↑
7627 8478  {
7628 8479          mptsas_t        *mpt = arg;
7629 8480          mptsas_target_t *ptgt = NULL;
7630 8481  
7631 8482          mutex_enter(&mpt->m_mutex);
7632 8483  
7633 8484          mpt->m_restart_cmd_timeid = 0;
7634 8485  
7635 8486          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
7636 8487              ptgt = refhash_next(mpt->m_targets, ptgt)) {
     8488 +                mutex_enter(&ptgt->m_t_mutex);
7637 8489                  if (ptgt->m_reset_delay == 0) {
7638 8490                          if (ptgt->m_t_throttle == QFULL_THROTTLE) {
7639 8491                                  mptsas_set_throttle(mpt, ptgt,
7640 8492                                      MAX_THROTTLE);
7641 8493                          }
7642 8494                  }
     8495 +                mutex_exit(&ptgt->m_t_mutex);
7643 8496          }
7644 8497          mptsas_restart_hba(mpt);
7645 8498          mutex_exit(&mpt->m_mutex);
7646 8499  }
7647 8500  
     8501 +/*
     8502 + * Assume some checks have been done prior to calling this
     8503 + * function so we don't need to consider taking the m_mutex.
     8504 + */
     8505 +static void
     8506 +mptsas_remove_cmd_nomtx(mptsas_t *mpt, mptsas_cmd_t *cmd)
     8507 +{
     8508 +        int             slot;
     8509 +        mptsas_slots_t  *slots = mpt->m_active;
     8510 +        mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
     8511 +
     8512 +        ASSERT(cmd != NULL);
     8513 +        ASSERT(cmd->cmd_queued == FALSE);
     8514 +        ASSERT((cmd->cmd_flags & CFLAG_CMDIOC) == 0);
     8515 +
     8516 +        slot = cmd->cmd_slot;
     8517 +
     8518 +        /*
     8519 +         * remove the cmd.
     8520 +         */
     8521 +        if (cmd == slots->m_slot[slot]) {
     8522 +                NDBG31(("mptsas_remove_cmd_nomtx: removing cmd=0x%p, flags "
     8523 +                    "0x%x", (void *)cmd, cmd->cmd_flags));
     8524 +                slots->m_slot[slot] = NULL;
     8525 +                ASSERT(mpt->m_ncmds != 0);
     8526 +                atomic_dec_32(&mpt->m_ncmds);
     8527 +                ASSERT(mpt->m_rep_post_queues[cmd->cmd_rpqidx].rpq_ncmds != 0);
     8528 +                atomic_dec_32(
     8529 +                    &mpt->m_rep_post_queues[cmd->cmd_rpqidx].rpq_ncmds);
     8530 +
     8531 +                /*
     8532 +                 * Decrement per target ncmds, we know this is not an
     8533 +                 * IOC cmd and it therefore has a target associated with it.
     8534 +                 */
     8535 +                mutex_enter(&ptgt->m_t_mutex);
     8536 +                ASSERT(ptgt->m_t_ncmds != 0);
     8537 +                ptgt->m_t_ncmds--;
     8538 +
     8539 +                /*
     8540 +                 * reset throttle if we just ran an untagged command
     8541 +                 * to a tagged target
     8542 +                 */
     8543 +                if ((ptgt->m_t_ncmds == 0) &&
     8544 +                    ((cmd->cmd_pkt_flags & FLAG_TAGMASK) == 0)) {
     8545 +                        mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     8546 +                }
     8547 +
     8548 +                /*
     8549 +                 * Remove this command from the active queue.
     8550 +                 */
     8551 +                if (cmd->cmd_active_expiration != 0) {
     8552 +                        TAILQ_REMOVE(&ptgt->m_active_cmdq, cmd,
     8553 +                            cmd_active_link);
     8554 +                        cmd->cmd_active_expiration = 0;
     8555 +                }
     8556 +                mutex_exit(&ptgt->m_t_mutex);
     8557 +        }
     8558 +
     8559 +        ASSERT(cmd != slots->m_slot[cmd->cmd_slot]);
     8560 +}
     8561 +
7648 8562  void
7649 8563  mptsas_remove_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
7650 8564  {
7651 8565          int             slot;
7652 8566          mptsas_slots_t  *slots = mpt->m_active;
7653      -        int             t;
7654 8567          mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
7655 8568  
7656 8569          ASSERT(cmd != NULL);
7657 8570          ASSERT(cmd->cmd_queued == FALSE);
7658 8571  
7659 8572          /*
7660 8573           * Task Management cmds are removed in their own routines.  Also,
7661 8574           * we don't want to modify timeout based on TM cmds.
7662 8575           */
7663 8576          if (cmd->cmd_flags & CFLAG_TM_CMD) {
7664 8577                  return;
7665 8578          }
7666 8579  
7667      -        t = Tgt(cmd);
7668 8580          slot = cmd->cmd_slot;
7669 8581  
7670 8582          /*
7671 8583           * remove the cmd.
7672 8584           */
7673 8585          if (cmd == slots->m_slot[slot]) {
7674      -                NDBG31(("mptsas_remove_cmd: removing cmd=0x%p", (void *)cmd));
     8586 +                NDBG31(("mptsas_remove_cmd: removing cmd=0x%p, flags 0x%x",
     8587 +                    (void *)cmd, cmd->cmd_flags));
7675 8588                  slots->m_slot[slot] = NULL;
7676      -                mpt->m_ncmds--;
     8589 +                ASSERT(mpt->m_ncmds != 0);
     8590 +                atomic_dec_32(&mpt->m_ncmds);
     8591 +                ASSERT(mpt->m_rep_post_queues[cmd->cmd_rpqidx].rpq_ncmds != 0);
     8592 +                atomic_dec_32(
     8593 +                    &mpt->m_rep_post_queues[cmd->cmd_rpqidx].rpq_ncmds);
7677 8594  
7678 8595                  /*
7679 8596                   * only decrement per target ncmds if command
7680 8597                   * has a target associated with it.
7681 8598                   */
7682 8599                  if ((cmd->cmd_flags & CFLAG_CMDIOC) == 0) {
     8600 +                        mutex_enter(&ptgt->m_t_mutex);
     8601 +                        ASSERT(ptgt->m_t_ncmds != 0);
7683 8602                          ptgt->m_t_ncmds--;
     8603 +
7684 8604                          /*
7685 8605                           * reset throttle if we just ran an untagged command
7686 8606                           * to a tagged target
7687 8607                           */
7688 8608                          if ((ptgt->m_t_ncmds == 0) &&
7689 8609                              ((cmd->cmd_pkt_flags & FLAG_TAGMASK) == 0)) {
7690 8610                                  mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
7691 8611                          }
     8612 +
     8613 +                        /*
     8614 +                         * Remove this command from the active queue.
     8615 +                         */
     8616 +                        if (cmd->cmd_active_expiration != 0) {
     8617 +                                TAILQ_REMOVE(&ptgt->m_active_cmdq, cmd,
     8618 +                                    cmd_active_link);
     8619 +                                cmd->cmd_active_expiration = 0;
     8620 +                        }
     8621 +                        mutex_exit(&ptgt->m_t_mutex);
7692 8622                  }
7693 8623  
7694 8624          }
7695 8625  
7696 8626          /*
7697 8627           * This is all we need to do for ioc commands.
7698 8628           */
7699 8629          if (cmd->cmd_flags & CFLAG_CMDIOC) {
7700 8630                  mptsas_return_to_pool(mpt, cmd);
7701 8631                  return;
7702 8632          }
7703 8633  
7704      -        /*
7705      -         * Figure out what to set tag Q timeout for...
7706      -         *
7707      -         * Optimize: If we have duplicate's of same timeout
7708      -         * we're using, then we'll use it again until we run
7709      -         * out of duplicates.  This should be the normal case
7710      -         * for block and raw I/O.
7711      -         * If no duplicates, we have to scan through tag que and
7712      -         * find the longest timeout value and use it.  This is
7713      -         * going to take a while...
7714      -         * Add 1 to m_n_normal to account for TM request.
7715      -         */
7716      -        if (cmd->cmd_pkt->pkt_time == ptgt->m_timebase) {
7717      -                if (--(ptgt->m_dups) == 0) {
7718      -                        if (ptgt->m_t_ncmds) {
7719      -                                mptsas_cmd_t *ssp;
7720      -                                uint_t n = 0;
7721      -                                ushort_t nslots = (slots->m_n_normal + 1);
7722      -                                ushort_t i;
7723      -                                /*
7724      -                                 * This crude check assumes we don't do
7725      -                                 * this too often which seems reasonable
7726      -                                 * for block and raw I/O.
7727      -                                 */
7728      -                                for (i = 0; i < nslots; i++) {
7729      -                                        ssp = slots->m_slot[i];
7730      -                                        if (ssp && (Tgt(ssp) == t) &&
7731      -                                            (ssp->cmd_pkt->pkt_time > n)) {
7732      -                                                n = ssp->cmd_pkt->pkt_time;
7733      -                                                ptgt->m_dups = 1;
7734      -                                        } else if (ssp && (Tgt(ssp) == t) &&
7735      -                                            (ssp->cmd_pkt->pkt_time == n)) {
7736      -                                                ptgt->m_dups++;
7737      -                                        }
7738      -                                }
7739      -                                ptgt->m_timebase = n;
7740      -                        } else {
7741      -                                ptgt->m_dups = 0;
7742      -                                ptgt->m_timebase = 0;
7743      -                        }
7744      -                }
7745      -        }
7746      -        ptgt->m_timeout = ptgt->m_timebase;
7747      -
7748 8634          ASSERT(cmd != slots->m_slot[cmd->cmd_slot]);
7749 8635  }
7750 8636  
7751 8637  /*
7752 8638   * accept all cmds on the tx_waitq if any and then
7753 8639   * start a fresh request from the top of the device queue.
7754 8640   *
7755 8641   * since there are always cmds queued on the tx_waitq, and rare cmds on
7756 8642   * the instance waitq, so this function should not be invoked in the ISR,
7757 8643   * the mptsas_restart_waitq() is invoked in the ISR instead. otherwise, the
7758 8644   * burden belongs to the IO dispatch CPUs is moved the interrupt CPU.
7759 8645   */
7760 8646  static void
7761 8647  mptsas_restart_hba(mptsas_t *mpt)
7762 8648  {
7763 8649          ASSERT(mutex_owned(&mpt->m_mutex));
7764 8650  
7765      -        mutex_enter(&mpt->m_tx_waitq_mutex);
7766      -        if (mpt->m_tx_waitq) {
7767      -                mptsas_accept_tx_waitq(mpt);
7768      -        }
7769      -        mutex_exit(&mpt->m_tx_waitq_mutex);
     8651 +        mptsas_accept_tx_waitqs(mpt);
7770 8652          mptsas_restart_waitq(mpt);
7771 8653  }
7772 8654  
7773 8655  /*
7774 8656   * start a fresh request from the top of the device queue
7775 8657   */
7776 8658  static void
7777 8659  mptsas_restart_waitq(mptsas_t *mpt)
7778 8660  {
7779 8661          mptsas_cmd_t    *cmd, *next_cmd;
↓ open down ↓ 47 lines elided ↑ open up ↑
7827 8709                                   */
7828 8710                                  cmd->cmd_flags |= CFLAG_PREPARED;
7829 8711                                  mptsas_waitq_delete(mpt, cmd);
7830 8712                                  mptsas_start_diag(mpt, cmd);
7831 8713                          }
7832 8714                          cmd = next_cmd;
7833 8715                          continue;
7834 8716                  }
7835 8717  
7836 8718                  ptgt = cmd->cmd_tgt_addr;
7837      -                if (ptgt && (ptgt->m_t_throttle == DRAIN_THROTTLE) &&
7838      -                    (ptgt->m_t_ncmds == 0)) {
7839      -                        mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     8719 +                if (ptgt) {
     8720 +                        mutex_enter(&ptgt->m_t_mutex);
     8721 +                        if ((ptgt->m_t_throttle == DRAIN_THROTTLE) &&
     8722 +                            (ptgt->m_t_ncmds == 0)) {
     8723 +                                mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     8724 +                        }
     8725 +                        if ((mpt->m_ncmds <= (mpt->m_max_requests - 2)) &&
     8726 +                            (ptgt->m_reset_delay == 0) &&
     8727 +                            (ptgt->m_t_ncmds < ptgt->m_t_throttle)) {
     8728 +                                mutex_exit(&ptgt->m_t_mutex);
     8729 +
     8730 +                                if (mptsas_save_cmd(mpt, cmd) == TRUE) {
     8731 +                                        mptsas_waitq_delete(mpt, cmd);
     8732 +                                        mutex_exit(&mpt->m_mutex);
     8733 +                                        (void) mptsas_start_cmd(mpt, cmd);
     8734 +                                        mutex_enter(&mpt->m_mutex);
     8735 +                                        cmd = mpt->m_waitq;
     8736 +                                        continue;
     8737 +                                }
     8738 +                        } else {
     8739 +                                mutex_exit(&ptgt->m_t_mutex);
     8740 +                        }
7840 8741                  }
7841      -                if ((mpt->m_ncmds <= (mpt->m_max_requests - 2)) &&
7842      -                    (ptgt && (ptgt->m_reset_delay == 0)) &&
7843      -                    (ptgt && (ptgt->m_t_ncmds <
7844      -                    ptgt->m_t_throttle))) {
7845      -                        if (mptsas_save_cmd(mpt, cmd) == TRUE) {
7846      -                                mptsas_waitq_delete(mpt, cmd);
     8742 +                cmd = next_cmd;
     8743 +        }
     8744 +}
     8745 +
     8746 +/*
     8747 + * Cmds are queued if scsi_start() doesn't get the m_mutex lock(no wait)
     8748 + * or if the decision has been made to always do that. Setting
     8749 + * mptsas_allow_txq_jumping to zero will allow higher performance on
     8750 + * a heavily loaded system as there is less disruption to the flow here.
     8751 + * There are 2 threads that handle one queue each. The idea is that
     8752 + * they take it in turn to grab the m_mutex to run the mptsas_accept_pkt()
     8753 + * function and then drop it while the cmd is started in mptsas_start_cmd().
     8754 + */
     8755 +static void
     8756 +mptsas_tx_waitq_thread(mptsas_thread_arg_t *arg)
     8757 +{
     8758 +        mptsas_t *mpt = arg->mpt;
     8759 +        mptsas_tx_waitqueue_t *txwq = &mpt->m_tx_waitq[arg->t];
     8760 +
     8761 +        mutex_enter(&txwq->txwq_mutex);
     8762 +        while (txwq->txwq_active) {
     8763 +                mptsas_drain_tx_waitq(mpt, txwq);
     8764 +                if (txwq->txwq_wdrain) {
     8765 +                        cv_signal(&txwq->txwq_drain_cv);
     8766 +                }
     8767 +                cv_wait(&txwq->txwq_cv, &txwq->txwq_mutex);
     8768 +        }
     8769 +        mutex_exit(&txwq->txwq_mutex);
     8770 +        mutex_enter(&mpt->m_qthread_mutex);
     8771 +        mpt->m_txwq_thread_n--;
     8772 +        cv_broadcast(&mpt->m_qthread_cv);
     8773 +        mutex_exit(&mpt->m_qthread_mutex);
     8774 +}
     8775 +
     8776 +/*
     8777 + * Set the draining flag, disconnect the list and process one at a time
     8778 + * so that the cmds are sent in order.
     8779 + */
     8780 +static void
     8781 +mptsas_drain_tx_waitq(mptsas_t *mpt, mptsas_tx_waitqueue_t *txwq)
     8782 +{
     8783 +        mptsas_cmd_t    *cmd, *ncmd;
     8784 +        int             rval, start;
     8785 +#ifdef MPTSAS_DEBUG
     8786 +        uint32_t        qlen;
     8787 +#endif
     8788 +
     8789 +        txwq->txwq_draining = TRUE;
     8790 +#ifndef __lock_lint
     8791 +        _NOTE(CONSTCOND)
     8792 +#endif
     8793 +        while (TRUE) {
     8794 +
     8795 +                /*
     8796 +                 * A Bus Reset could occur at any time but it will have to
     8797 +                 * wait for the main mutex before flushing the tx_waitq.
     8798 +                 * Pull all commands at once, then follow the list in order to
     8799 +                 * reduce txwq_mutex hold time. If there is a Bus Reset at
     8800 +                 * some point the commands will get to the waitq and then be
     8801 +                 * flushed.
     8802 +                 */
     8803 +                cmd = txwq->txwq_cmdq;
     8804 +
     8805 +                if (cmd == NULL) {
     8806 +                        txwq->txwq_draining = FALSE;
     8807 +                        return;
     8808 +                }
     8809 +                txwq->txwq_cmdq = NULL;
     8810 +                txwq->txwq_qtail = &txwq->txwq_cmdq;
     8811 +#ifdef MPTSAS_DEBUG
     8812 +                qlen = txwq->txwq_len;
     8813 +#endif
     8814 +                txwq->txwq_len = 0;
     8815 +                mutex_exit(&txwq->txwq_mutex);
     8816 +
     8817 +                while (cmd) {
     8818 +                        ncmd = cmd->cmd_linkp;
     8819 +                        cmd->cmd_linkp = NULL;
     8820 +                        mutex_enter(&mpt->m_mutex);
     8821 +                        start = mptsas_accept_pkt(mpt, cmd, &rval);
     8822 +                        mutex_exit(&mpt->m_mutex);
     8823 +                        if (start) {
7847 8824                                  (void) mptsas_start_cmd(mpt, cmd);
7848 8825                          }
     8826 +                        if (rval != TRAN_ACCEPT)
     8827 +                                cmn_err(CE_WARN,
     8828 +                                    "mpt: mptsas_drain_tx_waitq: failed "
     8829 +                                    "(rval=0x%x) to accept cmd 0x%p on queue\n",
     8830 +                                    rval, (void *)cmd);
     8831 +                        cmd = ncmd;
     8832 +#ifdef MPTSAS_DEBUG
     8833 +                        qlen--;
     8834 +#endif
7849 8835                  }
7850      -                cmd = next_cmd;
     8836 +                ASSERT(qlen == 0);
     8837 +                mutex_enter(&txwq->txwq_mutex);
7851 8838          }
7852 8839  }
     8840 +
7853 8841  /*
7854      - * Cmds are queued if tran_start() doesn't get the m_mutexlock(no wait).
7855      - * Accept all those queued cmds before new cmd is accept so that the
7856      - * cmds are sent in order.
     8842 + * Stop the drain threads from picking up a new list.
     8843 + * Optionally wait for the current list being processed to drain through.
     8844 + * Add to and processing the tx waitq is now on hold until unblock is called.
7857 8845   */
7858 8846  static void
7859      -mptsas_accept_tx_waitq(mptsas_t *mpt)
     8847 +mptsas_block_tx_waitqs(mptsas_t *mpt, int wait)
7860 8848  {
7861      -        mptsas_cmd_t *cmd;
     8849 +        int             i;
     8850 +        uint8_t         wdrain = 0;
     8851 +        mptsas_tx_waitqueue_t *txwq;
7862 8852  
7863 8853          ASSERT(mutex_owned(&mpt->m_mutex));
7864      -        ASSERT(mutex_owned(&mpt->m_tx_waitq_mutex));
     8854 +
     8855 +        if (mpt->m_txwq_thread_n == 0) {
     8856 +                return;
     8857 +        }
7865 8858  
7866 8859          /*
7867      -         * A Bus Reset could occur at any time and flush the tx_waitq,
7868      -         * so we cannot count on the tx_waitq to contain even one cmd.
7869      -         * And when the m_tx_waitq_mutex is released and run
7870      -         * mptsas_accept_pkt(), the tx_waitq may be flushed.
     8860 +         * Turn off the use of the tx wait queues by scsi_start().
     8861 +         * This is just a dynamic flag no need for a mutex.
7871 8862           */
7872      -        cmd = mpt->m_tx_waitq;
7873      -        for (;;) {
7874      -                if ((cmd = mpt->m_tx_waitq) == NULL) {
7875      -                        mpt->m_tx_draining = 0;
7876      -                        break;
7877      -                }
7878      -                if ((mpt->m_tx_waitq = cmd->cmd_linkp) == NULL) {
7879      -                        mpt->m_tx_waitqtail = &mpt->m_tx_waitq;
     8863 +        mpt->m_txwq_enabled = BLOCKED;
     8864 +
     8865 +        for (i = 0; i < NUM_TX_WAITQ; i++) {
     8866 +                txwq = &mpt->m_tx_waitq[i];
     8867 +                mutex_enter(&txwq->txwq_mutex);
     8868 +                txwq->txwq_wdrain = TRUE;
     8869 +                if (txwq->txwq_draining && wait)
     8870 +                        wdrain |= (1<<i);
     8871 +                mutex_exit(&txwq->txwq_mutex);
     8872 +        }
     8873 +
     8874 +        if (wdrain) {
     8875 +                /*
     8876 +                 * Because the threads disconnect the entire queue each time
     8877 +                 * round in order to drain to completely drain we have to
     8878 +                 * drop the main mutex otherwise the drain threads get stuck.
     8879 +                 */
     8880 +                mutex_exit(&mpt->m_mutex);
     8881 +                for (i = 0; i < NUM_TX_WAITQ; i++) {
     8882 +                        if (wdrain & (1<<i)) {
     8883 +                                txwq = &mpt->m_tx_waitq[i];
     8884 +                                mutex_enter(&txwq->txwq_mutex);
     8885 +                                while (txwq->txwq_draining) {
     8886 +                                        cv_wait(&txwq->txwq_drain_cv,
     8887 +                                            &txwq->txwq_mutex);
     8888 +                                }
     8889 +                                mutex_exit(&txwq->txwq_mutex);
     8890 +                        }
7880 8891                  }
7881      -                cmd->cmd_linkp = NULL;
7882      -                mutex_exit(&mpt->m_tx_waitq_mutex);
7883      -                if (mptsas_accept_pkt(mpt, cmd) != TRAN_ACCEPT)
7884      -                        cmn_err(CE_WARN, "mpt: mptsas_accept_tx_waitq: failed "
7885      -                            "to accept cmd on queue\n");
7886      -                mutex_enter(&mpt->m_tx_waitq_mutex);
     8892 +                mutex_enter(&mpt->m_mutex);
7887 8893          }
7888 8894  }
7889 8895  
     8896 +static void
     8897 +mptsas_unblock_tx_waitqs(mptsas_t *mpt)
     8898 +{
     8899 +        int                     i;
     8900 +        mptsas_tx_waitqueue_t   *txwq;
     8901 +
     8902 +        if (mpt->m_txwq_thread_n == 0) {
     8903 +                return;
     8904 +        }
     8905 +
     8906 +        for (i = 0; i < NUM_TX_WAITQ; i++) {
     8907 +                txwq = &mpt->m_tx_waitq[i];
     8908 +                mutex_enter(&txwq->txwq_mutex);
     8909 +                txwq->txwq_wdrain = FALSE;
     8910 +                cv_signal(&txwq->txwq_cv);
     8911 +                mutex_exit(&txwq->txwq_mutex);
     8912 +        }
     8913 +
     8914 +        mpt->m_txwq_enabled = FALSE;
     8915 +}
     8916 +
     8917 +static void
     8918 +mptsas_accept_tx_waitqs(mptsas_t *mpt)
     8919 +{
     8920 +        /*
     8921 +         * Block with drain and unblock will leave us in a state where
     8922 +         * we have the main mutex, there is nothing on the tx wait queues
     8923 +         * and they are not in use until watch notices high activity again.
     8924 +         */
     8925 +        mptsas_block_tx_waitqs(mpt, 1);
     8926 +        mptsas_unblock_tx_waitqs(mpt);
     8927 +}
7890 8928  
7891 8929  /*
7892 8930   * mpt tag type lookup
7893 8931   */
7894 8932  static char mptsas_tag_lookup[] =
7895 8933          {0, MSG_HEAD_QTAG, MSG_ORDERED_QTAG, 0, MSG_SIMPLE_QTAG};
7896 8934  
7897 8935  static int
7898 8936  mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
7899 8937  {
7900 8938          struct scsi_pkt         *pkt = CMD2PKT(cmd);
7901 8939          uint32_t                control = 0;
7902      -        int                     n;
7903      -        caddr_t                 mem;
     8940 +        caddr_t                 mem, arsbuf;
7904 8941          pMpi2SCSIIORequest_t    io_request;
7905 8942          ddi_dma_handle_t        dma_hdl = mpt->m_dma_req_frame_hdl;
7906 8943          ddi_acc_handle_t        acc_hdl = mpt->m_acc_req_frame_hdl;
7907 8944          mptsas_target_t         *ptgt = cmd->cmd_tgt_addr;
7908      -        uint16_t                SMID, io_flags = 0;
7909      -        uint32_t                request_desc_low, request_desc_high;
     8945 +        uint16_t                SMID, io_flags = 0, ars_size;
     8946 +        uint8_t                 MSIidx;
     8947 +        uint64_t                request_desc;
     8948 +        uint32_t                ars_dmaaddrlow;
     8949 +        mptsas_cmd_t            *c;
7910 8950  
7911      -        NDBG1(("mptsas_start_cmd: cmd=0x%p", (void *)cmd));
     8951 +        NDBG1(("mptsas_start_cmd: cmd=0x%p, flags 0x%x", (void *)cmd,
     8952 +            cmd->cmd_flags));
7912 8953  
7913 8954          /*
7914 8955           * Set SMID and increment index.  Rollover to 1 instead of 0 if index
7915 8956           * is at the max.  0 is an invalid SMID, so we call the first index 1.
7916 8957           */
7917 8958          SMID = cmd->cmd_slot;
     8959 +        MSIidx = cmd->cmd_rpqidx;
7918 8960  
7919 8961          /*
7920 8962           * It is possible for back to back device reset to
7921 8963           * happen before the reset delay has expired.  That's
7922 8964           * ok, just let the device reset go out on the bus.
7923 8965           */
7924 8966          if ((cmd->cmd_pkt_flags & FLAG_NOINTR) == 0) {
7925 8967                  ASSERT(ptgt->m_reset_delay == 0);
7926 8968          }
7927 8969  
7928 8970          /*
7929 8971           * if a non-tagged cmd is submitted to an active tagged target
7930 8972           * then drain before submitting this cmd; SCSI-2 allows RQSENSE
7931 8973           * to be untagged
7932 8974           */
     8975 +        mutex_enter(&ptgt->m_t_mutex);
7933 8976          if (((cmd->cmd_pkt_flags & FLAG_TAGMASK) == 0) &&
7934 8977              (ptgt->m_t_ncmds > 1) &&
7935 8978              ((cmd->cmd_flags & CFLAG_TM_CMD) == 0) &&
7936 8979              (*(cmd->cmd_pkt->pkt_cdbp) != SCMD_REQUEST_SENSE)) {
7937 8980                  if ((cmd->cmd_pkt_flags & FLAG_NOINTR) == 0) {
7938 8981                          NDBG23(("target=%d, untagged cmd, start draining\n",
7939 8982                              ptgt->m_devhdl));
7940 8983  
7941 8984                          if (ptgt->m_reset_delay == 0) {
7942 8985                                  mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
7943 8986                          }
     8987 +                        mutex_exit(&ptgt->m_t_mutex);
7944 8988  
     8989 +                        mutex_enter(&mpt->m_mutex);
7945 8990                          mptsas_remove_cmd(mpt, cmd);
7946 8991                          cmd->cmd_pkt_flags |= FLAG_HEAD;
7947 8992                          mptsas_waitq_add(mpt, cmd);
     8993 +                        mutex_exit(&mpt->m_mutex);
     8994 +                } else {
     8995 +                        mutex_exit(&ptgt->m_t_mutex);
7948 8996                  }
7949 8997                  return (DDI_FAILURE);
7950 8998          }
7951 8999  
7952 9000          /*
7953 9001           * Set correct tag bits.
7954 9002           */
7955 9003          if (cmd->cmd_pkt_flags & FLAG_TAGMASK) {
7956 9004                  switch (mptsas_tag_lookup[((cmd->cmd_pkt_flags &
7957 9005                      FLAG_TAGMASK) >> 12)]) {
↓ open down ↓ 10 lines elided ↑ open up ↑
7968 9016                          mptsas_log(mpt, CE_WARN, "mpt: Invalid tag type\n");
7969 9017                          break;
7970 9018                  }
7971 9019          } else {
7972 9020                  if (*(cmd->cmd_pkt->pkt_cdbp) != SCMD_REQUEST_SENSE) {
7973 9021                                  ptgt->m_t_throttle = 1;
7974 9022                  }
7975 9023                  control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
7976 9024          }
7977 9025  
     9026 +        /*
     9027 +         * Set timeout.
     9028 +         */
     9029 +        cmd->cmd_active_expiration =
     9030 +            gethrtime() + (hrtime_t)pkt->pkt_time * NANOSEC;
     9031 +
     9032 +        c = TAILQ_FIRST(&ptgt->m_active_cmdq);
     9033 +        if (c == NULL ||
     9034 +            c->cmd_active_expiration < cmd->cmd_active_expiration) {
     9035 +                /*
     9036 +                 * Common case is that this is the last pending expiration
     9037 +                 * (or queue is empty). Insert at head of the queue.
     9038 +                 */
     9039 +                TAILQ_INSERT_HEAD(&ptgt->m_active_cmdq, cmd, cmd_active_link);
     9040 +        } else {
     9041 +                /*
     9042 +                 * Queue is not empty and first element expires later than
     9043 +                 * this command. Search for element expiring sooner.
     9044 +                 */
     9045 +                while ((c = TAILQ_NEXT(c, cmd_active_link)) != NULL) {
     9046 +                        if (c->cmd_active_expiration <
     9047 +                            cmd->cmd_active_expiration) {
     9048 +                                TAILQ_INSERT_BEFORE(c, cmd, cmd_active_link);
     9049 +                                break;
     9050 +                        }
     9051 +                }
     9052 +                if (c == NULL) {
     9053 +                        /*
     9054 +                         * No element found expiring sooner, append to
     9055 +                         * non-empty queue.
     9056 +                         */
     9057 +                        TAILQ_INSERT_TAIL(&ptgt->m_active_cmdq, cmd,
     9058 +                            cmd_active_link);
     9059 +                }
     9060 +        }
     9061 +
     9062 +        mutex_exit(&ptgt->m_t_mutex);
     9063 +
7978 9064          if (cmd->cmd_pkt_flags & FLAG_TLR) {
7979 9065                  control |= MPI2_SCSIIO_CONTROL_TLR_ON;
7980 9066          }
7981 9067  
7982 9068          mem = mpt->m_req_frame + (mpt->m_req_frame_size * SMID);
7983 9069          io_request = (pMpi2SCSIIORequest_t)mem;
7984      -
     9070 +        if (cmd->cmd_extrqslen != 0) {
     9071 +                /*
     9072 +                 * Mapping of the buffer was done in mptsas_pkt_alloc_extern().
     9073 +                 * Calculate the DMA address with the same offset.
     9074 +                 */
     9075 +                arsbuf = cmd->cmd_arq_buf;
     9076 +                ars_size = cmd->cmd_extrqslen;
     9077 +                ars_dmaaddrlow = (mpt->m_req_sense_dma_addr +
     9078 +                    ((uintptr_t)arsbuf - (uintptr_t)mpt->m_req_sense)) &
     9079 +                    0xffffffffull;
     9080 +        } else {
     9081 +                arsbuf = mpt->m_req_sense + (mpt->m_req_sense_size * (SMID-1));
     9082 +                cmd->cmd_arq_buf = arsbuf;
     9083 +                ars_size = mpt->m_req_sense_size;
     9084 +                ars_dmaaddrlow = (mpt->m_req_sense_dma_addr +
     9085 +                    (mpt->m_req_sense_size * (SMID-1))) &
     9086 +                    0xffffffffull;
     9087 +        }
7985 9088          bzero(io_request, sizeof (Mpi2SCSIIORequest_t));
     9089 +        bzero(arsbuf, ars_size);
     9090 +
7986 9091          ddi_put8(acc_hdl, &io_request->SGLOffset0, offsetof
7987 9092              (MPI2_SCSI_IO_REQUEST, SGL) / 4);
7988 9093          mptsas_init_std_hdr(acc_hdl, io_request, ptgt->m_devhdl, Lun(cmd), 0,
7989 9094              MPI2_FUNCTION_SCSI_IO_REQUEST);
7990 9095  
7991 9096          (void) ddi_rep_put8(acc_hdl, (uint8_t *)pkt->pkt_cdbp,
7992 9097              io_request->CDB.CDB32, cmd->cmd_cdblen, DDI_DEV_AUTOINCR);
7993 9098  
7994 9099          io_flags = cmd->cmd_cdblen;
     9100 +        if (mptsas3_use_fastpath &&
     9101 +            ptgt->m_io_flags & MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) {
     9102 +                io_flags |= MPI25_SCSIIO_IOFLAGS_FAST_PATH;
     9103 +                request_desc = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
     9104 +        } else {
     9105 +                request_desc = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
     9106 +        }
7995 9107          ddi_put16(acc_hdl, &io_request->IoFlags, io_flags);
7996 9108          /*
7997 9109           * setup the Scatter/Gather DMA list for this request
7998 9110           */
7999 9111          if (cmd->cmd_cookiec > 0) {
8000 9112                  mptsas_sge_setup(mpt, cmd, &control, io_request, acc_hdl);
8001 9113          } else {
8002 9114                  ddi_put32(acc_hdl, &io_request->SGL.MpiSimple.FlagsLength,
8003 9115                      ((uint32_t)MPI2_SGE_FLAGS_LAST_ELEMENT |
8004 9116                      MPI2_SGE_FLAGS_END_OF_BUFFER |
8005 9117                      MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
8006 9118                      MPI2_SGE_FLAGS_END_OF_LIST) << MPI2_SGE_FLAGS_SHIFT);
8007 9119          }
8008 9120  
8009 9121          /*
8010 9122           * save ARQ information
8011 9123           */
8012 9124          ddi_put8(acc_hdl, &io_request->SenseBufferLength, cmd->cmd_rqslen);
8013      -        if ((cmd->cmd_flags & (CFLAG_SCBEXTERN | CFLAG_EXTARQBUFVALID)) ==
8014      -            (CFLAG_SCBEXTERN | CFLAG_EXTARQBUFVALID)) {
8015      -                ddi_put32(acc_hdl, &io_request->SenseBufferLowAddress,
8016      -                    cmd->cmd_ext_arqcookie.dmac_address);
8017      -        } else {
8018      -                ddi_put32(acc_hdl, &io_request->SenseBufferLowAddress,
8019      -                    cmd->cmd_arqcookie.dmac_address);
8020      -        }
     9125 +        ddi_put32(acc_hdl, &io_request->SenseBufferLowAddress, ars_dmaaddrlow);
8021 9126  
8022 9127          ddi_put32(acc_hdl, &io_request->Control, control);
8023 9128  
8024      -        NDBG31(("starting message=0x%p, with cmd=0x%p",
8025      -            (void *)(uintptr_t)mpt->m_req_frame_dma_addr, (void *)cmd));
     9129 +        NDBG31(("starting message=%d(0x%p), with cmd=0x%p",
     9130 +            SMID, (void *)io_request, (void *)cmd));
8026 9131  
8027 9132          (void) ddi_dma_sync(dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
8028 9133  
8029 9134          /*
8030 9135           * Build request descriptor and write it to the request desc post reg.
8031 9136           */
8032      -        request_desc_low = (SMID << 16) + MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
8033      -        request_desc_high = ptgt->m_devhdl << 16;
8034      -        MPTSAS_START_CMD(mpt, request_desc_low, request_desc_high);
8035      -
8036      -        /*
8037      -         * Start timeout.
8038      -         */
8039      -#ifdef MPTSAS_TEST
8040      -        /*
8041      -         * Temporarily set timebase = 0;  needed for
8042      -         * timeout torture test.
8043      -         */
8044      -        if (mptsas_test_timeouts) {
8045      -                ptgt->m_timebase = 0;
8046      -        }
8047      -#endif
8048      -        n = pkt->pkt_time - ptgt->m_timebase;
8049      -
8050      -        if (n == 0) {
8051      -                (ptgt->m_dups)++;
8052      -                ptgt->m_timeout = ptgt->m_timebase;
8053      -        } else if (n > 0) {
8054      -                ptgt->m_timeout =
8055      -                    ptgt->m_timebase = pkt->pkt_time;
8056      -                ptgt->m_dups = 1;
8057      -        } else if (n < 0) {
8058      -                ptgt->m_timeout = ptgt->m_timebase;
8059      -        }
8060      -#ifdef MPTSAS_TEST
8061      -        /*
8062      -         * Set back to a number higher than
8063      -         * mptsas_scsi_watchdog_tick
8064      -         * so timeouts will happen in mptsas_watchsubr
8065      -         */
8066      -        if (mptsas_test_timeouts) {
8067      -                ptgt->m_timebase = 60;
8068      -        }
8069      -#endif
     9137 +        request_desc |= (SMID << 16) + (MSIidx << 8);
     9138 +        request_desc |= ((uint64_t)ptgt->m_devhdl << 48);
     9139 +        MPTSAS_START_CMD(mpt, request_desc);
8070 9140  
     9141 +#if 0
     9142 +        /* Is this of any benefit here, what is it going to catch? */
8071 9143          if ((mptsas_check_dma_handle(dma_hdl) != DDI_SUCCESS) ||
8072 9144              (mptsas_check_acc_handle(acc_hdl) != DDI_SUCCESS)) {
8073 9145                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
8074 9146                  return (DDI_FAILURE);
8075 9147          }
     9148 +#endif
8076 9149          return (DDI_SUCCESS);
8077 9150  }
8078 9151  
8079 9152  /*
8080      - * Select a helper thread to handle current doneq
     9153 + * Select a helper thread to handle given doneq.
     9154 + * Note that we don't require to have the main m_mutex here, but worst case
     9155 + * is that we wont follow the thread rotation to the letter.
     9156 + * However must ensure we have the mutex that covers the source dlist when
     9157 + * we actually hand off.
8081 9158   */
8082 9159  static void
8083      -mptsas_deliver_doneq_thread(mptsas_t *mpt)
     9160 +mptsas_deliver_doneq_thread(mptsas_t *mpt, mptsas_done_list_t *dlist)
8084 9161  {
8085      -        uint64_t                        t, i;
     9162 +        uint32_t                        t, i, j = mpt->m_doneq_next_thread;
8086 9163          uint32_t                        min = 0xffffffff;
8087 9164          mptsas_doneq_thread_list_t      *item;
8088 9165  
     9166 +        /*
     9167 +         * No need to take indivudual list mutex's during the loop.
     9168 +         * We are only reading values and the worst that will happen is that
     9169 +         * we pick the wrong thread.
     9170 +         */
8089 9171          for (i = 0; i < mpt->m_doneq_thread_n; i++) {
8090      -                item = &mpt->m_doneq_thread_id[i];
     9172 +                item = &mpt->m_doneq_thread_id[j];
     9173 +
8091 9174                  /*
8092 9175                   * If the completed command on help thread[i] less than
8093      -                 * doneq_thread_threshold, then pick the thread[i]. Otherwise
     9176 +                 * doneq_thread_threshold, then pick the thread[j]. Otherwise
8094 9177                   * pick a thread which has least completed command.
8095 9178                   */
8096      -
8097      -                mutex_enter(&item->mutex);
8098      -                if (item->len < mpt->m_doneq_thread_threshold) {
8099      -                        t = i;
8100      -                        mutex_exit(&item->mutex);
     9179 +                if (item->dlist.dl_len < mpt->m_doneq_thread_threshold) {
     9180 +                        t = j;
8101 9181                          break;
8102 9182                  }
8103      -                if (item->len < min) {
8104      -                        min = item->len;
8105      -                        t = i;
     9183 +                if (item->dlist.dl_len < min) {
     9184 +                        min = item->dlist.dl_len;
     9185 +                        t = j;
8106 9186                  }
8107      -                mutex_exit(&item->mutex);
     9187 +                if (++j == mpt->m_doneq_thread_n) {
     9188 +                        j = 0;
     9189 +                }
     9190 +        }
     9191 +        item = &mpt->m_doneq_thread_id[t];
     9192 +        mutex_enter(&item->mutex);
     9193 +        mptsas_doneq_mv(dlist, item);
     9194 +        cv_signal(&item->cv);
     9195 +        mutex_exit(&item->mutex);
     9196 +
     9197 +        /*
     9198 +         * Next time start at the next thread.
     9199 +         * This will minimize the potential of grabing a lock
     9200 +         * for a thread that is busy, either on a very busy systems
     9201 +         * or on one that is configured to do all command completion
     9202 +         * processing through threads.
     9203 +         */
     9204 +        if (++t == mpt->m_doneq_thread_n) {
     9205 +                t = 0;
8108 9206          }
8109      -        mutex_enter(&mpt->m_doneq_thread_id[t].mutex);
8110      -        mptsas_doneq_mv(mpt, t);
8111      -        cv_signal(&mpt->m_doneq_thread_id[t].cv);
8112      -        mutex_exit(&mpt->m_doneq_thread_id[t].mutex);
     9207 +        mpt->m_doneq_next_thread = (uint16_t)t;
8113 9208  }
8114 9209  
8115 9210  /*
8116      - * move the current global doneq to the doneq of thead[t]
     9211 + * move one doneq to another.
8117 9212   */
8118 9213  static void
8119      -mptsas_doneq_mv(mptsas_t *mpt, uint64_t t)
     9214 +mptsas_doneq_mv(mptsas_done_list_t *from, mptsas_doneq_thread_list_t *item)
8120 9215  {
     9216 +        mptsas_done_list_t              *to = &item->dlist;
8121 9217          mptsas_cmd_t                    *cmd;
8122      -        mptsas_doneq_thread_list_t      *item = &mpt->m_doneq_thread_id[t];
8123 9218  
8124      -        ASSERT(mutex_owned(&item->mutex));
8125      -        while ((cmd = mpt->m_doneq) != NULL) {
8126      -                if ((mpt->m_doneq = cmd->cmd_linkp) == NULL) {
8127      -                        mpt->m_donetail = &mpt->m_doneq;
8128      -                }
8129      -                cmd->cmd_linkp = NULL;
8130      -                *item->donetail = cmd;
8131      -                item->donetail = &cmd->cmd_linkp;
8132      -                mpt->m_doneq_len--;
8133      -                item->len++;
     9219 +        if ((cmd = from->dl_q) != NULL) {
     9220 +                *to->dl_tail = cmd;
     9221 +                to->dl_tail = from->dl_tail;
     9222 +                to->dl_len += from->dl_len;
     9223 +                from->dl_q = NULL;
     9224 +                from->dl_tail = &from->dl_q;
     9225 +                from->dl_len = 0;
8134 9226          }
8135 9227  }
8136 9228  
8137 9229  void
8138 9230  mptsas_fma_check(mptsas_t *mpt, mptsas_cmd_t *cmd)
8139 9231  {
8140 9232          struct scsi_pkt *pkt = CMD2PKT(cmd);
8141 9233  
8142 9234          /* Check all acc and dma handles */
8143 9235          if ((mptsas_check_acc_handle(mpt->m_datap) !=
8144 9236              DDI_SUCCESS) ||
8145 9237              (mptsas_check_acc_handle(mpt->m_acc_req_frame_hdl) !=
8146 9238              DDI_SUCCESS) ||
     9239 +            (mptsas_check_acc_handle(mpt->m_acc_req_sense_hdl) !=
     9240 +            DDI_SUCCESS) ||
8147 9241              (mptsas_check_acc_handle(mpt->m_acc_reply_frame_hdl) !=
8148 9242              DDI_SUCCESS) ||
8149 9243              (mptsas_check_acc_handle(mpt->m_acc_free_queue_hdl) !=
8150 9244              DDI_SUCCESS) ||
8151 9245              (mptsas_check_acc_handle(mpt->m_acc_post_queue_hdl) !=
8152 9246              DDI_SUCCESS) ||
8153 9247              (mptsas_check_acc_handle(mpt->m_hshk_acc_hdl) !=
8154 9248              DDI_SUCCESS) ||
8155 9249              (mptsas_check_acc_handle(mpt->m_config_handle) !=
8156 9250              DDI_SUCCESS)) {
8157 9251                  ddi_fm_service_impact(mpt->m_dip,
8158 9252                      DDI_SERVICE_UNAFFECTED);
8159 9253                  ddi_fm_acc_err_clear(mpt->m_config_handle,
8160 9254                      DDI_FME_VER0);
8161 9255                  pkt->pkt_reason = CMD_TRAN_ERR;
8162 9256                  pkt->pkt_statistics = 0;
8163 9257          }
8164 9258          if ((mptsas_check_dma_handle(mpt->m_dma_req_frame_hdl) !=
8165 9259              DDI_SUCCESS) ||
     9260 +            (mptsas_check_dma_handle(mpt->m_dma_req_sense_hdl) !=
     9261 +            DDI_SUCCESS) ||
8166 9262              (mptsas_check_dma_handle(mpt->m_dma_reply_frame_hdl) !=
8167 9263              DDI_SUCCESS) ||
8168 9264              (mptsas_check_dma_handle(mpt->m_dma_free_queue_hdl) !=
8169 9265              DDI_SUCCESS) ||
8170 9266              (mptsas_check_dma_handle(mpt->m_dma_post_queue_hdl) !=
8171 9267              DDI_SUCCESS) ||
8172 9268              (mptsas_check_dma_handle(mpt->m_hshk_dma_hdl) !=
8173 9269              DDI_SUCCESS)) {
8174 9270                  ddi_fm_service_impact(mpt->m_dip,
8175 9271                      DDI_SERVICE_UNAFFECTED);
↓ open down ↓ 8 lines elided ↑ open up ↑
8184 9280          }
8185 9281          if ((cmd->cmd_extra_frames &&
8186 9282              ((mptsas_check_dma_handle(cmd->cmd_extra_frames->m_dma_hdl) !=
8187 9283              DDI_SUCCESS) ||
8188 9284              (mptsas_check_acc_handle(cmd->cmd_extra_frames->m_acc_hdl) !=
8189 9285              DDI_SUCCESS)))) {
8190 9286                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
8191 9287                  pkt->pkt_reason = CMD_TRAN_ERR;
8192 9288                  pkt->pkt_statistics = 0;
8193 9289          }
8194      -        if (cmd->cmd_arqhandle &&
8195      -            (mptsas_check_dma_handle(cmd->cmd_arqhandle) != DDI_SUCCESS)) {
8196      -                ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
8197      -                pkt->pkt_reason = CMD_TRAN_ERR;
8198      -                pkt->pkt_statistics = 0;
8199      -        }
8200      -        if (cmd->cmd_ext_arqhandle &&
8201      -            (mptsas_check_dma_handle(cmd->cmd_ext_arqhandle) != DDI_SUCCESS)) {
8202      -                ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
8203      -                pkt->pkt_reason = CMD_TRAN_ERR;
8204      -                pkt->pkt_statistics = 0;
8205      -        }
8206 9290  }
8207 9291  
8208 9292  /*
8209 9293   * These routines manipulate the queue of commands that
8210 9294   * are waiting for their completion routines to be called.
8211 9295   * The queue is usually in FIFO order but on an MP system
8212 9296   * it's possible for the completion routines to get out
8213 9297   * of order. If that's a problem you need to add a global
8214 9298   * mutex around the code that calls the completion routine
8215 9299   * in the interrupt handler.
↓ open down ↓ 10 lines elided ↑ open up ↑
8226 9310          cmd->cmd_flags |= CFLAG_FINISHED;
8227 9311          cmd->cmd_flags &= ~CFLAG_IN_TRANSPORT;
8228 9312  
8229 9313          mptsas_fma_check(mpt, cmd);
8230 9314  
8231 9315          /*
8232 9316           * only add scsi pkts that have completion routines to
8233 9317           * the doneq.  no intr cmds do not have callbacks.
8234 9318           */
8235 9319          if (pkt && (pkt->pkt_comp)) {
8236      -                *mpt->m_donetail = cmd;
8237      -                mpt->m_donetail = &cmd->cmd_linkp;
8238      -                mpt->m_doneq_len++;
     9320 +                *mpt->m_dlist.dl_tail = cmd;
     9321 +                mpt->m_dlist.dl_tail = &cmd->cmd_linkp;
     9322 +                mpt->m_dlist.dl_len++;
     9323 +        }
     9324 +}
     9325 +
     9326 +static void
     9327 +mptsas_rpdoneq_add(mptsas_t *mpt, mptsas_reply_pqueue_t *rpqp,
     9328 +    mptsas_cmd_t *cmd)
     9329 +{
     9330 +        struct scsi_pkt *pkt = CMD2PKT(cmd);
     9331 +
     9332 +        NDBG31(("mptsas_rpdoneq_add: cmd=0x%p", (void *)cmd));
     9333 +
     9334 +        ASSERT((cmd->cmd_flags & CFLAG_COMPLETED) == 0);
     9335 +        cmd->cmd_linkp = NULL;
     9336 +        cmd->cmd_flags |= CFLAG_FINISHED;
     9337 +        cmd->cmd_flags &= ~CFLAG_IN_TRANSPORT;
     9338 +
     9339 +        mptsas_fma_check(mpt, cmd);
     9340 +
     9341 +        /*
     9342 +         * only add scsi pkts that have completion routines to
     9343 +         * the doneq.  no intr cmds do not have callbacks.
     9344 +         */
     9345 +        if (pkt && (pkt->pkt_comp)) {
     9346 +                *rpqp->rpq_dlist.dl_tail = cmd;
     9347 +                rpqp->rpq_dlist.dl_tail = &cmd->cmd_linkp;
     9348 +                rpqp->rpq_dlist.dl_len++;
8239 9349          }
8240 9350  }
8241 9351  
8242 9352  static mptsas_cmd_t *
8243 9353  mptsas_doneq_thread_rm(mptsas_t *mpt, uint64_t t)
8244 9354  {
8245 9355          mptsas_cmd_t                    *cmd;
8246 9356          mptsas_doneq_thread_list_t      *item = &mpt->m_doneq_thread_id[t];
8247 9357  
8248 9358          /* pop one off the done queue */
8249      -        if ((cmd = item->doneq) != NULL) {
     9359 +        if ((cmd = item->dlist.dl_q) != NULL) {
8250 9360                  /* if the queue is now empty fix the tail pointer */
8251 9361                  NDBG31(("mptsas_doneq_thread_rm: cmd=0x%p", (void *)cmd));
8252      -                if ((item->doneq = cmd->cmd_linkp) == NULL) {
8253      -                        item->donetail = &item->doneq;
     9362 +                if ((item->dlist.dl_q = cmd->cmd_linkp) == NULL) {
     9363 +                        item->dlist.dl_tail = &item->dlist.dl_q;
8254 9364                  }
8255 9365                  cmd->cmd_linkp = NULL;
8256      -                item->len--;
     9366 +                item->dlist.dl_len--;
8257 9367          }
8258 9368          return (cmd);
8259 9369  }
8260 9370  
8261 9371  static void
8262 9372  mptsas_doneq_empty(mptsas_t *mpt)
8263 9373  {
8264      -        if (mpt->m_doneq && !mpt->m_in_callback) {
     9374 +        if (mpt->m_dlist.dl_q) {
8265 9375                  mptsas_cmd_t    *cmd, *next;
8266 9376                  struct scsi_pkt *pkt;
8267 9377  
8268      -                mpt->m_in_callback = 1;
8269      -                cmd = mpt->m_doneq;
8270      -                mpt->m_doneq = NULL;
8271      -                mpt->m_donetail = &mpt->m_doneq;
8272      -                mpt->m_doneq_len = 0;
     9378 +                cmd = mpt->m_dlist.dl_q;
     9379 +                mpt->m_dlist.dl_q = NULL;
     9380 +                mpt->m_dlist.dl_tail = &mpt->m_dlist.dl_q;
     9381 +                mpt->m_dlist.dl_len = 0;
8273 9382  
8274 9383                  mutex_exit(&mpt->m_mutex);
8275 9384                  /*
8276 9385                   * run the completion routines of all the
8277 9386                   * completed commands
8278 9387                   */
8279 9388                  while (cmd != NULL) {
8280 9389                          next = cmd->cmd_linkp;
8281 9390                          cmd->cmd_linkp = NULL;
8282 9391                          /* run this command's completion routine */
8283 9392                          cmd->cmd_flags |= CFLAG_COMPLETED;
8284 9393                          pkt = CMD2PKT(cmd);
8285 9394                          mptsas_pkt_comp(pkt, cmd);
8286 9395                          cmd = next;
8287 9396                  }
8288 9397                  mutex_enter(&mpt->m_mutex);
8289      -                mpt->m_in_callback = 0;
     9398 +        }
     9399 +}
     9400 +
     9401 +static void
     9402 +mptsas_rpdoneq_empty(mptsas_reply_pqueue_t *rpqp)
     9403 +{
     9404 +        if (rpqp->rpq_dlist.dl_q) {
     9405 +                mptsas_cmd_t    *cmd, *next;
     9406 +                struct scsi_pkt *pkt;
     9407 +
     9408 +                cmd = rpqp->rpq_dlist.dl_q;
     9409 +                rpqp->rpq_dlist.dl_q = NULL;
     9410 +                rpqp->rpq_dlist.dl_tail = &rpqp->rpq_dlist.dl_q;
     9411 +                rpqp->rpq_dlist.dl_len = 0;
     9412 +
     9413 +                mutex_exit(&rpqp->rpq_mutex);
     9414 +                /*
     9415 +                 * run the completion routines of all the
     9416 +                 * completed commands
     9417 +                 */
     9418 +                while (cmd != NULL) {
     9419 +                        next = cmd->cmd_linkp;
     9420 +                        cmd->cmd_linkp = NULL;
     9421 +                        /* run this command's completion routine */
     9422 +                        cmd->cmd_flags |= CFLAG_COMPLETED;
     9423 +                        pkt = CMD2PKT(cmd);
     9424 +                        mptsas_pkt_comp(pkt, cmd);
     9425 +                        cmd = next;
     9426 +                }
     9427 +                mutex_enter(&rpqp->rpq_mutex);
8290 9428          }
8291 9429  }
8292 9430  
8293 9431  /*
8294 9432   * These routines manipulate the target's queue of pending requests
8295 9433   */
8296 9434  void
8297 9435  mptsas_waitq_add(mptsas_t *mpt, mptsas_cmd_t *cmd)
8298 9436  {
8299 9437          NDBG7(("mptsas_waitq_add: cmd=0x%p", (void *)cmd));
↓ open down ↓ 69 lines elided ↑ open up ↑
8369 9507                          cmd->cmd_queued = FALSE;
8370 9508                          NDBG7(("mptsas_waitq_delete: mpt=0x%p cmd=0x%p",
8371 9509                              (void *)mpt, (void *)cmd));
8372 9510                          return;
8373 9511                  }
8374 9512                  prevp = prevp->cmd_linkp;
8375 9513          }
8376 9514          cmn_err(CE_PANIC, "mpt: mptsas_waitq_delete: queue botch");
8377 9515  }
8378 9516  
8379      -static mptsas_cmd_t *
8380      -mptsas_tx_waitq_rm(mptsas_t *mpt)
8381      -{
8382      -        mptsas_cmd_t *cmd;
8383      -        NDBG7(("mptsas_tx_waitq_rm"));
8384      -
8385      -        MPTSAS_TX_WAITQ_RM(mpt, cmd);
8386      -
8387      -        NDBG7(("mptsas_tx_waitq_rm: cmd=0x%p", (void *)cmd));
8388      -
8389      -        return (cmd);
8390      -}
8391      -
8392      -/*
8393      - * remove specified cmd from the middle of the tx_waitq.
8394      - */
8395      -static void
8396      -mptsas_tx_waitq_delete(mptsas_t *mpt, mptsas_cmd_t *cmd)
8397      -{
8398      -        mptsas_cmd_t *prevp = mpt->m_tx_waitq;
8399      -
8400      -        NDBG7(("mptsas_tx_waitq_delete: mpt=0x%p cmd=0x%p",
8401      -            (void *)mpt, (void *)cmd));
8402      -
8403      -        if (prevp == cmd) {
8404      -                if ((mpt->m_tx_waitq = cmd->cmd_linkp) == NULL)
8405      -                        mpt->m_tx_waitqtail = &mpt->m_tx_waitq;
8406      -
8407      -                cmd->cmd_linkp = NULL;
8408      -                cmd->cmd_queued = FALSE;
8409      -                NDBG7(("mptsas_tx_waitq_delete: mpt=0x%p cmd=0x%p",
8410      -                    (void *)mpt, (void *)cmd));
8411      -                return;
8412      -        }
8413      -
8414      -        while (prevp != NULL) {
8415      -                if (prevp->cmd_linkp == cmd) {
8416      -                        if ((prevp->cmd_linkp = cmd->cmd_linkp) == NULL)
8417      -                                mpt->m_tx_waitqtail = &prevp->cmd_linkp;
8418      -
8419      -                        cmd->cmd_linkp = NULL;
8420      -                        cmd->cmd_queued = FALSE;
8421      -                        NDBG7(("mptsas_tx_waitq_delete: mpt=0x%p cmd=0x%p",
8422      -                            (void *)mpt, (void *)cmd));
8423      -                        return;
8424      -                }
8425      -                prevp = prevp->cmd_linkp;
8426      -        }
8427      -        cmn_err(CE_PANIC, "mpt: mptsas_tx_waitq_delete: queue botch");
8428      -}
8429      -
8430 9517  /*
8431 9518   * device and bus reset handling
8432 9519   *
8433 9520   * Notes:
8434 9521   *      - RESET_ALL:    reset the controller
8435 9522   *      - RESET_TARGET: reset the target specified in scsi_address
8436 9523   */
8437 9524  static int
8438 9525  mptsas_scsi_reset(struct scsi_address *ap, int level)
8439 9526  {
↓ open down ↓ 95 lines elided ↑ open up ↑
8535 9622                  return (0);
8536 9623          }
8537 9624  }
8538 9625  
8539 9626  static int
8540 9627  mptsas_get_bus_addr(struct scsi_device *sd, char *name, int len)
8541 9628  {
8542 9629          return (mptsas_get_name(sd, name, len));
8543 9630  }
8544 9631  
8545      -void
     9632 +static void
8546 9633  mptsas_set_throttle(mptsas_t *mpt, mptsas_target_t *ptgt, int what)
8547 9634  {
8548 9635  
8549 9636          NDBG25(("mptsas_set_throttle: throttle=%x", what));
8550 9637  
8551 9638          /*
8552 9639           * if the bus is draining/quiesced, no changes to the throttles
8553 9640           * are allowed. Not allowing change of throttles during draining
8554 9641           * limits error recovery but will reduce draining time
8555 9642           *
↓ open down ↓ 3 lines elided ↑ open up ↑
8559 9646                  return;
8560 9647          }
8561 9648  
8562 9649          if (what == HOLD_THROTTLE) {
8563 9650                  ptgt->m_t_throttle = HOLD_THROTTLE;
8564 9651          } else if (ptgt->m_reset_delay == 0) {
8565 9652                  ptgt->m_t_throttle = what;
8566 9653          }
8567 9654  }
8568 9655  
     9656 +static void
     9657 +mptsas_set_throttle_mtx(mptsas_t *mpt, mptsas_target_t *ptgt, int what)
     9658 +{
     9659 +        if (mpt->m_softstate & (MPTSAS_SS_QUIESCED | MPTSAS_SS_DRAINING)) {
     9660 +                return;
     9661 +        }
     9662 +
     9663 +        mutex_enter(&ptgt->m_t_mutex);
     9664 +        mptsas_set_throttle(mpt, ptgt, what);
     9665 +        mutex_exit(&ptgt->m_t_mutex);
     9666 +}
     9667 +
     9668 +/*
     9669 + * Find all commands in the tx_waitq's for target and lun (if lun not -1),
     9670 + * remove them from the queues and return the linked list.
     9671 + */
     9672 +static mptsas_cmd_t *
     9673 +mptsas_strip_targetlun_from_txwqs(mptsas_t *mpt, ushort_t target, int lun)
     9674 +{
     9675 +        mptsas_cmd_t            *cmd, *clist, **tailp, **prev_tailp;
     9676 +        mptsas_tx_waitqueue_t   *txwq;
     9677 +        int                     i;
     9678 +
     9679 +        clist = NULL;
     9680 +        tailp = &clist;
     9681 +
     9682 +        for (i = 0; i < NUM_TX_WAITQ; i++) {
     9683 +                txwq = &mpt->m_tx_waitq[i];
     9684 +                mutex_enter(&txwq->txwq_mutex);
     9685 +                prev_tailp = &txwq->txwq_cmdq;
     9686 +                cmd = txwq->txwq_cmdq;
     9687 +                while (cmd != NULL) {
     9688 +                        if (Tgt(cmd) == target &&
     9689 +                            (lun == -1 || (Lun(cmd) == lun))) {
     9690 +                                *prev_tailp = cmd->cmd_linkp;
     9691 +                                *tailp = cmd;
     9692 +                                tailp = &cmd->cmd_linkp;
     9693 +                                cmd = cmd->cmd_linkp;
     9694 +                                *tailp = NULL;
     9695 +                        } else {
     9696 +                                prev_tailp = &cmd->cmd_linkp;
     9697 +                                cmd = cmd->cmd_linkp;
     9698 +                        }
     9699 +                }
     9700 +                txwq->txwq_qtail = prev_tailp;
     9701 +                mutex_exit(&txwq->txwq_mutex);
     9702 +        }
     9703 +        return (clist);
     9704 +}
     9705 +
8569 9706  /*
8570 9707   * Clean up from a device reset.
8571 9708   * For the case of target reset, this function clears the waitq of all
8572 9709   * commands for a particular target.   For the case of abort task set, this
8573 9710   * function clears the waitq of all commonds for a particular target/lun.
8574 9711   */
8575 9712  static void
8576 9713  mptsas_flush_target(mptsas_t *mpt, ushort_t target, int lun, uint8_t tasktype)
8577 9714  {
8578 9715          mptsas_slots_t  *slots = mpt->m_active;
8579 9716          mptsas_cmd_t    *cmd, *next_cmd;
8580 9717          int             slot;
8581 9718          uchar_t         reason;
8582 9719          uint_t          stat;
     9720 +        hrtime_t        timestamp;
8583 9721  
8584 9722          NDBG25(("mptsas_flush_target: target=%d lun=%d", target, lun));
8585 9723  
     9724 +        timestamp = gethrtime();
     9725 +
8586 9726          /*
8587 9727           * Make sure the I/O Controller has flushed all cmds
8588 9728           * that are associated with this target for a target reset
8589 9729           * and target/lun for abort task set.
8590 9730           * Account for TM requests, which use the last SMID.
8591 9731           */
8592 9732          for (slot = 0; slot <= mpt->m_active->m_n_normal; slot++) {
8593 9733                  if ((cmd = slots->m_slot[slot]) == NULL)
8594 9734                          continue;
8595 9735                  reason = CMD_RESET;
8596 9736                  stat = STAT_DEV_RESET;
8597 9737                  switch (tasktype) {
8598 9738                  case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
8599 9739                          if (Tgt(cmd) == target) {
     9740 +                                if (cmd->cmd_active_expiration <= timestamp) {
     9741 +                                        /*
     9742 +                                         * When timeout requested, propagate
     9743 +                                         * proper reason and statistics to
     9744 +                                         * target drivers.
     9745 +                                         */
     9746 +                                        reason = CMD_TIMEOUT;
     9747 +                                        stat |= STAT_TIMEOUT;
     9748 +                                }
8600 9749                                  NDBG25(("mptsas_flush_target discovered non-"
8601 9750                                      "NULL cmd in slot %d, tasktype 0x%x", slot,
8602 9751                                      tasktype));
8603 9752                                  mptsas_dump_cmd(mpt, cmd);
8604 9753                                  mptsas_remove_cmd(mpt, cmd);
8605 9754                                  mptsas_set_pkt_reason(mpt, cmd, reason, stat);
8606 9755                                  mptsas_doneq_add(mpt, cmd);
8607 9756                          }
8608 9757                          break;
8609 9758                  case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
8610 9759                          reason = CMD_ABORTED;
8611 9760                          stat = STAT_ABORTED;
8612 9761                          /*FALLTHROUGH*/
8613 9762                  case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
8614 9763                          if ((Tgt(cmd) == target) && (Lun(cmd) == lun)) {
     9764 +                                if (cmd->cmd_active_expiration <= timestamp) {
     9765 +                                        stat |= STAT_TIMEOUT;
     9766 +                                }
8615 9767  
8616 9768                                  NDBG25(("mptsas_flush_target discovered non-"
8617 9769                                      "NULL cmd in slot %d, tasktype 0x%x", slot,
8618 9770                                      tasktype));
8619 9771                                  mptsas_dump_cmd(mpt, cmd);
8620 9772                                  mptsas_remove_cmd(mpt, cmd);
8621      -                                mptsas_set_pkt_reason(mpt, cmd, reason,
8622      -                                    stat);
     9773 +                                mptsas_set_pkt_reason(mpt, cmd, reason, stat);
8623 9774                                  mptsas_doneq_add(mpt, cmd);
8624 9775                          }
8625 9776                          break;
8626 9777                  default:
8627 9778                          break;
8628 9779                  }
8629 9780          }
8630 9781  
8631 9782          /*
8632 9783           * Flush the waitq and tx_waitq of this target's cmds
↓ open down ↓ 8 lines elided ↑ open up ↑
8641 9792                  while (cmd != NULL) {
8642 9793                          next_cmd = cmd->cmd_linkp;
8643 9794                          if (Tgt(cmd) == target) {
8644 9795                                  mptsas_waitq_delete(mpt, cmd);
8645 9796                                  mptsas_set_pkt_reason(mpt, cmd,
8646 9797                                      reason, stat);
8647 9798                                  mptsas_doneq_add(mpt, cmd);
8648 9799                          }
8649 9800                          cmd = next_cmd;
8650 9801                  }
8651      -                mutex_enter(&mpt->m_tx_waitq_mutex);
8652      -                cmd = mpt->m_tx_waitq;
     9802 +                cmd = mptsas_strip_targetlun_from_txwqs(mpt, target, -1);
8653 9803                  while (cmd != NULL) {
8654 9804                          next_cmd = cmd->cmd_linkp;
8655      -                        if (Tgt(cmd) == target) {
8656      -                                mptsas_tx_waitq_delete(mpt, cmd);
8657      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
8658      -                                mptsas_set_pkt_reason(mpt, cmd,
8659      -                                    reason, stat);
8660      -                                mptsas_doneq_add(mpt, cmd);
8661      -                                mutex_enter(&mpt->m_tx_waitq_mutex);
8662      -                        }
     9805 +                        mptsas_set_pkt_reason(mpt, cmd, reason, stat);
     9806 +                        mptsas_doneq_add(mpt, cmd);
8663 9807                          cmd = next_cmd;
8664 9808                  }
8665      -                mutex_exit(&mpt->m_tx_waitq_mutex);
8666 9809                  break;
8667 9810          case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
8668 9811                  reason = CMD_ABORTED;
8669 9812                  stat =  STAT_ABORTED;
8670 9813                  /*FALLTHROUGH*/
8671 9814          case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
8672 9815                  while (cmd != NULL) {
8673 9816                          next_cmd = cmd->cmd_linkp;
8674 9817                          if ((Tgt(cmd) == target) && (Lun(cmd) == lun)) {
8675 9818                                  mptsas_waitq_delete(mpt, cmd);
8676 9819                                  mptsas_set_pkt_reason(mpt, cmd,
8677 9820                                      reason, stat);
8678 9821                                  mptsas_doneq_add(mpt, cmd);
8679 9822                          }
8680 9823                          cmd = next_cmd;
8681 9824                  }
8682      -                mutex_enter(&mpt->m_tx_waitq_mutex);
8683      -                cmd = mpt->m_tx_waitq;
     9825 +                cmd = mptsas_strip_targetlun_from_txwqs(mpt, target, lun);
8684 9826                  while (cmd != NULL) {
8685 9827                          next_cmd = cmd->cmd_linkp;
8686      -                        if ((Tgt(cmd) == target) && (Lun(cmd) == lun)) {
8687      -                                mptsas_tx_waitq_delete(mpt, cmd);
8688      -                                mutex_exit(&mpt->m_tx_waitq_mutex);
8689      -                                mptsas_set_pkt_reason(mpt, cmd,
8690      -                                    reason, stat);
8691      -                                mptsas_doneq_add(mpt, cmd);
8692      -                                mutex_enter(&mpt->m_tx_waitq_mutex);
8693      -                        }
     9828 +                        mptsas_set_pkt_reason(mpt, cmd, reason, stat);
     9829 +                        mptsas_doneq_add(mpt, cmd);
8694 9830                          cmd = next_cmd;
8695 9831                  }
8696      -                mutex_exit(&mpt->m_tx_waitq_mutex);
8697 9832                  break;
8698 9833          default:
8699 9834                  mptsas_log(mpt, CE_WARN, "Unknown task management type %d.",
8700 9835                      tasktype);
8701 9836                  break;
8702 9837          }
8703 9838  }
8704 9839  
8705 9840  /*
8706 9841   * Clean up hba state, abort all outstanding command and commands in waitq
8707 9842   * reset timeout of all targets.
8708 9843   */
8709 9844  static void
8710 9845  mptsas_flush_hba(mptsas_t *mpt)
8711 9846  {
8712 9847          mptsas_slots_t  *slots = mpt->m_active;
8713      -        mptsas_cmd_t    *cmd;
8714      -        int             slot;
     9848 +        mptsas_cmd_t    *cmd, *ncmd;
     9849 +        int             slot, i;
8715 9850  
8716 9851          NDBG25(("mptsas_flush_hba"));
8717 9852  
8718 9853          /*
8719 9854           * The I/O Controller should have already sent back
8720 9855           * all commands via the scsi I/O reply frame.  Make
8721 9856           * sure all commands have been flushed.
8722 9857           * Account for TM request, which use the last SMID.
8723 9858           */
8724 9859          for (slot = 0; slot <= mpt->m_active->m_n_normal; slot++) {
↓ open down ↓ 4 lines elided ↑ open up ↑
8729 9864                          /*
8730 9865                           * Need to make sure to tell everyone that might be
8731 9866                           * waiting on this command that it's going to fail.  If
8732 9867                           * we get here, this command will never timeout because
8733 9868                           * the active command table is going to be re-allocated,
8734 9869                           * so there will be nothing to check against a time out.
8735 9870                           * Instead, mark the command as failed due to reset.
8736 9871                           */
8737 9872                          mptsas_set_pkt_reason(mpt, cmd, CMD_RESET,
8738 9873                              STAT_BUS_RESET);
8739      -                        if ((cmd->cmd_flags & CFLAG_PASSTHRU) ||
8740      -                            (cmd->cmd_flags & CFLAG_CONFIG) ||
8741      -                            (cmd->cmd_flags & CFLAG_FW_DIAG)) {
     9874 +                        if ((cmd->cmd_flags &
     9875 +                            (CFLAG_PASSTHRU | CFLAG_CONFIG | CFLAG_FW_DIAG))) {
8742 9876                                  cmd->cmd_flags |= CFLAG_FINISHED;
8743 9877                                  cv_broadcast(&mpt->m_passthru_cv);
8744 9878                                  cv_broadcast(&mpt->m_config_cv);
8745 9879                                  cv_broadcast(&mpt->m_fw_diag_cv);
8746 9880                          }
8747 9881                          continue;
8748 9882                  }
8749 9883  
8750 9884                  NDBG25(("mptsas_flush_hba discovered non-NULL cmd in slot %d",
8751 9885                      slot));
↓ open down ↓ 15 lines elided ↑ open up ↑
8767 9901                          cmd->cmd_flags |= CFLAG_FINISHED;
8768 9902                          cv_broadcast(&mpt->m_passthru_cv);
8769 9903                          cv_broadcast(&mpt->m_config_cv);
8770 9904                          cv_broadcast(&mpt->m_fw_diag_cv);
8771 9905                  } else {
8772 9906                          mptsas_doneq_add(mpt, cmd);
8773 9907                  }
8774 9908          }
8775 9909  
8776 9910          /*
8777      -         * Flush the tx_waitq
     9911 +         * Flush the tx_waitqs
8778 9912           */
8779      -        mutex_enter(&mpt->m_tx_waitq_mutex);
8780      -        while ((cmd = mptsas_tx_waitq_rm(mpt)) != NULL) {
8781      -                mutex_exit(&mpt->m_tx_waitq_mutex);
8782      -                mptsas_set_pkt_reason(mpt, cmd, CMD_RESET, STAT_BUS_RESET);
8783      -                mptsas_doneq_add(mpt, cmd);
8784      -                mutex_enter(&mpt->m_tx_waitq_mutex);
     9913 +        for (i = 0; i < NUM_TX_WAITQ; i++) {
     9914 +                mutex_enter(&mpt->m_tx_waitq[i].txwq_mutex);
     9915 +                cmd = mpt->m_tx_waitq[i].txwq_cmdq;
     9916 +                mpt->m_tx_waitq[i].txwq_cmdq = NULL;
     9917 +                mpt->m_tx_waitq[i].txwq_qtail = &mpt->m_tx_waitq[i].txwq_cmdq;
     9918 +                mutex_exit(&mpt->m_tx_waitq[i].txwq_mutex);
     9919 +                while (cmd != NULL) {
     9920 +                        ncmd = cmd->cmd_linkp;
     9921 +                        mptsas_set_pkt_reason(mpt, cmd, CMD_RESET,
     9922 +                            STAT_BUS_RESET);
     9923 +                        mptsas_doneq_add(mpt, cmd);
     9924 +                        cmd = ncmd;
     9925 +                }
8785 9926          }
8786      -        mutex_exit(&mpt->m_tx_waitq_mutex);
8787 9927  
8788 9928          /*
8789 9929           * Drain the taskqs prior to reallocating resources.
8790 9930           */
8791 9931          mutex_exit(&mpt->m_mutex);
8792 9932          ddi_taskq_wait(mpt->m_event_taskq);
8793 9933          ddi_taskq_wait(mpt->m_dr_taskq);
8794 9934          mutex_enter(&mpt->m_mutex);
8795 9935  }
8796 9936  
↓ open down ↓ 37 lines elided ↑ open up ↑
8834 9974  static void
8835 9975  mptsas_setup_bus_reset_delay(mptsas_t *mpt)
8836 9976  {
8837 9977          mptsas_target_t *ptgt = NULL;
8838 9978  
8839 9979          ASSERT(MUTEX_HELD(&mpt->m_mutex));
8840 9980  
8841 9981          NDBG22(("mptsas_setup_bus_reset_delay"));
8842 9982          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
8843 9983              ptgt = refhash_next(mpt->m_targets, ptgt)) {
     9984 +                mutex_enter(&ptgt->m_t_mutex);
8844 9985                  mptsas_set_throttle(mpt, ptgt, HOLD_THROTTLE);
8845 9986                  ptgt->m_reset_delay = mpt->m_scsi_reset_delay;
     9987 +                mutex_exit(&ptgt->m_t_mutex);
8846 9988          }
8847 9989  
8848 9990          mptsas_start_watch_reset_delay();
8849 9991  }
8850 9992  
8851 9993  /*
8852 9994   * mptsas_watch_reset_delay(_subr) is invoked by timeout() and checks every
8853 9995   * mpt instance for active reset delays
8854 9996   */
8855 9997  static void
↓ open down ↓ 33 lines elided ↑ open up ↑
8889 10031          int             done = 0;
8890 10032          int             restart = 0;
8891 10033          mptsas_target_t *ptgt = NULL;
8892 10034  
8893 10035          NDBG22(("mptsas_watch_reset_delay_subr: mpt=0x%p", (void *)mpt));
8894 10036  
8895 10037          ASSERT(mutex_owned(&mpt->m_mutex));
8896 10038  
8897 10039          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
8898 10040              ptgt = refhash_next(mpt->m_targets, ptgt)) {
     10041 +                mutex_enter(&ptgt->m_t_mutex);
8899 10042                  if (ptgt->m_reset_delay != 0) {
8900 10043                          ptgt->m_reset_delay -=
8901 10044                              MPTSAS_WATCH_RESET_DELAY_TICK;
8902 10045                          if (ptgt->m_reset_delay <= 0) {
8903 10046                                  ptgt->m_reset_delay = 0;
8904 10047                                  mptsas_set_throttle(mpt, ptgt,
8905 10048                                      MAX_THROTTLE);
8906 10049                                  restart++;
8907 10050                          } else {
8908 10051                                  done = -1;
8909 10052                          }
8910 10053                  }
     10054 +                mutex_exit(&ptgt->m_t_mutex);
8911 10055          }
8912 10056  
8913 10057          if (restart > 0) {
8914 10058                  mptsas_restart_hba(mpt);
8915 10059          }
8916 10060          return (done);
8917 10061  }
8918 10062  
8919 10063  #ifdef MPTSAS_TEST
8920 10064  static void
↓ open down ↓ 196 lines elided ↑ open up ↑
9117 10261          return (rval);
9118 10262  }
9119 10263  
9120 10264  /*
9121 10265   * (*tran_setcap).  Set the capability named to the value given.
9122 10266   */
9123 10267  static int
9124 10268  mptsas_scsi_setcap(struct scsi_address *ap, char *cap, int value, int tgtonly)
9125 10269  {
9126 10270          mptsas_t        *mpt = ADDR2MPT(ap);
     10271 +        mptsas_target_t *ptgt;
9127 10272          int             ckey;
9128 10273          int             rval = FALSE;
9129 10274  
9130 10275          NDBG24(("mptsas_scsi_setcap: target=%d, cap=%s value=%x tgtonly=%x",
9131 10276              ap->a_target, cap, value, tgtonly));
9132 10277  
9133 10278          if (!tgtonly) {
9134 10279                  return (rval);
9135 10280          }
9136 10281  
↓ open down ↓ 21 lines elided ↑ open up ↑
9158 10303                  /*
9159 10304                   * We cannot turn off arq so return false if asked to
9160 10305                   */
9161 10306                  if (value) {
9162 10307                          rval = TRUE;
9163 10308                  } else {
9164 10309                          rval = FALSE;
9165 10310                  }
9166 10311                  break;
9167 10312          case SCSI_CAP_TAGGED_QING:
9168      -                mptsas_set_throttle(mpt, ((mptsas_tgt_private_t *)
9169      -                    (ap->a_hba_tran->tran_tgt_private))->t_private,
9170      -                    MAX_THROTTLE);
     10313 +                ptgt = ((mptsas_tgt_private_t *)
     10314 +                    (ap->a_hba_tran->tran_tgt_private))->t_private;
     10315 +                mptsas_set_throttle_mtx(mpt, ptgt, MAX_THROTTLE);
9171 10316                  rval = TRUE;
9172 10317                  break;
9173 10318          case SCSI_CAP_QFULL_RETRIES:
9174 10319                  ((mptsas_tgt_private_t *)(ap->a_hba_tran->tran_tgt_private))->
9175 10320                      t_private->m_qfull_retries = (uchar_t)value;
9176 10321                  rval = TRUE;
9177 10322                  break;
9178 10323          case SCSI_CAP_QFULL_RETRY_INTERVAL:
9179 10324                  ((mptsas_tgt_private_t *)(ap->a_hba_tran->tran_tgt_private))->
9180 10325                      t_private->m_qfull_retry_interval =
↓ open down ↓ 67 lines elided ↑ open up ↑
9248 10393          if (active == NULL)
9249 10394                  return;
9250 10395          size = active->m_size;
9251 10396          kmem_free(active, size);
9252 10397          mpt->m_active = NULL;
9253 10398  }
9254 10399  
9255 10400  /*
9256 10401   * Error logging, printing, and debug print routines.
9257 10402   */
9258      -static char *mptsas_label = "mpt_sas";
     10403 +static char *mptsas_label = "mpt_sas3";
9259 10404  
9260 10405  /*PRINTFLIKE3*/
9261 10406  void
9262 10407  mptsas_log(mptsas_t *mpt, int level, char *fmt, ...)
9263 10408  {
9264 10409          dev_info_t      *dev;
9265 10410          va_list         ap;
9266 10411  
9267 10412          if (mpt) {
9268 10413                  dev = mpt->m_dip;
↓ open down ↓ 10 lines elided ↑ open up ↑
9279 10424          if (level == CE_CONT) {
9280 10425                  scsi_log(dev, mptsas_label, level, "%s\n", mptsas_log_buf);
9281 10426          } else {
9282 10427                  scsi_log(dev, mptsas_label, level, "%s", mptsas_log_buf);
9283 10428          }
9284 10429  
9285 10430          mutex_exit(&mptsas_log_mutex);
9286 10431  }
9287 10432  
9288 10433  #ifdef MPTSAS_DEBUG
     10434 +/*
     10435 + * Use a circular buffer to log messages to private memory.
     10436 + * No mutexes, so there is the opportunity for this to miss lines.
     10437 + * But it's fast and does not hold up the proceedings too much.
     10438 + */
     10439 +static char mptsas_dbglog_bufs[32][256];
     10440 +static uint32_t mptsas_dbglog_idx = 1;
     10441 +
     10442 +/*PRINTFLIKE1*/
     10443 +void
     10444 +mptsas_debug_log(char *fmt, ...)
     10445 +{
     10446 +        va_list         ap;
     10447 +        uint32_t        idx;
     10448 +
     10449 +        if (!mptsas_dbglog_idx) {
     10450 +                return;
     10451 +        }
     10452 +        idx = (mptsas_dbglog_idx++) & 0x1f;
     10453 +
     10454 +        va_start(ap, fmt);
     10455 +        (void) vsnprintf(mptsas_dbglog_bufs[idx],
     10456 +            sizeof (mptsas_dbglog_bufs[0]), fmt, ap);
     10457 +        va_end(ap);
     10458 +}
     10459 +
9289 10460  /*PRINTFLIKE1*/
9290 10461  void
9291 10462  mptsas_printf(char *fmt, ...)
9292 10463  {
9293 10464          dev_info_t      *dev = 0;
9294 10465          va_list         ap;
9295 10466  
9296 10467          mutex_enter(&mptsas_log_mutex);
9297 10468  
9298 10469          va_start(ap, fmt);
9299 10470          (void) vsprintf(mptsas_log_buf, fmt, ap);
9300 10471          va_end(ap);
9301 10472  
9302 10473  #ifdef PROM_PRINTF
9303 10474          prom_printf("%s:\t%s\n", mptsas_label, mptsas_log_buf);
9304 10475  #else
9305      -        scsi_log(dev, mptsas_label, SCSI_DEBUG, "%s\n", mptsas_log_buf);
     10476 +        scsi_log(dev, mptsas_label, CE_CONT, "!%s\n", mptsas_log_buf);
9306 10477  #endif
9307 10478          mutex_exit(&mptsas_log_mutex);
9308 10479  }
9309 10480  #endif
9310 10481  
9311 10482  /*
9312 10483   * timeout handling
9313 10484   */
9314 10485  static void
9315 10486  mptsas_watch(void *arg)
↓ open down ↓ 51 lines elided ↑ open up ↑
9367 10538                  mutex_exit(&mpt->m_mutex);
9368 10539          }
9369 10540          rw_exit(&mptsas_global_rwlock);
9370 10541  
9371 10542          mutex_enter(&mptsas_global_mutex);
9372 10543          if (mptsas_timeouts_enabled)
9373 10544                  mptsas_timeout_id = timeout(mptsas_watch, NULL, mptsas_tick);
9374 10545          mutex_exit(&mptsas_global_mutex);
9375 10546  }
9376 10547  
     10548 +int mptsas_monitor_for_txwqs = 1;
9377 10549  static void
9378 10550  mptsas_watchsubr(mptsas_t *mpt)
9379 10551  {
9380 10552          int             i;
9381 10553          mptsas_cmd_t    *cmd;
9382 10554          mptsas_target_t *ptgt = NULL;
     10555 +        hrtime_t        timestamp = gethrtime();
     10556 +        boolean_t       restart_hba = B_FALSE;
9383 10557  
9384 10558          ASSERT(MUTEX_HELD(&mpt->m_mutex));
9385 10559  
9386      -        NDBG30(("mptsas_watchsubr: mpt=0x%p", (void *)mpt));
     10560 +        NDBG30(("mptsas_watchsubr: mpt=0x%p, ncmds %d, nstarted %d",
     10561 +            (void *)mpt, mpt->m_ncmds, mpt->m_ncstarted));
9387 10562  
9388      -#ifdef MPTSAS_TEST
9389      -        if (mptsas_enable_untagged) {
9390      -                mptsas_test_untagged++;
     10563 +        mpt->m_lncstarted = mpt->m_ncstarted;
     10564 +        if (mpt->m_txwq_thread_n != 0 && mpt->m_txwq_enabled != BLOCKED &&
     10565 +            mptsas_monitor_for_txwqs) {
     10566 +                i = mpt->m_ncstarted/mptsas_scsi_watchdog_tick;
     10567 +                if (i > mpt->m_txwq_thread_threshold) {
     10568 +                        mpt->m_txwq_enabled = TRUE;
     10569 +                } else if (i < (mpt->m_txwq_thread_threshold>>1)) {
     10570 +                        mpt->m_txwq_enabled = FALSE;
     10571 +                }
9391 10572          }
9392      -#endif
     10573 +        mpt->m_ncstarted = 0;
9393 10574  
9394 10575          /*
9395 10576           * Check for commands stuck in active slot
9396 10577           * Account for TM requests, which use the last SMID.
9397 10578           */
9398 10579          for (i = 0; i <= mpt->m_active->m_n_normal; i++) {
9399 10580                  if ((cmd = mpt->m_active->m_slot[i]) != NULL) {
9400      -                        if ((cmd->cmd_flags & CFLAG_CMDIOC) == 0) {
9401      -                                cmd->cmd_active_timeout -=
9402      -                                    mptsas_scsi_watchdog_tick;
9403      -                                if (cmd->cmd_active_timeout <= 0) {
     10581 +                        if (cmd->cmd_active_expiration <= timestamp) {
     10582 +                                if ((cmd->cmd_flags & CFLAG_CMDIOC) == 0) {
9404 10583                                          /*
9405 10584                                           * There seems to be a command stuck
9406 10585                                           * in the active slot.  Drain throttle.
9407 10586                                           */
9408      -                                        mptsas_set_throttle(mpt,
9409      -                                            cmd->cmd_tgt_addr,
     10587 +                                        ptgt = cmd->cmd_tgt_addr;
     10588 +                                        mptsas_set_throttle_mtx(mpt, ptgt,
9410 10589                                              DRAIN_THROTTLE);
9411      -                                }
9412      -                        }
9413      -                        if ((cmd->cmd_flags & CFLAG_PASSTHRU) ||
9414      -                            (cmd->cmd_flags & CFLAG_CONFIG) ||
9415      -                            (cmd->cmd_flags & CFLAG_FW_DIAG)) {
9416      -                                cmd->cmd_active_timeout -=
9417      -                                    mptsas_scsi_watchdog_tick;
9418      -                                if (cmd->cmd_active_timeout <= 0) {
     10590 +                                } else if (cmd->cmd_flags &
     10591 +                                    (CFLAG_PASSTHRU | CFLAG_CONFIG |
     10592 +                                    CFLAG_FW_DIAG)) {
9419 10593                                          /*
9420 10594                                           * passthrough command timeout
9421 10595                                           */
9422 10596                                          cmd->cmd_flags |= (CFLAG_FINISHED |
9423 10597                                              CFLAG_TIMEOUT);
9424 10598                                          cv_broadcast(&mpt->m_passthru_cv);
9425 10599                                          cv_broadcast(&mpt->m_config_cv);
9426 10600                                          cv_broadcast(&mpt->m_fw_diag_cv);
9427 10601                                  }
9428 10602                          }
9429 10603                  }
9430 10604          }
9431 10605  
9432 10606          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
9433 10607              ptgt = refhash_next(mpt->m_targets, ptgt)) {
     10608 +                mutex_enter(&ptgt->m_t_mutex);
9434 10609                  /*
9435 10610                   * If we were draining due to a qfull condition,
9436 10611                   * go back to full throttle.
9437 10612                   */
9438 10613                  if ((ptgt->m_t_throttle < MAX_THROTTLE) &&
9439 10614                      (ptgt->m_t_throttle > HOLD_THROTTLE) &&
9440 10615                      (ptgt->m_t_ncmds < ptgt->m_t_throttle)) {
9441 10616                          mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
9442      -                        mptsas_restart_hba(mpt);
     10617 +                        restart_hba = B_TRUE;
9443 10618                  }
9444 10619  
9445      -                if ((ptgt->m_t_ncmds > 0) &&
9446      -                    (ptgt->m_timebase)) {
9447      -
9448      -                        if (ptgt->m_timebase <=
9449      -                            mptsas_scsi_watchdog_tick) {
9450      -                                ptgt->m_timebase +=
9451      -                                    mptsas_scsi_watchdog_tick;
9452      -                                continue;
9453      -                        }
     10620 +                cmd = TAILQ_LAST(&ptgt->m_active_cmdq, mptsas_active_cmdq);
     10621 +                if (cmd != NULL) {
     10622 +                        if (cmd->cmd_active_expiration <= timestamp) {
     10623 +                                /*
     10624 +                                 * Earliest command timeout expired.
     10625 +                                 * Drain throttle.
     10626 +                                 */
     10627 +                                mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
9454 10628  
9455      -                        ptgt->m_timeout -= mptsas_scsi_watchdog_tick;
     10629 +                                /*
     10630 +                                 * Check for remaining commands.
     10631 +                                 */
     10632 +                                cmd = TAILQ_FIRST(&ptgt->m_active_cmdq);
     10633 +                                if (cmd->cmd_active_expiration > timestamp) {
     10634 +                                        /*
     10635 +                                         * Wait for remaining commands to
     10636 +                                         * complete or time out.
     10637 +                                         */
     10638 +                                        NDBG23(("command timed out, "
     10639 +                                            "pending drain"));
     10640 +                                } else {
     10641 +                                        mutex_exit(&ptgt->m_t_mutex);
9456 10642  
9457      -                        if (ptgt->m_timeout < 0) {
9458      -                                mptsas_cmd_timeout(mpt, ptgt->m_devhdl);
9459      -                                continue;
9460      -                        }
     10643 +                                        /*
     10644 +                                         * All command timeouts expired.
     10645 +                                         */
     10646 +                                        mptsas_log(mpt, CE_NOTE,
     10647 +                                            "Timeout of %d seconds "
     10648 +                                            "expired with %d commands on "
     10649 +                                            "target %d lun %d.",
     10650 +                                            cmd->cmd_pkt->pkt_time,
     10651 +                                            ptgt->m_t_ncmds,
     10652 +                                            ptgt->m_devhdl, Lun(cmd));
9461 10653  
9462      -                        if ((ptgt->m_timeout) <=
9463      -                            mptsas_scsi_watchdog_tick) {
     10654 +                                        mptsas_cmd_timeout(mpt, ptgt);
     10655 +                                        continue;
     10656 +                                }
     10657 +                        } else if (cmd->cmd_active_expiration <= timestamp +
     10658 +                            (hrtime_t)mptsas_scsi_watchdog_tick * NANOSEC) {
9464 10659                                  NDBG23(("pending timeout"));
9465      -                                mptsas_set_throttle(mpt, ptgt,
9466      -                                    DRAIN_THROTTLE);
     10660 +                                mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
9467 10661                          }
9468 10662                  }
     10663 +                mutex_exit(&ptgt->m_t_mutex);
     10664 +        }
     10665 +        if (restart_hba == B_TRUE) {
     10666 +                mptsas_restart_hba(mpt);
9469 10667          }
9470 10668  }
9471 10669  
9472 10670  /*
9473 10671   * timeout recovery
9474 10672   */
9475 10673  static void
9476      -mptsas_cmd_timeout(mptsas_t *mpt, uint16_t devhdl)
     10674 +mptsas_cmd_timeout(mptsas_t *mpt, mptsas_target_t *ptgt)
9477 10675  {
     10676 +        uint16_t        devhdl;
     10677 +        uint64_t        sas_wwn;
     10678 +        uint8_t         phy;
     10679 +        char            wwn_str[MPTSAS_WWN_STRLEN];
     10680 +
     10681 +        devhdl = ptgt->m_devhdl;
     10682 +        sas_wwn = ptgt->m_addr.mta_wwn;
     10683 +        phy = ptgt->m_phynum;
     10684 +        if (sas_wwn == 0) {
     10685 +                (void) sprintf(wwn_str, "p%x", phy);
     10686 +        } else {
     10687 +                (void) sprintf(wwn_str, "w%016"PRIx64, sas_wwn);
     10688 +        }
9478 10689  
9479 10690          NDBG29(("mptsas_cmd_timeout: target=%d", devhdl));
9480 10691          mptsas_log(mpt, CE_WARN, "Disconnected command timeout for "
9481      -            "Target %d", devhdl);
     10692 +            "target %d %s,  enclosure %u .", devhdl, wwn_str,
     10693 +            ptgt->m_enclosure);
9482 10694  
9483 10695          /*
9484      -         * If the current target is not the target passed in,
9485      -         * try to reset that target.
     10696 +         * Abort all outstanding commands on the device.
9486 10697           */
9487 10698          NDBG29(("mptsas_cmd_timeout: device reset"));
9488 10699          if (mptsas_do_scsi_reset(mpt, devhdl) != TRUE) {
9489 10700                  mptsas_log(mpt, CE_WARN, "Target %d reset for command timeout "
9490 10701                      "recovery failed!", devhdl);
9491 10702          }
9492 10703  }
9493 10704  
9494 10705  /*
9495 10706   * Device / Hotplug control
↓ open down ↓ 28 lines elided ↑ open up ↑
9524 10735  mptsas_quiesce_bus(mptsas_t *mpt)
9525 10736  {
9526 10737          mptsas_target_t *ptgt = NULL;
9527 10738  
9528 10739          NDBG28(("mptsas_quiesce_bus"));
9529 10740          mutex_enter(&mpt->m_mutex);
9530 10741  
9531 10742          /* Set all the throttles to zero */
9532 10743          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
9533 10744              ptgt = refhash_next(mpt->m_targets, ptgt)) {
9534      -                mptsas_set_throttle(mpt, ptgt, HOLD_THROTTLE);
     10745 +                mptsas_set_throttle_mtx(mpt, ptgt, HOLD_THROTTLE);
9535 10746          }
9536 10747  
9537 10748          /* If there are any outstanding commands in the queue */
9538 10749          if (mpt->m_ncmds) {
9539 10750                  mpt->m_softstate |= MPTSAS_SS_DRAINING;
9540 10751                  mpt->m_quiesce_timeid = timeout(mptsas_ncmds_checkdrain,
9541 10752                      mpt, (MPTSAS_QUIESCE_TIMEOUT * drv_usectohz(1000000)));
9542 10753                  if (cv_wait_sig(&mpt->m_cv, &mpt->m_mutex) == 0) {
9543 10754                          /*
9544 10755                           * Quiesce has been interrupted
9545 10756                           */
9546 10757                          mpt->m_softstate &= ~MPTSAS_SS_DRAINING;
9547 10758                          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
9548 10759                              ptgt = refhash_next(mpt->m_targets, ptgt)) {
9549      -                                mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     10760 +                                mptsas_set_throttle_mtx(mpt, ptgt,
     10761 +                                    MAX_THROTTLE);
9550 10762                          }
9551 10763                          mptsas_restart_hba(mpt);
9552 10764                          if (mpt->m_quiesce_timeid != 0) {
9553 10765                                  timeout_id_t tid = mpt->m_quiesce_timeid;
9554 10766                                  mpt->m_quiesce_timeid = 0;
9555 10767                                  mutex_exit(&mpt->m_mutex);
9556 10768                                  (void) untimeout(tid);
9557 10769                                  return (-1);
9558 10770                          }
9559 10771                          mutex_exit(&mpt->m_mutex);
↓ open down ↓ 16 lines elided ↑ open up ↑
9576 10788  static int
9577 10789  mptsas_unquiesce_bus(mptsas_t *mpt)
9578 10790  {
9579 10791          mptsas_target_t *ptgt = NULL;
9580 10792  
9581 10793          NDBG28(("mptsas_unquiesce_bus"));
9582 10794          mutex_enter(&mpt->m_mutex);
9583 10795          mpt->m_softstate &= ~MPTSAS_SS_QUIESCED;
9584 10796          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
9585 10797              ptgt = refhash_next(mpt->m_targets, ptgt)) {
9586      -                mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     10798 +                mptsas_set_throttle_mtx(mpt, ptgt, MAX_THROTTLE);
9587 10799          }
9588 10800          mptsas_restart_hba(mpt);
9589 10801          mutex_exit(&mpt->m_mutex);
9590 10802          return (0);
9591 10803  }
9592 10804  
9593 10805  static void
9594 10806  mptsas_ncmds_checkdrain(void *arg)
9595 10807  {
9596 10808          mptsas_t        *mpt = arg;
↓ open down ↓ 5 lines elided ↑ open up ↑
9602 10814                  if (mpt->m_ncmds == 0) {
9603 10815                          /* Command queue has been drained */
9604 10816                          cv_signal(&mpt->m_cv);
9605 10817                  } else {
9606 10818                          /*
9607 10819                           * The throttle may have been reset because
9608 10820                           * of a SCSI bus reset
9609 10821                           */
9610 10822                          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
9611 10823                              ptgt = refhash_next(mpt->m_targets, ptgt)) {
9612      -                                mptsas_set_throttle(mpt, ptgt, HOLD_THROTTLE);
     10824 +                                mptsas_set_throttle_mtx(mpt, ptgt,
     10825 +                                    HOLD_THROTTLE);
9613 10826                          }
9614 10827  
9615 10828                          mpt->m_quiesce_timeid = timeout(mptsas_ncmds_checkdrain,
9616 10829                              mpt, (MPTSAS_QUIESCE_TIMEOUT *
9617 10830                              drv_usectohz(1000000)));
9618 10831                  }
9619 10832          }
9620 10833          mutex_exit(&mpt->m_mutex);
9621 10834  }
9622 10835  
↓ open down ↓ 15 lines elided ↑ open up ↑
9638 10851          (void) sprintf(&buf[strlen(buf)], " ]");
9639 10852          NDBG25(("?%s\n", buf));
9640 10853          NDBG25(("?pkt_flags=0x%x pkt_statistics=0x%x pkt_state=0x%x\n",
9641 10854              cmd->cmd_pkt->pkt_flags, cmd->cmd_pkt->pkt_statistics,
9642 10855              cmd->cmd_pkt->pkt_state));
9643 10856          NDBG25(("?pkt_scbp=0x%x cmd_flags=0x%x\n", cmd->cmd_pkt->pkt_scbp ?
9644 10857              *(cmd->cmd_pkt->pkt_scbp) : 0, cmd->cmd_flags));
9645 10858  }
9646 10859  
9647 10860  static void
     10861 +mptsas_passthru_sge(ddi_acc_handle_t acc_hdl, mptsas_pt_request_t *pt,
     10862 +    pMpi2SGESimple64_t sgep)
     10863 +{
     10864 +        uint32_t                sge_flags;
     10865 +        uint32_t                data_size, dataout_size;
     10866 +        ddi_dma_cookie_t        data_cookie;
     10867 +        ddi_dma_cookie_t        dataout_cookie;
     10868 +
     10869 +        data_size = pt->data_size;
     10870 +        dataout_size = pt->dataout_size;
     10871 +        data_cookie = pt->data_cookie;
     10872 +        dataout_cookie = pt->dataout_cookie;
     10873 +
     10874 +        if (dataout_size) {
     10875 +                sge_flags = dataout_size |
     10876 +                    ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
     10877 +                    MPI2_SGE_FLAGS_END_OF_BUFFER |
     10878 +                    MPI2_SGE_FLAGS_HOST_TO_IOC |
     10879 +                    MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
     10880 +                    MPI2_SGE_FLAGS_SHIFT);
     10881 +                ddi_put32(acc_hdl, &sgep->FlagsLength, sge_flags);
     10882 +                ddi_put32(acc_hdl, &sgep->Address.Low,
     10883 +                    (uint32_t)(dataout_cookie.dmac_laddress & 0xffffffffull));
     10884 +                ddi_put32(acc_hdl, &sgep->Address.High,
     10885 +                    (uint32_t)(dataout_cookie.dmac_laddress >> 32));
     10886 +                sgep++;
     10887 +        }
     10888 +        sge_flags = data_size;
     10889 +        sge_flags |= ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
     10890 +            MPI2_SGE_FLAGS_LAST_ELEMENT |
     10891 +            MPI2_SGE_FLAGS_END_OF_BUFFER |
     10892 +            MPI2_SGE_FLAGS_END_OF_LIST |
     10893 +            MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
     10894 +            MPI2_SGE_FLAGS_SHIFT);
     10895 +        if (pt->direction == MPTSAS_PASS_THRU_DIRECTION_WRITE) {
     10896 +                sge_flags |= ((uint32_t)(MPI2_SGE_FLAGS_HOST_TO_IOC) <<
     10897 +                    MPI2_SGE_FLAGS_SHIFT);
     10898 +        } else {
     10899 +                sge_flags |= ((uint32_t)(MPI2_SGE_FLAGS_IOC_TO_HOST) <<
     10900 +                    MPI2_SGE_FLAGS_SHIFT);
     10901 +        }
     10902 +        ddi_put32(acc_hdl, &sgep->FlagsLength, sge_flags);
     10903 +        ddi_put32(acc_hdl, &sgep->Address.Low,
     10904 +            (uint32_t)(data_cookie.dmac_laddress & 0xffffffffull));
     10905 +        ddi_put32(acc_hdl, &sgep->Address.High,
     10906 +            (uint32_t)(data_cookie.dmac_laddress >> 32));
     10907 +}
     10908 +
     10909 +static void
     10910 +mptsas_passthru_ieee_sge(ddi_acc_handle_t acc_hdl, mptsas_pt_request_t *pt,
     10911 +    pMpi2IeeeSgeSimple64_t ieeesgep)
     10912 +{
     10913 +        uint8_t                 sge_flags;
     10914 +        uint32_t                data_size, dataout_size;
     10915 +        ddi_dma_cookie_t        data_cookie;
     10916 +        ddi_dma_cookie_t        dataout_cookie;
     10917 +
     10918 +        data_size = pt->data_size;
     10919 +        dataout_size = pt->dataout_size;
     10920 +        data_cookie = pt->data_cookie;
     10921 +        dataout_cookie = pt->dataout_cookie;
     10922 +
     10923 +        sge_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
     10924 +            MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
     10925 +        if (dataout_size) {
     10926 +                ddi_put32(acc_hdl, &ieeesgep->Length, dataout_size);
     10927 +                ddi_put32(acc_hdl, &ieeesgep->Address.Low,
     10928 +                    (uint32_t)(dataout_cookie.dmac_laddress &
     10929 +                    0xffffffffull));
     10930 +                ddi_put32(acc_hdl, &ieeesgep->Address.High,
     10931 +                    (uint32_t)(dataout_cookie.dmac_laddress >> 32));
     10932 +                ddi_put8(acc_hdl, &ieeesgep->Flags, sge_flags);
     10933 +                ieeesgep++;
     10934 +        }
     10935 +        sge_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
     10936 +        ddi_put32(acc_hdl, &ieeesgep->Length, data_size);
     10937 +        ddi_put32(acc_hdl, &ieeesgep->Address.Low,
     10938 +            (uint32_t)(data_cookie.dmac_laddress & 0xffffffffull));
     10939 +        ddi_put32(acc_hdl, &ieeesgep->Address.High,
     10940 +            (uint32_t)(data_cookie.dmac_laddress >> 32));
     10941 +        ddi_put8(acc_hdl, &ieeesgep->Flags, sge_flags);
     10942 +}
     10943 +
     10944 +static void
9648 10945  mptsas_start_passthru(mptsas_t *mpt, mptsas_cmd_t *cmd)
9649 10946  {
9650 10947          caddr_t                 memp;
9651 10948          pMPI2RequestHeader_t    request_hdrp;
9652 10949          struct scsi_pkt         *pkt = cmd->cmd_pkt;
9653 10950          mptsas_pt_request_t     *pt = pkt->pkt_ha_private;
9654      -        uint32_t                request_size, data_size, dataout_size;
9655      -        uint32_t                direction;
9656      -        ddi_dma_cookie_t        data_cookie;
9657      -        ddi_dma_cookie_t        dataout_cookie;
9658      -        uint32_t                request_desc_low, request_desc_high = 0;
9659      -        uint32_t                i, sense_bufp;
     10951 +        uint32_t                request_size;
     10952 +        uint64_t                request_desc = 0;
     10953 +        uint64_t                sense_bufp;
9660 10954          uint8_t                 desc_type;
9661 10955          uint8_t                 *request, function;
9662 10956          ddi_dma_handle_t        dma_hdl = mpt->m_dma_req_frame_hdl;
9663 10957          ddi_acc_handle_t        acc_hdl = mpt->m_acc_req_frame_hdl;
9664 10958  
9665 10959          desc_type = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
9666 10960  
9667 10961          request = pt->request;
9668      -        direction = pt->direction;
9669 10962          request_size = pt->request_size;
9670      -        data_size = pt->data_size;
9671      -        dataout_size = pt->dataout_size;
9672      -        data_cookie = pt->data_cookie;
9673      -        dataout_cookie = pt->dataout_cookie;
9674 10963  
9675 10964          /*
9676 10965           * Store the passthrough message in memory location
9677 10966           * corresponding to our slot number
9678 10967           */
9679 10968          memp = mpt->m_req_frame + (mpt->m_req_frame_size * cmd->cmd_slot);
9680 10969          request_hdrp = (pMPI2RequestHeader_t)memp;
9681 10970          bzero(memp, mpt->m_req_frame_size);
9682 10971  
9683      -        for (i = 0; i < request_size; i++) {
9684      -                bcopy(request + i, memp + i, 1);
9685      -        }
     10972 +        bcopy(request, memp, request_size);
9686 10973  
9687      -        if (data_size || dataout_size) {
9688      -                pMpi2SGESimple64_t      sgep;
9689      -                uint32_t                sge_flags;
9690      -
9691      -                sgep = (pMpi2SGESimple64_t)((uint8_t *)request_hdrp +
9692      -                    request_size);
9693      -                if (dataout_size) {
9694      -
9695      -                        sge_flags = dataout_size |
9696      -                            ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
9697      -                            MPI2_SGE_FLAGS_END_OF_BUFFER |
9698      -                            MPI2_SGE_FLAGS_HOST_TO_IOC |
9699      -                            MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
9700      -                            MPI2_SGE_FLAGS_SHIFT);
9701      -                        ddi_put32(acc_hdl, &sgep->FlagsLength, sge_flags);
9702      -                        ddi_put32(acc_hdl, &sgep->Address.Low,
9703      -                            (uint32_t)(dataout_cookie.dmac_laddress &
9704      -                            0xffffffffull));
9705      -                        ddi_put32(acc_hdl, &sgep->Address.High,
9706      -                            (uint32_t)(dataout_cookie.dmac_laddress
9707      -                            >> 32));
9708      -                        sgep++;
9709      -                }
9710      -                sge_flags = data_size;
9711      -                sge_flags |= ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
9712      -                    MPI2_SGE_FLAGS_LAST_ELEMENT |
9713      -                    MPI2_SGE_FLAGS_END_OF_BUFFER |
9714      -                    MPI2_SGE_FLAGS_END_OF_LIST |
9715      -                    MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
9716      -                    MPI2_SGE_FLAGS_SHIFT);
9717      -                if (direction == MPTSAS_PASS_THRU_DIRECTION_WRITE) {
9718      -                        sge_flags |= ((uint32_t)(MPI2_SGE_FLAGS_HOST_TO_IOC) <<
9719      -                            MPI2_SGE_FLAGS_SHIFT);
9720      -                } else {
9721      -                        sge_flags |= ((uint32_t)(MPI2_SGE_FLAGS_IOC_TO_HOST) <<
9722      -                            MPI2_SGE_FLAGS_SHIFT);
9723      -                }
9724      -                ddi_put32(acc_hdl, &sgep->FlagsLength,
9725      -                    sge_flags);
9726      -                ddi_put32(acc_hdl, &sgep->Address.Low,
9727      -                    (uint32_t)(data_cookie.dmac_laddress &
9728      -                    0xffffffffull));
9729      -                ddi_put32(acc_hdl, &sgep->Address.High,
9730      -                    (uint32_t)(data_cookie.dmac_laddress >> 32));
     10974 +        NDBG15(("mptsas_start_passthru: Func 0x%x, MsgFlags 0x%x, "
     10975 +            "size=%d, in %d, out %d", request_hdrp->Function,
     10976 +            request_hdrp->MsgFlags, request_size,
     10977 +            pt->data_size, pt->dataout_size));
     10978 +
     10979 +        /*
     10980 +         * Add an SGE, even if the length is zero.
     10981 +         */
     10982 +        if (mpt->m_MPI25 && pt->simple == 0) {
     10983 +                mptsas_passthru_ieee_sge(acc_hdl, pt,
     10984 +                    (pMpi2IeeeSgeSimple64_t)
     10985 +                    ((uint8_t *)request_hdrp + pt->sgl_offset));
     10986 +        } else {
     10987 +                mptsas_passthru_sge(acc_hdl, pt,
     10988 +                    (pMpi2SGESimple64_t)
     10989 +                    ((uint8_t *)request_hdrp + pt->sgl_offset));
9731 10990          }
9732 10991  
9733 10992          function = request_hdrp->Function;
9734 10993          if ((function == MPI2_FUNCTION_SCSI_IO_REQUEST) ||
9735 10994              (function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH)) {
9736 10995                  pMpi2SCSIIORequest_t    scsi_io_req;
9737 10996  
     10997 +                NDBG15(("mptsas_start_passthru: Is SCSI IO Req"));
9738 10998                  scsi_io_req = (pMpi2SCSIIORequest_t)request_hdrp;
9739 10999                  /*
9740 11000                   * Put SGE for data and data_out buffer at the end of
9741 11001                   * scsi_io_request message header.(64 bytes in total)
9742 11002                   * Following above SGEs, the residual space will be
9743 11003                   * used by sense data.
9744 11004                   */
9745 11005                  ddi_put8(acc_hdl,
9746 11006                      &scsi_io_req->SenseBufferLength,
9747 11007                      (uint8_t)(request_size - 64));
9748 11008  
9749      -                sense_bufp = mpt->m_req_frame_dma_addr +
9750      -                    (mpt->m_req_frame_size * cmd->cmd_slot);
     11009 +                sense_bufp = (uint32_t)(mpt->m_req_frame_dma_addr +
     11010 +                    (mpt->m_req_frame_size * cmd->cmd_slot) & 0xffffffffull);
9751 11011                  sense_bufp += 64;
9752 11012                  ddi_put32(acc_hdl,
9753 11013                      &scsi_io_req->SenseBufferLowAddress, sense_bufp);
9754 11014  
9755 11015                  /*
9756 11016                   * Set SGLOffset0 value
9757 11017                   */
9758 11018                  ddi_put8(acc_hdl, &scsi_io_req->SGLOffset0,
9759 11019                      offsetof(MPI2_SCSI_IO_REQUEST, SGL) / 4);
9760 11020  
9761 11021                  /*
9762 11022                   * Setup descriptor info.  RAID passthrough must use the
9763 11023                   * default request descriptor which is already set, so if this
9764 11024                   * is a SCSI IO request, change the descriptor to SCSI IO.
9765 11025                   */
9766 11026                  if (function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
9767 11027                          desc_type = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
9768      -                        request_desc_high = (ddi_get16(acc_hdl,
9769      -                            &scsi_io_req->DevHandle) << 16);
     11028 +                        request_desc = (((uint64_t)ddi_get16(acc_hdl,
     11029 +                            &scsi_io_req->DevHandle)) << 48);
9770 11030                  }
9771 11031          }
9772 11032  
9773 11033          /*
9774 11034           * We must wait till the message has been completed before
9775 11035           * beginning the next message so we wait for this one to
9776 11036           * finish.
9777 11037           */
9778 11038          (void) ddi_dma_sync(dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
9779      -        request_desc_low = (cmd->cmd_slot << 16) + desc_type;
     11039 +        request_desc |= ((cmd->cmd_slot << 16) | desc_type);
9780 11040          cmd->cmd_rfm = NULL;
9781      -        MPTSAS_START_CMD(mpt, request_desc_low, request_desc_high);
     11041 +        MPTSAS_START_CMD(mpt, request_desc);
9782 11042          if ((mptsas_check_dma_handle(dma_hdl) != DDI_SUCCESS) ||
9783 11043              (mptsas_check_acc_handle(acc_hdl) != DDI_SUCCESS)) {
9784 11044                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
9785 11045          }
9786 11046  }
9787 11047  
     11048 +typedef void (mps_pre_f)(mptsas_t *, mptsas_pt_request_t *);
     11049 +static mps_pre_f        mpi_pre_ioc_facts;
     11050 +static mps_pre_f        mpi_pre_port_facts;
     11051 +static mps_pre_f        mpi_pre_fw_download;
     11052 +static mps_pre_f        mpi_pre_fw_25_download;
     11053 +static mps_pre_f        mpi_pre_fw_upload;
     11054 +static mps_pre_f        mpi_pre_fw_25_upload;
     11055 +static mps_pre_f        mpi_pre_sata_passthrough;
     11056 +static mps_pre_f        mpi_pre_smp_passthrough;
     11057 +static mps_pre_f        mpi_pre_config;
     11058 +static mps_pre_f        mpi_pre_sas_io_unit_control;
     11059 +static mps_pre_f        mpi_pre_scsi_io_req;
     11060 +
     11061 +/*
     11062 + * Prepare the pt for a SAS2 FW_DOWNLOAD request.
     11063 + */
     11064 +static void
     11065 +mpi_pre_fw_download(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11066 +{
     11067 +        pMpi2FWDownloadTCSGE_t tcsge;
     11068 +        pMpi2FWDownloadRequest req;
     11069 +
     11070 +        /*
     11071 +         * If SAS3, call separate function.
     11072 +         */
     11073 +        if (mpt->m_MPI25) {
     11074 +                mpi_pre_fw_25_download(mpt, pt);
     11075 +                return;
     11076 +        }
     11077 +
     11078 +        /*
     11079 +         * User requests should come in with the Transaction
     11080 +         * context element where the SGL will go. Putting the
     11081 +         * SGL after that seems to work, but don't really know
     11082 +         * why. Other drivers tend to create an extra SGL and
     11083 +         * refer to the TCE through that.
     11084 +         */
     11085 +        req = (pMpi2FWDownloadRequest)pt->request;
     11086 +        tcsge = (pMpi2FWDownloadTCSGE_t)&req->SGL;
     11087 +        if (tcsge->ContextSize != 0 || tcsge->DetailsLength != 12 ||
     11088 +            tcsge->Flags != MPI2_SGE_FLAGS_TRANSACTION_ELEMENT) {
     11089 +                mptsas_log(mpt, CE_WARN, "FW Download tce invalid!");
     11090 +        }
     11091 +
     11092 +        pt->sgl_offset = offsetof(MPI2_FW_DOWNLOAD_REQUEST, SGL) +
     11093 +            sizeof (*tcsge);
     11094 +        if (pt->request_size != pt->sgl_offset)
     11095 +                NDBG15(("mpi_pre_fw_download(): Incorrect req size, "
     11096 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11097 +                    (int)pt->request_size, (int)pt->sgl_offset,
     11098 +                    (int)pt->dataout_size));
     11099 +        if (pt->data_size < sizeof (MPI2_FW_DOWNLOAD_REPLY))
     11100 +                NDBG15(("mpi_pre_fw_download(): Incorrect rep size, "
     11101 +                    "0x%x, should be 0x%x", pt->data_size,
     11102 +                    (int)sizeof (MPI2_FW_DOWNLOAD_REPLY)));
     11103 +}
     11104 +
     11105 +/*
     11106 + * Prepare the pt for a SAS3 FW_DOWNLOAD request.
     11107 + */
     11108 +static void
     11109 +mpi_pre_fw_25_download(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11110 +{
     11111 +        pMpi2FWDownloadTCSGE_t tcsge;
     11112 +        pMpi2FWDownloadRequest req2;
     11113 +        pMpi25FWDownloadRequest req25;
     11114 +
     11115 +        /*
     11116 +         * User requests should come in with the Transaction
     11117 +         * context element where the SGL will go. The new firmware
     11118 +         * Doesn't use TCE and has space in the main request for
     11119 +         * this information. So move to the right place.
     11120 +         */
     11121 +        req2 = (pMpi2FWDownloadRequest)pt->request;
     11122 +        req25 = (pMpi25FWDownloadRequest)pt->request;
     11123 +        tcsge = (pMpi2FWDownloadTCSGE_t)&req2->SGL;
     11124 +        if (tcsge->ContextSize != 0 || tcsge->DetailsLength != 12 ||
     11125 +            tcsge->Flags != MPI2_SGE_FLAGS_TRANSACTION_ELEMENT) {
     11126 +                mptsas_log(mpt, CE_WARN, "FW Download tce invalid!");
     11127 +        }
     11128 +        req25->ImageOffset = tcsge->ImageOffset;
     11129 +        req25->ImageSize = tcsge->ImageSize;
     11130 +
     11131 +        pt->sgl_offset = offsetof(MPI25_FW_DOWNLOAD_REQUEST, SGL);
     11132 +        if (pt->request_size != pt->sgl_offset)
     11133 +                NDBG15(("mpi_pre_fw_25_download(): Incorrect req size, "
     11134 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11135 +                    pt->request_size, pt->sgl_offset,
     11136 +                    pt->dataout_size));
     11137 +        if (pt->data_size < sizeof (MPI2_FW_DOWNLOAD_REPLY))
     11138 +                NDBG15(("mpi_pre_fw_25_download(): Incorrect rep size, "
     11139 +                    "0x%x, should be 0x%x", pt->data_size,
     11140 +                    (int)sizeof (MPI2_FW_UPLOAD_REPLY)));
     11141 +}
     11142 +
     11143 +/*
     11144 + * Prepare the pt for a SAS2 FW_UPLOAD request.
     11145 + */
     11146 +static void
     11147 +mpi_pre_fw_upload(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11148 +{
     11149 +        pMpi2FWUploadTCSGE_t tcsge;
     11150 +        pMpi2FWUploadRequest_t req;
     11151 +
     11152 +        /*
     11153 +         * If SAS3, call separate function.
     11154 +         */
     11155 +        if (mpt->m_MPI25) {
     11156 +                mpi_pre_fw_25_upload(mpt, pt);
     11157 +                return;
     11158 +        }
     11159 +
     11160 +        /*
     11161 +         * User requests should come in with the Transaction
     11162 +         * context element where the SGL will go. Putting the
     11163 +         * SGL after that seems to work, but don't really know
     11164 +         * why. Other drivers tend to create an extra SGL and
     11165 +         * refer to the TCE through that.
     11166 +         */
     11167 +        req = (pMpi2FWUploadRequest_t)pt->request;
     11168 +        tcsge = (pMpi2FWUploadTCSGE_t)&req->SGL;
     11169 +        if (tcsge->ContextSize != 0 || tcsge->DetailsLength != 12 ||
     11170 +            tcsge->Flags != MPI2_SGE_FLAGS_TRANSACTION_ELEMENT) {
     11171 +                mptsas_log(mpt, CE_WARN, "FW Upload tce invalid!");
     11172 +        }
     11173 +
     11174 +        pt->sgl_offset = offsetof(MPI2_FW_UPLOAD_REQUEST, SGL) +
     11175 +            sizeof (*tcsge);
     11176 +        if (pt->request_size != pt->sgl_offset)
     11177 +                NDBG15(("mpi_pre_fw_upload(): Incorrect req size, "
     11178 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11179 +                    pt->request_size, pt->sgl_offset,
     11180 +                    pt->dataout_size));
     11181 +        if (pt->data_size < sizeof (MPI2_FW_UPLOAD_REPLY))
     11182 +                NDBG15(("mpi_pre_fw_upload(): Incorrect rep size, "
     11183 +                    "0x%x, should be 0x%x", pt->data_size,
     11184 +                    (int)sizeof (MPI2_FW_UPLOAD_REPLY)));
     11185 +}
     11186 +
     11187 +/*
     11188 + * Prepare the pt a SAS3 FW_UPLOAD request.
     11189 + */
     11190 +static void
     11191 +mpi_pre_fw_25_upload(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11192 +{
     11193 +        pMpi2FWUploadTCSGE_t tcsge;
     11194 +        pMpi2FWUploadRequest_t req2;
     11195 +        pMpi25FWUploadRequest_t req25;
     11196 +
     11197 +        /*
     11198 +         * User requests should come in with the Transaction
     11199 +         * context element where the SGL will go. The new firmware
     11200 +         * Doesn't use TCE and has space in the main request for
     11201 +         * this information. So move to the right place.
     11202 +         */
     11203 +        req2 = (pMpi2FWUploadRequest_t)pt->request;
     11204 +        req25 = (pMpi25FWUploadRequest_t)pt->request;
     11205 +        tcsge = (pMpi2FWUploadTCSGE_t)&req2->SGL;
     11206 +        if (tcsge->ContextSize != 0 || tcsge->DetailsLength != 12 ||
     11207 +            tcsge->Flags != MPI2_SGE_FLAGS_TRANSACTION_ELEMENT) {
     11208 +                mptsas_log(mpt, CE_WARN, "FW Upload tce invalid!");
     11209 +        }
     11210 +        req25->ImageOffset = tcsge->ImageOffset;
     11211 +        req25->ImageSize = tcsge->ImageSize;
     11212 +
     11213 +        pt->sgl_offset = offsetof(MPI25_FW_UPLOAD_REQUEST, SGL);
     11214 +        if (pt->request_size != pt->sgl_offset)
     11215 +                NDBG15(("mpi_pre_fw_25_upload(): Incorrect req size, "
     11216 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11217 +                    pt->request_size, pt->sgl_offset,
     11218 +                    pt->dataout_size));
     11219 +        if (pt->data_size < sizeof (MPI2_FW_UPLOAD_REPLY))
     11220 +                NDBG15(("mpi_pre_fw_25_upload(): Incorrect rep size, "
     11221 +                    "0x%x, should be 0x%x", pt->data_size,
     11222 +                    (int)sizeof (MPI2_FW_UPLOAD_REPLY)));
     11223 +}
     11224 +
     11225 +/*
     11226 + * Prepare the pt for an IOC_FACTS request.
     11227 + */
     11228 +static void
     11229 +mpi_pre_ioc_facts(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11230 +{
     11231 +#ifndef __lock_lint
     11232 +        _NOTE(ARGUNUSED(mpt))
     11233 +#endif
     11234 +        if (pt->request_size != sizeof (MPI2_IOC_FACTS_REQUEST))
     11235 +                NDBG15(("mpi_pre_ioc_facts(): Incorrect req size, "
     11236 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11237 +                    pt->request_size,
     11238 +                    (int)sizeof (MPI2_IOC_FACTS_REQUEST),
     11239 +                    pt->dataout_size));
     11240 +        if (pt->data_size != sizeof (MPI2_IOC_FACTS_REPLY))
     11241 +                NDBG15(("mpi_pre_ioc_facts(): Incorrect rep size, "
     11242 +                    "0x%x, should be 0x%x", pt->data_size,
     11243 +                    (int)sizeof (MPI2_IOC_FACTS_REPLY)));
     11244 +        pt->sgl_offset = (uint16_t)pt->request_size;
     11245 +}
     11246 +
     11247 +/*
     11248 + * Prepare the pt for a PORT_FACTS request.
     11249 + */
     11250 +static void
     11251 +mpi_pre_port_facts(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11252 +{
     11253 +#ifndef __lock_lint
     11254 +        _NOTE(ARGUNUSED(mpt))
     11255 +#endif
     11256 +        if (pt->request_size != sizeof (MPI2_PORT_FACTS_REQUEST))
     11257 +                NDBG15(("mpi_pre_port_facts(): Incorrect req size, "
     11258 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11259 +                    pt->request_size,
     11260 +                    (int)sizeof (MPI2_PORT_FACTS_REQUEST),
     11261 +                    pt->dataout_size));
     11262 +        if (pt->data_size != sizeof (MPI2_PORT_FACTS_REPLY))
     11263 +                NDBG15(("mpi_pre_port_facts(): Incorrect rep size, "
     11264 +                    "0x%x, should be 0x%x", pt->data_size,
     11265 +                    (int)sizeof (MPI2_PORT_FACTS_REPLY)));
     11266 +        pt->sgl_offset = (uint16_t)pt->request_size;
     11267 +}
     11268 +
     11269 +/*
     11270 + * Prepare pt for a SATA_PASSTHROUGH request.
     11271 + */
     11272 +static void
     11273 +mpi_pre_sata_passthrough(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11274 +{
     11275 +#ifndef __lock_lint
     11276 +        _NOTE(ARGUNUSED(mpt))
     11277 +#endif
     11278 +        pt->sgl_offset = offsetof(MPI2_SATA_PASSTHROUGH_REQUEST, SGL);
     11279 +        if (pt->request_size != pt->sgl_offset)
     11280 +                NDBG15(("mpi_pre_sata_passthrough(): Incorrect req size, "
     11281 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11282 +                    pt->request_size, pt->sgl_offset,
     11283 +                    pt->dataout_size));
     11284 +        if (pt->data_size != sizeof (MPI2_SATA_PASSTHROUGH_REPLY))
     11285 +                NDBG15(("mpi_pre_sata_passthrough(): Incorrect rep size, "
     11286 +                    "0x%x, should be 0x%x", pt->data_size,
     11287 +                    (int)sizeof (MPI2_SATA_PASSTHROUGH_REPLY)));
     11288 +}
     11289 +
     11290 +static void
     11291 +mpi_pre_smp_passthrough(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11292 +{
     11293 +#ifndef __lock_lint
     11294 +        _NOTE(ARGUNUSED(mpt))
     11295 +#endif
     11296 +        pt->sgl_offset = offsetof(MPI2_SMP_PASSTHROUGH_REQUEST, SGL);
     11297 +        if (pt->request_size != pt->sgl_offset)
     11298 +                NDBG15(("mpi_pre_smp_passthrough(): Incorrect req size, "
     11299 +                    "0x%x, should be 0x%x, dataoutsz 0x%x",
     11300 +                    pt->request_size, pt->sgl_offset,
     11301 +                    pt->dataout_size));
     11302 +        if (pt->data_size != sizeof (MPI2_SMP_PASSTHROUGH_REPLY))
     11303 +                NDBG15(("mpi_pre_smp_passthrough(): Incorrect rep size, "
     11304 +                    "0x%x, should be 0x%x", pt->data_size,
     11305 +                    (int)sizeof (MPI2_SMP_PASSTHROUGH_REPLY)));
     11306 +}
     11307 +
     11308 +/*
     11309 + * Prepare pt for a CONFIG request.
     11310 + */
     11311 +static void
     11312 +mpi_pre_config(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11313 +{
     11314 +#ifndef __lock_lint
     11315 +        _NOTE(ARGUNUSED(mpt))
     11316 +#endif
     11317 +        pt->sgl_offset = offsetof(MPI2_CONFIG_REQUEST, PageBufferSGE);
     11318 +        if (pt->request_size != pt->sgl_offset)
     11319 +                NDBG15(("mpi_pre_config(): Incorrect req size, 0x%x, "
     11320 +                    "should be 0x%x, dataoutsz 0x%x", pt->request_size,
     11321 +                    pt->sgl_offset, pt->dataout_size));
     11322 +        if (pt->data_size != sizeof (MPI2_CONFIG_REPLY))
     11323 +                NDBG15(("mpi_pre_config(): Incorrect rep size, 0x%x, "
     11324 +                    "should be 0x%x", pt->data_size,
     11325 +                    (int)sizeof (MPI2_CONFIG_REPLY)));
     11326 +        pt->simple = 1;
     11327 +}
     11328 +
     11329 +/*
     11330 + * Prepare pt for a SCSI_IO_REQ request.
     11331 + */
     11332 +static void
     11333 +mpi_pre_scsi_io_req(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11334 +{
     11335 +#ifndef __lock_lint
     11336 +        _NOTE(ARGUNUSED(mpt))
     11337 +#endif
     11338 +        pt->sgl_offset = offsetof(MPI2_SCSI_IO_REQUEST, SGL);
     11339 +        if (pt->request_size != pt->sgl_offset)
     11340 +                NDBG15(("mpi_pre_config(): Incorrect req size, 0x%x, "
     11341 +                    "should be 0x%x, dataoutsz 0x%x", pt->request_size,
     11342 +                    pt->sgl_offset,
     11343 +                    pt->dataout_size));
     11344 +        if (pt->data_size != sizeof (MPI2_SCSI_IO_REPLY))
     11345 +                NDBG15(("mpi_pre_config(): Incorrect rep size, 0x%x, "
     11346 +                    "should be 0x%x", pt->data_size,
     11347 +                    (int)sizeof (MPI2_SCSI_IO_REPLY)));
     11348 +}
     11349 +
     11350 +/*
     11351 + * Prepare the mps_command for a SAS_IO_UNIT_CONTROL request.
     11352 + */
     11353 +static void
     11354 +mpi_pre_sas_io_unit_control(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11355 +{
     11356 +#ifndef __lock_lint
     11357 +        _NOTE(ARGUNUSED(mpt))
     11358 +#endif
     11359 +        pt->sgl_offset = (uint16_t)pt->request_size;
     11360 +}
     11361 +
     11362 +/*
     11363 + * A set of functions to prepare an mps_command for the various
     11364 + * supported requests.
     11365 + */
     11366 +struct mps_func {
     11367 +        U8              Function;
     11368 +        char            *Name;
     11369 +        mps_pre_f       *f_pre;
     11370 +} mps_func_list[] = {
     11371 +        { MPI2_FUNCTION_IOC_FACTS, "IOC_FACTS",         mpi_pre_ioc_facts },
     11372 +        { MPI2_FUNCTION_PORT_FACTS, "PORT_FACTS",       mpi_pre_port_facts },
     11373 +        { MPI2_FUNCTION_FW_DOWNLOAD, "FW_DOWNLOAD",     mpi_pre_fw_download },
     11374 +        { MPI2_FUNCTION_FW_UPLOAD, "FW_UPLOAD",         mpi_pre_fw_upload },
     11375 +        { MPI2_FUNCTION_SATA_PASSTHROUGH, "SATA_PASSTHROUGH",
     11376 +            mpi_pre_sata_passthrough },
     11377 +        { MPI2_FUNCTION_SMP_PASSTHROUGH, "SMP_PASSTHROUGH",
     11378 +            mpi_pre_smp_passthrough},
     11379 +        { MPI2_FUNCTION_SCSI_IO_REQUEST, "SCSI_IO_REQUEST",
     11380 +            mpi_pre_scsi_io_req},
     11381 +        { MPI2_FUNCTION_CONFIG, "CONFIG",               mpi_pre_config},
     11382 +        { MPI2_FUNCTION_SAS_IO_UNIT_CONTROL, "SAS_IO_UNIT_CONTROL",
     11383 +            mpi_pre_sas_io_unit_control },
     11384 +        { 0xFF, NULL,                           NULL } /* list end */
     11385 +};
     11386 +
     11387 +static void
     11388 +mptsas_prep_sgl_offset(mptsas_t *mpt, mptsas_pt_request_t *pt)
     11389 +{
     11390 +        pMPI2RequestHeader_t    hdr;
     11391 +        struct mps_func         *f;
     11392 +
     11393 +        hdr = (pMPI2RequestHeader_t)pt->request;
     11394 +
     11395 +        for (f = mps_func_list; f->f_pre != NULL; f++) {
     11396 +                if (hdr->Function == f->Function) {
     11397 +                        f->f_pre(mpt, pt);
     11398 +                        NDBG15(("mptsas_prep_sgl_offset: Function %s,"
     11399 +                            " sgl_offset 0x%x", f->Name,
     11400 +                            pt->sgl_offset));
     11401 +                        return;
     11402 +                }
     11403 +        }
     11404 +        NDBG15(("mptsas_prep_sgl_offset: Unknown Function 0x%02x,"
     11405 +            " returning req_size 0x%x for sgl_offset",
     11406 +            hdr->Function, pt->request_size));
     11407 +        pt->sgl_offset = (uint16_t)pt->request_size;
     11408 +}
9788 11409  
9789 11410  
9790 11411  static int
9791 11412  mptsas_do_passthru(mptsas_t *mpt, uint8_t *request, uint8_t *reply,
9792 11413      uint8_t *data, uint32_t request_size, uint32_t reply_size,
9793      -    uint32_t data_size, uint32_t direction, uint8_t *dataout,
     11414 +    uint32_t data_size, uint8_t direction, uint8_t *dataout,
9794 11415      uint32_t dataout_size, short timeout, int mode)
9795 11416  {
9796 11417          mptsas_pt_request_t             pt;
9797 11418          mptsas_dma_alloc_state_t        data_dma_state;
9798 11419          mptsas_dma_alloc_state_t        dataout_dma_state;
9799 11420          caddr_t                         memp;
9800 11421          mptsas_cmd_t                    *cmd = NULL;
9801 11422          struct scsi_pkt                 *pkt;
9802 11423          uint32_t                        reply_len = 0, sense_len = 0;
9803 11424          pMPI2RequestHeader_t            request_hdrp;
↓ open down ↓ 56 lines elided ↑ open up ↑
9860 11481                                          mutex_enter(&mpt->m_mutex);
9861 11482                                          status = EFAULT;
9862 11483                                          mptsas_log(mpt, CE_WARN, "failed to "
9863 11484                                              "copy read data");
9864 11485                                          goto out;
9865 11486                                  }
9866 11487                          }
9867 11488                          mutex_enter(&mpt->m_mutex);
9868 11489                  }
9869 11490          }
     11491 +        else
     11492 +                bzero(&data_dma_state, sizeof (data_dma_state));
9870 11493  
9871 11494          if (dataout_size != 0) {
9872 11495                  dataout_dma_state.size = dataout_size;
9873 11496                  if (mptsas_dma_alloc(mpt, &dataout_dma_state) != DDI_SUCCESS) {
9874 11497                          status = ENOMEM;
9875 11498                          mptsas_log(mpt, CE_WARN, "failed to alloc DMA "
9876 11499                              "resource");
9877 11500                          goto out;
9878 11501                  }
9879 11502                  pt_flags |= MPTSAS_DATAOUT_ALLOCATED;
↓ open down ↓ 3 lines elided ↑ open up ↑
9883 11506                              dataout_dma_state.memp + i, 1, mode)) {
9884 11507                                  mutex_enter(&mpt->m_mutex);
9885 11508                                  mptsas_log(mpt, CE_WARN, "failed to copy out"
9886 11509                                      " data");
9887 11510                                  status = EFAULT;
9888 11511                                  goto out;
9889 11512                          }
9890 11513                  }
9891 11514                  mutex_enter(&mpt->m_mutex);
9892 11515          }
     11516 +        else
     11517 +                bzero(&dataout_dma_state, sizeof (dataout_dma_state));
9893 11518  
9894 11519          if ((rvalue = (mptsas_request_from_pool(mpt, &cmd, &pkt))) == -1) {
9895 11520                  status = EAGAIN;
9896 11521                  mptsas_log(mpt, CE_NOTE, "event ack command pool is full");
9897 11522                  goto out;
9898 11523          }
9899 11524          pt_flags |= MPTSAS_REQUEST_POOL_CMD;
9900 11525  
9901 11526          bzero((caddr_t)cmd, sizeof (*cmd));
9902 11527          bzero((caddr_t)pkt, scsi_pkt_size());
9903 11528          bzero((caddr_t)&pt, sizeof (pt));
9904 11529  
9905 11530          cmd->ioc_cmd_slot = (uint32_t)(rvalue);
9906 11531  
9907 11532          pt.request = (uint8_t *)request_msg;
9908 11533          pt.direction = direction;
     11534 +        pt.simple = 0;
9909 11535          pt.request_size = request_size;
9910 11536          pt.data_size = data_size;
9911 11537          pt.dataout_size = dataout_size;
9912 11538          pt.data_cookie = data_dma_state.cookie;
9913 11539          pt.dataout_cookie = dataout_dma_state.cookie;
     11540 +        mptsas_prep_sgl_offset(mpt, &pt);
9914 11541  
9915 11542          /*
9916 11543           * Form a blank cmd/pkt to store the acknowledgement message
9917 11544           */
9918 11545          pkt->pkt_cdbp           = (opaque_t)&cmd->cmd_cdb[0];
9919 11546          pkt->pkt_scbp           = (opaque_t)&cmd->cmd_scb;
9920 11547          pkt->pkt_ha_private     = (opaque_t)&pt;
9921 11548          pkt->pkt_flags          = FLAG_HEAD;
9922 11549          pkt->pkt_time           = timeout;
9923 11550          cmd->cmd_pkt            = pkt;
↓ open down ↓ 36 lines elided ↑ open up ↑
9960 11587                   * reply and no PCI Write to post the free reply SMFA
9961 11588                   * because no reply message frame is used.
9962 11589                   * cmd_rfm is non-zero means the reply is a ADDRESS
9963 11590                   * reply and reply message frame is used.
9964 11591                   */
9965 11592                  pt_flags |= MPTSAS_ADDRESS_REPLY;
9966 11593                  (void) ddi_dma_sync(mpt->m_dma_reply_frame_hdl, 0, 0,
9967 11594                      DDI_DMA_SYNC_FORCPU);
9968 11595                  reply_msg = (pMPI2DefaultReply_t)
9969 11596                      (mpt->m_reply_frame + (cmd->cmd_rfm -
9970      -                    mpt->m_reply_frame_dma_addr));
     11597 +                    (mpt->m_reply_frame_dma_addr&0xfffffffful)));
9971 11598          }
9972 11599  
9973 11600          mptsas_fma_check(mpt, cmd);
9974 11601          if (pkt->pkt_reason == CMD_TRAN_ERR) {
9975 11602                  status = EAGAIN;
9976 11603                  mptsas_log(mpt, CE_WARN, "passthru fma error");
9977 11604                  goto out;
9978 11605          }
9979 11606          if (pkt->pkt_reason == CMD_RESET) {
9980 11607                  status = EAGAIN;
↓ open down ↓ 135 lines elided ↑ open up ↑
10116 11743                          data->DataOutSize = 0;
10117 11744                  }
10118 11745                  /*
10119 11746                   * Send passthru request messages
10120 11747                   */
10121 11748                  return (mptsas_do_passthru(mpt,
10122 11749                      (uint8_t *)((uintptr_t)data->PtrRequest),
10123 11750                      (uint8_t *)((uintptr_t)data->PtrReply),
10124 11751                      (uint8_t *)((uintptr_t)data->PtrData),
10125 11752                      data->RequestSize, data->ReplySize,
10126      -                    data->DataSize, data->DataDirection,
     11753 +                    data->DataSize, (uint8_t)data->DataDirection,
10127 11754                      (uint8_t *)((uintptr_t)data->PtrDataOut),
10128 11755                      data->DataOutSize, data->Timeout, mode));
10129 11756          } else {
10130 11757                  return (EINVAL);
10131 11758          }
10132 11759  }
10133 11760  
10134 11761  static uint8_t
10135 11762  mptsas_get_fw_diag_buffer_number(mptsas_t *mpt, uint32_t unique_id)
10136 11763  {
↓ open down ↓ 8 lines elided ↑ open up ↑
10145 11772          return (MPTSAS_FW_DIAGNOSTIC_UID_NOT_FOUND);
10146 11773  }
10147 11774  
10148 11775  static void
10149 11776  mptsas_start_diag(mptsas_t *mpt, mptsas_cmd_t *cmd)
10150 11777  {
10151 11778          pMpi2DiagBufferPostRequest_t    pDiag_post_msg;
10152 11779          pMpi2DiagReleaseRequest_t       pDiag_release_msg;
10153 11780          struct scsi_pkt                 *pkt = cmd->cmd_pkt;
10154 11781          mptsas_diag_request_t           *diag = pkt->pkt_ha_private;
10155      -        uint32_t                        request_desc_low, i;
     11782 +        uint32_t                        i;
     11783 +        uint64_t                        request_desc;
10156 11784  
10157 11785          ASSERT(mutex_owned(&mpt->m_mutex));
10158 11786  
10159 11787          /*
10160 11788           * Form the diag message depending on the post or release function.
10161 11789           */
10162 11790          if (diag->function == MPI2_FUNCTION_DIAG_BUFFER_POST) {
10163 11791                  pDiag_post_msg = (pMpi2DiagBufferPostRequest_t)
10164 11792                      (mpt->m_req_frame + (mpt->m_req_frame_size *
10165 11793                      cmd->cmd_slot));
↓ open down ↓ 32 lines elided ↑ open up ↑
10198 11826                  ddi_put8(mpt->m_acc_req_frame_hdl,
10199 11827                      &pDiag_release_msg->BufferType,
10200 11828                      diag->pBuffer->buffer_type);
10201 11829          }
10202 11830  
10203 11831          /*
10204 11832           * Send the message
10205 11833           */
10206 11834          (void) ddi_dma_sync(mpt->m_dma_req_frame_hdl, 0, 0,
10207 11835              DDI_DMA_SYNC_FORDEV);
10208      -        request_desc_low = (cmd->cmd_slot << 16) +
     11836 +        request_desc = (cmd->cmd_slot << 16) |
10209 11837              MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
10210 11838          cmd->cmd_rfm = NULL;
10211      -        MPTSAS_START_CMD(mpt, request_desc_low, 0);
     11839 +        MPTSAS_START_CMD(mpt, request_desc);
10212 11840          if ((mptsas_check_dma_handle(mpt->m_dma_req_frame_hdl) !=
10213 11841              DDI_SUCCESS) ||
10214 11842              (mptsas_check_acc_handle(mpt->m_acc_req_frame_hdl) !=
10215 11843              DDI_SUCCESS)) {
10216 11844                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
10217 11845          }
10218 11846  }
10219 11847  
10220 11848  static int
10221 11849  mptsas_post_fw_diag_buffer(mptsas_t *mpt,
↓ open down ↓ 77 lines elided ↑ open up ↑
10299 11927          /*
10300 11928           * cmd_rfm points to the reply message if a reply was given.  Check the
10301 11929           * IOCStatus to make sure everything went OK with the FW diag request
10302 11930           * and set buffer flags.
10303 11931           */
10304 11932          if (cmd->cmd_rfm) {
10305 11933                  post_flags |= MPTSAS_ADDRESS_REPLY;
10306 11934                  (void) ddi_dma_sync(mpt->m_dma_reply_frame_hdl, 0, 0,
10307 11935                      DDI_DMA_SYNC_FORCPU);
10308 11936                  reply = (pMpi2DiagBufferPostReply_t)(mpt->m_reply_frame +
10309      -                    (cmd->cmd_rfm - mpt->m_reply_frame_dma_addr));
     11937 +                    (cmd->cmd_rfm -
     11938 +                    (mpt->m_reply_frame_dma_addr&0xfffffffful)));
10310 11939  
10311 11940                  /*
10312 11941                   * Get the reply message data
10313 11942                   */
10314 11943                  iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
10315 11944                      &reply->IOCStatus);
10316 11945                  iocloginfo = ddi_get32(mpt->m_acc_reply_frame_hdl,
10317 11946                      &reply->IOCLogInfo);
10318 11947                  transfer_length = ddi_get32(mpt->m_acc_reply_frame_hdl,
10319 11948                      &reply->TransferLength);
↓ open down ↓ 133 lines elided ↑ open up ↑
10453 12082          /*
10454 12083           * cmd_rfm points to the reply message if a reply was given.  Check the
10455 12084           * IOCStatus to make sure everything went OK with the FW diag request
10456 12085           * and set buffer flags.
10457 12086           */
10458 12087          if (cmd->cmd_rfm) {
10459 12088                  rel_flags |= MPTSAS_ADDRESS_REPLY;
10460 12089                  (void) ddi_dma_sync(mpt->m_dma_reply_frame_hdl, 0, 0,
10461 12090                      DDI_DMA_SYNC_FORCPU);
10462 12091                  reply = (pMpi2DiagReleaseReply_t)(mpt->m_reply_frame +
10463      -                    (cmd->cmd_rfm - mpt->m_reply_frame_dma_addr));
     12092 +                    (cmd->cmd_rfm -
     12093 +                    (mpt->m_reply_frame_dma_addr&0xfffffffful)));
10464 12094  
10465 12095                  /*
10466 12096                   * Get the reply message data
10467 12097                   */
10468 12098                  iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
10469 12099                      &reply->IOCStatus);
10470 12100                  iocloginfo = ddi_get32(mpt->m_acc_reply_frame_hdl,
10471 12101                      &reply->IOCLogInfo);
10472 12102  
10473 12103                  /*
↓ open down ↓ 727 lines elided ↑ open up ↑
11201 12831           */
11202 12832          adapter_data->MpiFirmwareVersion = mpt->m_fwversion;
11203 12833  }
11204 12834  
11205 12835  static void
11206 12836  mptsas_read_adapter_data(mptsas_t *mpt, mptsas_adapter_data_t *adapter_data)
11207 12837  {
11208 12838          char    *driver_verstr = MPTSAS_MOD_STRING;
11209 12839  
11210 12840          mptsas_lookup_pci_data(mpt, adapter_data);
11211      -        adapter_data->AdapterType = MPTIOCTL_ADAPTER_TYPE_SAS2;
     12841 +        adapter_data->AdapterType = MPTIOCTL_ADAPTER_TYPE_SAS3;
11212 12842          adapter_data->PCIDeviceHwId = (uint32_t)mpt->m_devid;
11213 12843          adapter_data->PCIDeviceHwRev = (uint32_t)mpt->m_revid;
11214 12844          adapter_data->SubSystemId = (uint32_t)mpt->m_ssid;
11215 12845          adapter_data->SubsystemVendorId = (uint32_t)mpt->m_svid;
11216 12846          (void) strcpy((char *)&adapter_data->DriverVersion[0], driver_verstr);
11217 12847          adapter_data->BiosVersion = 0;
11218 12848          (void) mptsas_get_bios_page3(mpt, &adapter_data->BiosVersion);
11219 12849  }
11220 12850  
11221 12851  static void
↓ open down ↓ 253 lines elided ↑ open up ↑
11475 13105          mptsas_phymask_t        phymask = 0;
11476 13106          struct devctl_iocdata   *dcp = NULL;
11477 13107          char                    *addr = NULL;
11478 13108          mptsas_target_t         *ptgt = NULL;
11479 13109  
11480 13110          *rval = MPTIOCTL_STATUS_GOOD;
11481 13111          if (secpolicy_sys_config(credp, B_FALSE) != 0) {
11482 13112                  return (EPERM);
11483 13113          }
11484 13114  
11485      -        mpt = ddi_get_soft_state(mptsas_state, MINOR2INST(getminor(dev)));
     13115 +        mpt = ddi_get_soft_state(mptsas3_state, MINOR2INST(getminor(dev)));
11486 13116          if (mpt == NULL) {
11487 13117                  /*
11488 13118                   * Called from iport node, get the states
11489 13119                   */
11490 13120                  iport_flag = 1;
11491 13121                  dip = mptsas_get_dip_from_dev(dev, &phymask);
11492 13122                  if (dip == NULL) {
11493 13123                          return (ENXIO);
11494 13124                  }
11495 13125                  mpt = DIP2MPT(dip);
11496 13126          }
11497 13127          /* Make sure power level is D0 before accessing registers */
11498 13128          mutex_enter(&mpt->m_mutex);
11499 13129          if (mpt->m_options & MPTSAS_OPT_PM) {
11500 13130                  (void) pm_busy_component(mpt->m_dip, 0);
11501 13131                  if (mpt->m_power_level != PM_LEVEL_D0) {
11502 13132                          mutex_exit(&mpt->m_mutex);
11503 13133                          if (pm_raise_power(mpt->m_dip, 0, PM_LEVEL_D0) !=
11504 13134                              DDI_SUCCESS) {
11505 13135                                  mptsas_log(mpt, CE_WARN,
11506      -                                    "mptsas%d: mptsas_ioctl: Raise power "
     13136 +                                    "mptsas3%d: mptsas_ioctl: Raise power "
11507 13137                                      "request failed.", mpt->m_instance);
11508 13138                                  (void) pm_idle_component(mpt->m_dip, 0);
11509 13139                                  return (ENXIO);
11510 13140                          }
11511 13141                  } else {
11512 13142                          mutex_exit(&mpt->m_mutex);
11513 13143                  }
11514 13144          } else {
11515 13145                  mutex_exit(&mpt->m_mutex);
11516 13146          }
↓ open down ↓ 219 lines elided ↑ open up ↑
11736 13366           * rebuilt, I/O errors will occur.  This flag allows I/Os to be marked
11737 13367           * so that they can be retried.
11738 13368           */
11739 13369          mpt->m_in_reset = TRUE;
11740 13370  
11741 13371          /*
11742 13372           * Set all throttles to HOLD
11743 13373           */
11744 13374          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
11745 13375              ptgt = refhash_next(mpt->m_targets, ptgt)) {
11746      -                mptsas_set_throttle(mpt, ptgt, HOLD_THROTTLE);
     13376 +                mptsas_set_throttle_mtx(mpt, ptgt, HOLD_THROTTLE);
11747 13377          }
11748 13378  
11749 13379          /*
11750 13380           * Disable interrupts
11751 13381           */
11752 13382          MPTSAS_DISABLE_INTR(mpt);
11753 13383  
11754 13384          /*
11755 13385           * Abort all commands: outstanding commands, commands in waitq and
11756 13386           * tx_waitq.
↓ open down ↓ 17 lines elided ↑ open up ↑
11774 13404           */
11775 13405          if (rval == DDI_SUCCESS) {
11776 13406                  mptsas_update_driver_data(mpt);
11777 13407          }
11778 13408  
11779 13409          /*
11780 13410           * Reset the throttles
11781 13411           */
11782 13412          for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
11783 13413              ptgt = refhash_next(mpt->m_targets, ptgt)) {
11784      -                mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
     13414 +                mptsas_set_throttle_mtx(mpt, ptgt, MAX_THROTTLE);
11785 13415          }
11786 13416  
11787 13417          mptsas_doneq_empty(mpt);
11788 13418          mptsas_restart_hba(mpt);
11789 13419  
11790 13420          if (rval != DDI_SUCCESS) {
11791 13421                  mptsas_fm_ereport(mpt, DDI_FM_DEVICE_NO_RESPONSE);
11792 13422                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_LOST);
11793 13423          }
11794 13424  
↓ open down ↓ 2 lines elided ↑ open up ↑
11797 13427           */
11798 13428          mpt->m_in_reset = FALSE;
11799 13429  
11800 13430          return (rval);
11801 13431  }
11802 13432  
11803 13433  static int
11804 13434  mptsas_init_chip(mptsas_t *mpt, int first_time)
11805 13435  {
11806 13436          ddi_dma_cookie_t        cookie;
11807      -        uint32_t                i;
     13437 +        mptsas_reply_pqueue_t   *rpqp;
     13438 +        uint32_t                i, j;
11808 13439          int                     rval;
11809 13440  
11810 13441          /*
11811 13442           * Check to see if the firmware image is valid
11812 13443           */
11813 13444          if (ddi_get32(mpt->m_datap, &mpt->m_reg->HostDiagnostic) &
11814 13445              MPI2_DIAG_FLASH_BAD_SIG) {
11815 13446                  mptsas_log(mpt, CE_WARN, "mptsas bad flash signature!");
11816 13447                  goto fail;
11817 13448          }
↓ open down ↓ 22 lines elided ↑ open up ↑
11840 13471          /*
11841 13472           * IOC facts can change after a diag reset so all buffers that are
11842 13473           * based on these numbers must be de-allocated and re-allocated.  Get
11843 13474           * new IOC facts each time chip is initialized.
11844 13475           */
11845 13476          if (mptsas_ioc_get_facts(mpt) == DDI_FAILURE) {
11846 13477                  mptsas_log(mpt, CE_WARN, "mptsas_ioc_get_facts failed");
11847 13478                  goto fail;
11848 13479          }
11849 13480  
     13481 +        /*
     13482 +         * Now we know chip MSIX capabilitites and it's not been done
     13483 +         * previously register interrupts accordingly. Need to know this
     13484 +         * information before allocating the reply frames below.
     13485 +         */
     13486 +        if (mpt->m_intr_cnt == 0) {
     13487 +                if (mptsas_register_intrs(mpt) == FALSE)
     13488 +                        goto fail;
     13489 +        }
     13490 +
11850 13491          mpt->m_targets = refhash_create(MPTSAS_TARGET_BUCKET_COUNT,
11851 13492              mptsas_target_addr_hash, mptsas_target_addr_cmp,
11852 13493              mptsas_target_free, sizeof (mptsas_target_t),
11853 13494              offsetof(mptsas_target_t, m_link),
11854 13495              offsetof(mptsas_target_t, m_addr), KM_SLEEP);
11855 13496  
11856 13497          if (mptsas_alloc_active_slots(mpt, KM_SLEEP)) {
11857 13498                  goto fail;
11858 13499          }
11859 13500          /*
11860 13501           * Allocate request message frames, reply free queue, reply descriptor
11861 13502           * post queue, and reply message frames using latest IOC facts.
11862 13503           */
11863 13504          if (mptsas_alloc_request_frames(mpt) == DDI_FAILURE) {
11864 13505                  mptsas_log(mpt, CE_WARN, "mptsas_alloc_request_frames failed");
11865 13506                  goto fail;
11866 13507          }
     13508 +        if (mptsas_alloc_sense_bufs(mpt) == DDI_FAILURE) {
     13509 +                mptsas_log(mpt, CE_WARN, "mptsas_alloc_sense_bufs failed");
     13510 +                goto fail;
     13511 +        }
11867 13512          if (mptsas_alloc_free_queue(mpt) == DDI_FAILURE) {
11868 13513                  mptsas_log(mpt, CE_WARN, "mptsas_alloc_free_queue failed!");
11869 13514                  goto fail;
11870 13515          }
11871 13516          if (mptsas_alloc_post_queue(mpt) == DDI_FAILURE) {
11872 13517                  mptsas_log(mpt, CE_WARN, "mptsas_alloc_post_queue failed!");
11873 13518                  goto fail;
11874 13519          }
11875 13520          if (mptsas_alloc_reply_frames(mpt) == DDI_FAILURE) {
11876 13521                  mptsas_log(mpt, CE_WARN, "mptsas_alloc_reply_frames failed!");
↓ open down ↓ 5 lines elided ↑ open up ↑
11882 13527           * Re-Initialize ioc to operational state
11883 13528           */
11884 13529          if (mptsas_ioc_init(mpt) == DDI_FAILURE) {
11885 13530                  mptsas_log(mpt, CE_WARN, "mptsas_ioc_init failed");
11886 13531                  goto fail;
11887 13532          }
11888 13533  
11889 13534          mptsas_alloc_reply_args(mpt);
11890 13535  
11891 13536          /*
11892      -         * Initialize reply post index.  Reply free index is initialized after
11893      -         * the next loop.
11894      -         */
11895      -        mpt->m_post_index = 0;
11896      -
11897      -        /*
11898 13537           * Initialize the Reply Free Queue with the physical addresses of our
11899 13538           * reply frames.
11900 13539           */
11901      -        cookie.dmac_address = mpt->m_reply_frame_dma_addr;
     13540 +        cookie.dmac_address = mpt->m_reply_frame_dma_addr&0xfffffffful;
11902 13541          for (i = 0; i < mpt->m_max_replies; i++) {
11903 13542                  ddi_put32(mpt->m_acc_free_queue_hdl,
11904 13543                      &((uint32_t *)(void *)mpt->m_free_queue)[i],
11905 13544                      cookie.dmac_address);
11906 13545                  cookie.dmac_address += mpt->m_reply_frame_size;
11907 13546          }
11908 13547          (void) ddi_dma_sync(mpt->m_dma_free_queue_hdl, 0, 0,
11909 13548              DDI_DMA_SYNC_FORDEV);
11910 13549  
11911 13550          /*
11912 13551           * Initialize the reply free index to one past the last frame on the
11913 13552           * queue.  This will signify that the queue is empty to start with.
11914 13553           */
11915 13554          mpt->m_free_index = i;
11916 13555          ddi_put32(mpt->m_datap, &mpt->m_reg->ReplyFreeHostIndex, i);
11917 13556  
11918 13557          /*
11919      -         * Initialize the reply post queue to 0xFFFFFFFF,0xFFFFFFFF's.
     13558 +         * Initialize the reply post queue to 0xFFFFFFFF,0xFFFFFFFF's
     13559 +         * and the indexes to 0.
11920 13560           */
11921      -        for (i = 0; i < mpt->m_post_queue_depth; i++) {
11922      -                ddi_put64(mpt->m_acc_post_queue_hdl,
11923      -                    &((uint64_t *)(void *)mpt->m_post_queue)[i],
11924      -                    0xFFFFFFFFFFFFFFFF);
     13561 +        rpqp = mpt->m_rep_post_queues;
     13562 +        for (j = 0; j < mpt->m_post_reply_qcount; j++) {
     13563 +                for (i = 0; i < mpt->m_post_queue_depth; i++) {
     13564 +                        ddi_put64(mpt->m_acc_post_queue_hdl,
     13565 +                            &((uint64_t *)(void *)rpqp->rpq_queue)[i],
     13566 +                            0xFFFFFFFFFFFFFFFF);
     13567 +                }
     13568 +                rpqp->rpq_index = 0;
     13569 +                rpqp++;
11925 13570          }
11926 13571          (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
11927 13572              DDI_DMA_SYNC_FORDEV);
11928 13573  
11929 13574          /*
     13575 +         * Initialise all the reply post queue indexes.
     13576 +         */
     13577 +        for (j = 0; j < mpt->m_post_reply_qcount; j++) {
     13578 +                ddi_put32(mpt->m_datap, &mpt->m_reg->ReplyPostHostIndex,
     13579 +                    j << MPI2_RPHI_MSIX_INDEX_SHIFT);
     13580 +        }
     13581 +
     13582 +        /*
11930 13583           * Enable ports
11931 13584           */
11932 13585          if (mptsas_ioc_enable_port(mpt) == DDI_FAILURE) {
11933 13586                  mptsas_log(mpt, CE_WARN, "mptsas_ioc_enable_port failed");
11934 13587                  goto fail;
11935 13588          }
11936 13589  
11937 13590          /*
11938 13591           * enable events
11939 13592           */
11940 13593          if (mptsas_ioc_enable_event_notification(mpt)) {
     13594 +                mptsas_log(mpt, CE_WARN,
     13595 +                    "mptsas_ioc_enable_event_notification failed");
11941 13596                  goto fail;
11942 13597          }
11943 13598  
11944 13599          /*
11945 13600           * We need checks in attach and these.
11946 13601           * chip_init is called in mult. places
11947 13602           */
11948 13603  
11949 13604          if ((mptsas_check_dma_handle(mpt->m_dma_req_frame_hdl) !=
11950 13605              DDI_SUCCESS) ||
     13606 +            (mptsas_check_dma_handle(mpt->m_dma_req_sense_hdl) !=
     13607 +            DDI_SUCCESS) ||
11951 13608              (mptsas_check_dma_handle(mpt->m_dma_reply_frame_hdl) !=
11952 13609              DDI_SUCCESS) ||
11953 13610              (mptsas_check_dma_handle(mpt->m_dma_free_queue_hdl) !=
11954 13611              DDI_SUCCESS) ||
11955 13612              (mptsas_check_dma_handle(mpt->m_dma_post_queue_hdl) !=
11956 13613              DDI_SUCCESS) ||
11957 13614              (mptsas_check_dma_handle(mpt->m_hshk_dma_hdl) !=
11958 13615              DDI_SUCCESS)) {
11959 13616                  ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
11960 13617                  goto fail;
11961 13618          }
11962 13619  
11963 13620          /* Check all acc handles */
11964 13621          if ((mptsas_check_acc_handle(mpt->m_datap) != DDI_SUCCESS) ||
11965 13622              (mptsas_check_acc_handle(mpt->m_acc_req_frame_hdl) !=
11966 13623              DDI_SUCCESS) ||
     13624 +            (mptsas_check_acc_handle(mpt->m_acc_req_sense_hdl) !=
     13625 +            DDI_SUCCESS) ||
11967 13626              (mptsas_check_acc_handle(mpt->m_acc_reply_frame_hdl) !=
11968 13627              DDI_SUCCESS) ||
11969 13628              (mptsas_check_acc_handle(mpt->m_acc_free_queue_hdl) !=
11970 13629              DDI_SUCCESS) ||
11971 13630              (mptsas_check_acc_handle(mpt->m_acc_post_queue_hdl) !=
11972 13631              DDI_SUCCESS) ||
11973 13632              (mptsas_check_acc_handle(mpt->m_hshk_acc_hdl) !=
11974 13633              DDI_SUCCESS) ||
11975 13634              (mptsas_check_acc_handle(mpt->m_config_handle) !=
11976 13635              DDI_SUCCESS)) {
↓ open down ↓ 48 lines elided ↑ open up ↑
12025 13684                  }
12026 13685  
12027 13686                  /*
12028 13687                   * Get next capability and check that it is valid.
12029 13688                   * For now, we only support power management.
12030 13689                   */
12031 13690                  cap = pci_config_get8(mpt->m_config_handle, caps_ptr);
12032 13691                  switch (cap) {
12033 13692                          case PCI_CAP_ID_PM:
12034 13693                                  mptsas_log(mpt, CE_NOTE,
12035      -                                    "?mptsas%d supports power management.\n",
     13694 +                                    "?mptsas3%d supports power management.\n",
12036 13695                                      mpt->m_instance);
12037 13696                                  mpt->m_options |= MPTSAS_OPT_PM;
12038 13697  
12039 13698                                  /* Save PMCSR offset */
12040 13699                                  mpt->m_pmcsr_offset = caps_ptr + PCI_PMCSR;
12041 13700                                  break;
     13701 +                        case PCI_CAP_ID_MSI:
     13702 +                                mptsas_log(mpt, CE_NOTE,
     13703 +                                    "?mptsas3%d supports MSI.\n",
     13704 +                                    mpt->m_instance);
     13705 +                                mpt->m_options |= MPTSAS_OPT_MSI;
     13706 +                                break;
     13707 +                        case PCI_CAP_ID_MSI_X:
     13708 +                                mptsas_log(mpt, CE_NOTE,
     13709 +                                    "?mptsas3%d supports MSI-X.\n",
     13710 +                                    mpt->m_instance);
     13711 +                                mpt->m_options |= MPTSAS_OPT_MSI_X;
     13712 +                                break;
12042 13713                          /*
12043 13714                           * The following capabilities are valid.  Any others
12044 13715                           * will cause a message to be logged.
12045 13716                           */
12046 13717                          case PCI_CAP_ID_VPD:
12047      -                        case PCI_CAP_ID_MSI:
12048 13718                          case PCI_CAP_ID_PCIX:
12049 13719                          case PCI_CAP_ID_PCI_E:
12050      -                        case PCI_CAP_ID_MSI_X:
12051 13720                                  break;
12052 13721                          default:
12053 13722                                  mptsas_log(mpt, CE_NOTE,
12054      -                                    "?mptsas%d unrecognized capability "
     13723 +                                    "?mptsas3%d unrecognized capability "
12055 13724                                      "0x%x.\n", mpt->m_instance, cap);
12056 13725                                  break;
12057 13726                  }
12058 13727  
12059 13728                  /*
12060 13729                   * Get next capabilities pointer and clear bits 0,1.
12061 13730                   */
12062 13731                  caps_ptr = P2ALIGN(pci_config_get8(mpt->m_config_handle,
12063 13732                      (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
12064 13733          }
↓ open down ↓ 5 lines elided ↑ open up ↑
12070 13739  {
12071 13740          char            pmc_name[16];
12072 13741          char            *pmc[] = {
12073 13742                                  NULL,
12074 13743                                  "0=Off (PCI D3 State)",
12075 13744                                  "3=On (PCI D0 State)",
12076 13745                                  NULL
12077 13746                          };
12078 13747          uint16_t        pmcsr_stat;
12079 13748  
12080      -        if (mptsas_get_pci_cap(mpt) == FALSE) {
12081      -                return (DDI_FAILURE);
12082      -        }
12083 13749          /*
12084 13750           * If PCI's capability does not support PM, then don't need
12085 13751           * to registe the pm-components
12086 13752           */
12087 13753          if (!(mpt->m_options & MPTSAS_OPT_PM))
12088 13754                  return (DDI_SUCCESS);
12089 13755          /*
12090 13756           * If power management is supported by this chip, create
12091 13757           * pm-components property for the power management framework
12092 13758           */
12093      -        (void) sprintf(pmc_name, "NAME=mptsas%d", mpt->m_instance);
     13759 +        (void) sprintf(pmc_name, "NAME=mptsas3%d", mpt->m_instance);
12094 13760          pmc[0] = pmc_name;
12095 13761          if (ddi_prop_update_string_array(DDI_DEV_T_NONE, mpt->m_dip,
12096 13762              "pm-components", pmc, 3) != DDI_PROP_SUCCESS) {
12097 13763                  mpt->m_options &= ~MPTSAS_OPT_PM;
12098 13764                  mptsas_log(mpt, CE_WARN,
12099      -                    "mptsas%d: pm-component property creation failed.",
     13765 +                    "mptsas3%d: pm-component property creation failed.",
12100 13766                      mpt->m_instance);
12101 13767                  return (DDI_FAILURE);
12102 13768          }
12103 13769  
12104 13770          /*
12105 13771           * Power on device.
12106 13772           */
12107 13773          (void) pm_busy_component(mpt->m_dip, 0);
12108 13774          pmcsr_stat = pci_config_get16(mpt->m_config_handle,
12109 13775              mpt->m_pmcsr_offset);
12110 13776          if ((pmcsr_stat & PCI_PMCSR_STATE_MASK) != PCI_PMCSR_D0) {
12111      -                mptsas_log(mpt, CE_WARN, "mptsas%d: Power up the device",
     13777 +                mptsas_log(mpt, CE_WARN, "mptsas3%d: Power up the device",
12112 13778                      mpt->m_instance);
12113 13779                  pci_config_put16(mpt->m_config_handle, mpt->m_pmcsr_offset,
12114 13780                      PCI_PMCSR_D0);
12115 13781          }
12116 13782          if (pm_power_has_changed(mpt->m_dip, 0, PM_LEVEL_D0) != DDI_SUCCESS) {
12117 13783                  mptsas_log(mpt, CE_WARN, "pm_power_has_changed failed");
12118 13784                  return (DDI_FAILURE);
12119 13785          }
12120 13786          mpt->m_power_level = PM_LEVEL_D0;
12121 13787          /*
↓ open down ↓ 16 lines elided ↑ open up ↑
12138 13804          /* Get supported interrupt types */
12139 13805          if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
12140 13806                  mptsas_log(mpt, CE_WARN, "ddi_intr_get_supported_types "
12141 13807                      "failed\n");
12142 13808                  return (FALSE);
12143 13809          }
12144 13810  
12145 13811          NDBG6(("ddi_intr_get_supported_types() returned: 0x%x", intr_types));
12146 13812  
12147 13813          /*
     13814 +         * Try MSIX first.
     13815 +         */
     13816 +        if (mptsas_enable_msix && (intr_types & DDI_INTR_TYPE_MSIX)) {
     13817 +                if (mptsas_add_intrs(mpt, DDI_INTR_TYPE_MSIX) == DDI_SUCCESS) {
     13818 +                        NDBG6(("Using MSI-X interrupt type"));
     13819 +                        mpt->m_intr_type = DDI_INTR_TYPE_MSIX;
     13820 +                        return (TRUE);
     13821 +                }
     13822 +        }
     13823 +
     13824 +        /*
12148 13825           * Try MSI, but fall back to FIXED
12149 13826           */
12150 13827          if (mptsas_enable_msi && (intr_types & DDI_INTR_TYPE_MSI)) {
12151 13828                  if (mptsas_add_intrs(mpt, DDI_INTR_TYPE_MSI) == DDI_SUCCESS) {
12152      -                        NDBG0(("Using MSI interrupt type"));
     13829 +                        NDBG6(("Using MSI interrupt type"));
12153 13830                          mpt->m_intr_type = DDI_INTR_TYPE_MSI;
12154 13831                          return (TRUE);
12155 13832                  }
12156 13833          }
12157 13834          if (intr_types & DDI_INTR_TYPE_FIXED) {
12158 13835                  if (mptsas_add_intrs(mpt, DDI_INTR_TYPE_FIXED) == DDI_SUCCESS) {
12159      -                        NDBG0(("Using FIXED interrupt type"));
     13836 +                        NDBG6(("Using FIXED interrupt type"));
12160 13837                          mpt->m_intr_type = DDI_INTR_TYPE_FIXED;
12161 13838                          return (TRUE);
12162 13839                  } else {
12163      -                        NDBG0(("FIXED interrupt registration failed"));
     13840 +                        NDBG6(("FIXED interrupt registration failed"));
12164 13841                          return (FALSE);
12165 13842                  }
12166 13843          }
12167 13844  
12168 13845          return (FALSE);
12169 13846  }
12170 13847  
12171 13848  static void
12172 13849  mptsas_unregister_intrs(mptsas_t *mpt)
12173 13850  {
↓ open down ↓ 16 lines elided ↑ open up ↑
12190 13867  
12191 13868          /* Get number of interrupts */
12192 13869          ret = ddi_intr_get_nintrs(dip, intr_type, &count);
12193 13870          if ((ret != DDI_SUCCESS) || (count <= 0)) {
12194 13871                  mptsas_log(mpt, CE_WARN, "ddi_intr_get_nintrs() failed, "
12195 13872                      "ret %d count %d\n", ret, count);
12196 13873  
12197 13874                  return (DDI_FAILURE);
12198 13875          }
12199 13876  
12200      -        /* Get number of available interrupts */
     13877 +        /* Get number of interrupts available to this device */
12201 13878          ret = ddi_intr_get_navail(dip, intr_type, &avail);
12202 13879          if ((ret != DDI_SUCCESS) || (avail == 0)) {
12203 13880                  mptsas_log(mpt, CE_WARN, "ddi_intr_get_navail() failed, "
12204 13881                      "ret %d avail %d\n", ret, avail);
12205 13882  
12206 13883                  return (DDI_FAILURE);
12207 13884          }
12208 13885  
12209      -        if (avail < count) {
     13886 +        if (count < avail) {
12210 13887                  mptsas_log(mpt, CE_NOTE, "ddi_intr_get_nvail returned %d, "
12211 13888                      "navail() returned %d", count, avail);
12212 13889          }
12213 13890  
12214      -        /* Mpt only have one interrupt routine */
12215      -        if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
12216      -                count = 1;
     13891 +        NDBG6(("mptsas_add_intrs:count %d, avail %d", count, avail));
     13892 +
     13893 +        if (intr_type == DDI_INTR_TYPE_MSIX) {
     13894 +                if (!mptsas3_max_msix_intrs) {
     13895 +                        return (DDI_FAILURE);
     13896 +                }
     13897 +
     13898 +                /*
     13899 +                 * Restrict the number of interrupts, firstly by
     13900 +                 * the number returned from the IOCInfo, then by
     13901 +                 * overall restriction.
     13902 +                 */
     13903 +                if (avail > mpt->m_max_msix_vectors) {
     13904 +                        avail = mpt->m_max_msix_vectors?
     13905 +                            mpt->m_max_msix_vectors:1;
     13906 +                        NDBG6(("mptsas_add_intrs: mmmv avail %d", avail));
     13907 +                }
     13908 +                if (avail > mptsas3_max_msix_intrs) {
     13909 +                        avail = mptsas3_max_msix_intrs;
     13910 +                        NDBG6(("mptsas_add_intrs: m3mmi avail %d", avail));
     13911 +                }
     13912 +        }
     13913 +        if (intr_type == DDI_INTR_TYPE_MSI) {
     13914 +                NDBG6(("mptsas_add_intrs: MSI avail %d", avail));
     13915 +                avail = 1;
12217 13916          }
12218 13917  
12219 13918          /* Allocate an array of interrupt handles */
12220      -        mpt->m_intr_size = count * sizeof (ddi_intr_handle_t);
     13919 +        mpt->m_intr_size = avail * sizeof (ddi_intr_handle_t);
12221 13920          mpt->m_htable = kmem_alloc(mpt->m_intr_size, KM_SLEEP);
12222 13921  
12223 13922          flag = DDI_INTR_ALLOC_NORMAL;
12224 13923  
12225 13924          /* call ddi_intr_alloc() */
12226 13925          ret = ddi_intr_alloc(dip, mpt->m_htable, intr_type, 0,
12227      -            count, &actual, flag);
     13926 +            avail, &actual, flag);
12228 13927  
12229 13928          if ((ret != DDI_SUCCESS) || (actual == 0)) {
12230 13929                  mptsas_log(mpt, CE_WARN, "ddi_intr_alloc() failed, ret %d\n",
12231 13930                      ret);
12232 13931                  kmem_free(mpt->m_htable, mpt->m_intr_size);
12233 13932                  return (DDI_FAILURE);
12234 13933          }
12235 13934  
     13935 +        NDBG6(("mptsas_add_intrs: actual %d, avail %d", actual, avail));
12236 13936          /* use interrupt count returned or abort? */
12237      -        if (actual < count) {
12238      -                mptsas_log(mpt, CE_NOTE, "Requested: %d, Received: %d\n",
12239      -                    count, actual);
     13937 +        if (actual < avail) {
     13938 +                mptsas_log(mpt, CE_NOTE,
     13939 +                    "Interrupts requested: %d, received: %d\n",
     13940 +                    avail, actual);
12240 13941          }
12241 13942  
12242      -        mpt->m_intr_cnt = actual;
12243      -
12244 13943          /*
12245 13944           * Get priority for first msi, assume remaining are all the same
12246 13945           */
12247 13946          if ((ret = ddi_intr_get_pri(mpt->m_htable[0],
12248 13947              &mpt->m_intr_pri)) != DDI_SUCCESS) {
12249 13948                  mptsas_log(mpt, CE_WARN, "ddi_intr_get_pri() failed %d\n", ret);
12250 13949  
12251 13950                  /* Free already allocated intr */
12252 13951                  for (i = 0; i < actual; i++) {
12253 13952                          (void) ddi_intr_free(mpt->m_htable[i]);
↓ open down ↓ 40 lines elided ↑ open up ↑
12294 13993  
12295 13994                  /* Free already allocated intr */
12296 13995                  for (i = 0; i < actual; i++) {
12297 13996                          (void) ddi_intr_free(mpt->m_htable[i]);
12298 13997                  }
12299 13998  
12300 13999                  kmem_free(mpt->m_htable, mpt->m_intr_size);
12301 14000                  return (DDI_FAILURE);
12302 14001          }
12303 14002  
     14003 +        mpt->m_intr_cnt = actual;
     14004 +
12304 14005          /*
12305 14006           * Enable interrupts
12306 14007           */
12307 14008          if (mpt->m_intr_cap & DDI_INTR_FLAG_BLOCK) {
12308 14009                  /* Call ddi_intr_block_enable() for MSI interrupts */
12309 14010                  (void) ddi_intr_block_enable(mpt->m_htable, mpt->m_intr_cnt);
12310 14011          } else {
12311 14012                  /* Call ddi_intr_enable for MSI or FIXED interrupts */
12312 14013                  for (i = 0; i < mpt->m_intr_cnt; i++) {
12313 14014                          (void) ddi_intr_enable(mpt->m_htable[i]);
12314 14015                  }
12315 14016          }
     14017 +
     14018 +        switch (intr_type) {
     14019 +        case DDI_INTR_TYPE_MSIX:
     14020 +                mptsas_log(mpt, CE_NOTE, "?Using %d MSI-X interrupt(s) "
     14021 +                    "(Available sys %d, mpt %d, Requested %d)\n",
     14022 +                    actual, count, mpt->m_max_msix_vectors, avail);
     14023 +                break;
     14024 +        case DDI_INTR_TYPE_MSI:
     14025 +                mptsas_log(mpt, CE_NOTE, "Using single MSI interrupt\n");
     14026 +                break;
     14027 +        case DDI_INTR_TYPE_FIXED:
     14028 +        default:
     14029 +                mptsas_log(mpt, CE_NOTE, "Using single fixed interrupt\n");
     14030 +                break;
     14031 +        }
     14032 +
12316 14033          return (DDI_SUCCESS);
12317 14034  }
12318 14035  
12319 14036  /*
12320 14037   * mptsas_rem_intrs:
12321 14038   *
12322 14039   * Unregister FIXED or MSI interrupts
12323 14040   */
12324 14041  static void
12325 14042  mptsas_rem_intrs(mptsas_t *mpt)
↓ open down ↓ 10 lines elided ↑ open up ↑
12336 14053                  for (i = 0; i < mpt->m_intr_cnt; i++) {
12337 14054                          (void) ddi_intr_disable(mpt->m_htable[i]);
12338 14055                  }
12339 14056          }
12340 14057  
12341 14058          /* Call ddi_intr_remove_handler() */
12342 14059          for (i = 0; i < mpt->m_intr_cnt; i++) {
12343 14060                  (void) ddi_intr_remove_handler(mpt->m_htable[i]);
12344 14061                  (void) ddi_intr_free(mpt->m_htable[i]);
12345 14062          }
12346      -
12347 14063          kmem_free(mpt->m_htable, mpt->m_intr_size);
     14064 +        mpt->m_intr_cnt = 0;
12348 14065  }
12349 14066  
12350 14067  /*
12351 14068   * The IO fault service error handling callback function
12352 14069   */
12353 14070  /*ARGSUSED*/
12354 14071  static int
12355 14072  mptsas_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
12356 14073  {
12357 14074          /*
↓ open down ↓ 129 lines elided ↑ open up ↑
12487 14204      uint16_t *dev_handle, mptsas_target_t **pptgt)
12488 14205  {
12489 14206          int             rval;
12490 14207          uint32_t        dev_info;
12491 14208          uint64_t        sas_wwn;
12492 14209          mptsas_phymask_t phymask;
12493 14210          uint8_t         physport, phynum, config, disk;
12494 14211          uint64_t        devicename;
12495 14212          uint16_t        pdev_hdl;
12496 14213          mptsas_target_t *tmp_tgt = NULL;
12497      -        uint16_t        bay_num, enclosure;
     14214 +        uint16_t        bay_num, enclosure, io_flags;
12498 14215  
12499 14216          ASSERT(*pptgt == NULL);
12500 14217  
12501 14218          rval = mptsas_get_sas_device_page0(mpt, page_address, dev_handle,
12502 14219              &sas_wwn, &dev_info, &physport, &phynum, &pdev_hdl,
12503      -            &bay_num, &enclosure);
     14220 +            &bay_num, &enclosure, &io_flags);
12504 14221          if (rval != DDI_SUCCESS) {
12505 14222                  rval = DEV_INFO_FAIL_PAGE0;
12506 14223                  return (rval);
12507 14224          }
12508 14225  
12509 14226          if ((dev_info & (MPI2_SAS_DEVICE_INFO_SSP_TARGET |
12510 14227              MPI2_SAS_DEVICE_INFO_SATA_DEVICE |
12511 14228              MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE)) == NULL) {
12512 14229                  rval = DEV_INFO_WRONG_DEVICE_TYPE;
12513 14230                  return (rval);
↓ open down ↓ 24 lines elided ↑ open up ↑
12538 14255                  mutex_exit(&mpt->m_mutex);
12539 14256                  /* alloc a tmp_tgt to send the cmd */
12540 14257                  tmp_tgt = kmem_zalloc(sizeof (struct mptsas_target),
12541 14258                      KM_SLEEP);
12542 14259                  tmp_tgt->m_devhdl = *dev_handle;
12543 14260                  tmp_tgt->m_deviceinfo = dev_info;
12544 14261                  tmp_tgt->m_qfull_retries = QFULL_RETRIES;
12545 14262                  tmp_tgt->m_qfull_retry_interval =
12546 14263                      drv_usectohz(QFULL_RETRY_INTERVAL * 1000);
12547 14264                  tmp_tgt->m_t_throttle = MAX_THROTTLE;
     14265 +                mutex_init(&tmp_tgt->m_t_mutex, NULL, MUTEX_DRIVER, NULL);
12548 14266                  devicename = mptsas_get_sata_guid(mpt, tmp_tgt, 0);
     14267 +                mutex_destroy(&tmp_tgt->m_t_mutex);
12549 14268                  kmem_free(tmp_tgt, sizeof (struct mptsas_target));
12550 14269                  mutex_enter(&mpt->m_mutex);
12551 14270                  if (devicename != 0 && (((devicename >> 56) & 0xf0) == 0x50)) {
12552 14271                          sas_wwn = devicename;
12553 14272                  } else if (dev_info & MPI2_SAS_DEVICE_INFO_DIRECT_ATTACH) {
12554 14273                          sas_wwn = 0;
12555 14274                  }
12556 14275          }
12557 14276  
12558 14277          phymask = mptsas_physport_to_phymask(mpt, physport);
12559 14278          *pptgt = mptsas_tgt_alloc(mpt, *dev_handle, sas_wwn,
12560 14279              dev_info, phymask, phynum);
12561 14280          if (*pptgt == NULL) {
12562 14281                  mptsas_log(mpt, CE_WARN, "Failed to allocated target"
12563 14282                      "structure!");
12564 14283                  rval = DEV_INFO_FAIL_ALLOC;
12565 14284                  return (rval);
12566 14285          }
     14286 +        (*pptgt)->m_io_flags = io_flags;
12567 14287          (*pptgt)->m_enclosure = enclosure;
12568 14288          (*pptgt)->m_slot_num = bay_num;
12569 14289          return (DEV_INFO_SUCCESS);
12570 14290  }
12571 14291  
12572 14292  uint64_t
12573 14293  mptsas_get_sata_guid(mptsas_t *mpt, mptsas_target_t *ptgt, int lun)
12574 14294  {
12575 14295          uint64_t        sata_guid = 0, *pwwn = NULL;
12576 14296          int             target = ptgt->m_devhdl;
↓ open down ↓ 117 lines elided ↑ open up ↑
12694 14414          ap->a_hba_tran = tran_clone;
12695 14415  
12696 14416          pktp = scsi_init_pkt(ap, (struct scsi_pkt *)NULL,
12697 14417              data_bp, cdblen, sizeof (struct scsi_arq_status),
12698 14418              0, PKT_CONSISTENT, NULL, NULL);
12699 14419          if (pktp == NULL) {
12700 14420                  goto out;
12701 14421          }
12702 14422          bcopy(cdb, pktp->pkt_cdbp, cdblen);
12703 14423          pktp->pkt_flags = FLAG_NOPARITY;
     14424 +        pktp->pkt_time = mptsas_scsi_pkt_time;
12704 14425          if (scsi_poll(pktp) < 0) {
12705 14426                  goto out;
12706 14427          }
12707 14428          if (((struct scsi_status *)pktp->pkt_scbp)->sts_chk) {
12708 14429                  goto out;
12709 14430          }
12710 14431          if (resid != NULL) {
12711 14432                  *resid = pktp->pkt_resid;
12712 14433          }
12713 14434  
↓ open down ↓ 828 lines elided ↑ open up ↑
13542 15263          }
13543 15264          return (rval);
13544 15265  }
13545 15266  
13546 15267  /*
13547 15268   * Return fail if not all the childs/paths are freed.
13548 15269   * if there is any path under the HBA, the return value will be always fail
13549 15270   * because we didn't call mdi_pi_free for path
13550 15271   */
13551 15272  static int
13552      -mptsas_offline_target(dev_info_t *pdip, char *name)
     15273 +mptsas_offline_targetdev(dev_info_t *pdip, char *name)
13553 15274  {
13554 15275          dev_info_t              *child = NULL, *prechild = NULL;
13555 15276          mdi_pathinfo_t          *pip = NULL, *savepip = NULL;
13556 15277          int                     tmp_rval, rval = DDI_SUCCESS;
13557 15278          char                    *addr, *cp;
13558 15279          size_t                  s;
13559 15280          mptsas_t                *mpt = DIP2MPT(pdip);
13560 15281  
13561 15282          child = ddi_get_child(pdip);
13562 15283          while (child) {
↓ open down ↓ 14 lines elided ↑ open up ↑
13577 15298                          continue;
13578 15299                  }
13579 15300  
13580 15301                  tmp_rval = mptsas_offline_lun(pdip, prechild, NULL,
13581 15302                      NDI_DEVI_REMOVE);
13582 15303                  if (tmp_rval != DDI_SUCCESS) {
13583 15304                          rval = DDI_FAILURE;
13584 15305                          if (ndi_prop_create_boolean(DDI_DEV_T_NONE,
13585 15306                              prechild, MPTSAS_DEV_GONE) !=
13586 15307                              DDI_PROP_SUCCESS) {
13587      -                                mptsas_log(mpt, CE_WARN, "mptsas driver "
     15308 +                                mptsas_log(mpt, CE_WARN,
13588 15309                                      "unable to create property for "
13589 15310                                      "SAS %s (MPTSAS_DEV_GONE)", addr);
13590 15311                          }
13591 15312                  }
13592 15313          }
13593 15314  
13594 15315          pip = mdi_get_next_client_path(pdip, NULL);
13595 15316          while (pip) {
13596 15317                  addr = MDI_PI(pip)->pi_addr;
13597 15318                  savepip = pip;
↓ open down ↓ 356 lines elided ↑ open up ↑
13954 15675  {
13955 15676          int                     target;
13956 15677          char                    *nodename = NULL;
13957 15678          char                    **compatible = NULL;
13958 15679          int                     ncompatible     = 0;
13959 15680          int                     mdi_rtn = MDI_FAILURE;
13960 15681          int                     rval = DDI_FAILURE;
13961 15682          char                    *old_guid = NULL;
13962 15683          mptsas_t                *mpt = DIP2MPT(pdip);
13963 15684          char                    *lun_addr = NULL;
13964      -        char                    *wwn_str = NULL;
13965      -        char                    *attached_wwn_str = NULL;
     15685 +        char                    wwn_str[MPTSAS_WWN_STRLEN];
13966 15686          char                    *component = NULL;
13967 15687          uint8_t                 phy = 0xFF;
13968 15688          uint64_t                sas_wwn;
13969 15689          int64_t                 lun64 = 0;
13970 15690          uint32_t                devinfo;
13971 15691          uint16_t                dev_hdl;
13972 15692          uint16_t                pdev_hdl;
13973 15693          uint64_t                dev_sas_wwn;
13974 15694          uint64_t                pdev_sas_wwn;
13975 15695          uint32_t                pdev_info;
13976 15696          uint8_t                 physport;
13977 15697          uint8_t                 phy_id;
13978 15698          uint32_t                page_address;
13979      -        uint16_t                bay_num, enclosure;
     15699 +        uint16_t                bay_num, enclosure, io_flags;
13980 15700          char                    pdev_wwn_str[MPTSAS_WWN_STRLEN];
13981 15701          uint32_t                dev_info;
13982 15702  
13983 15703          mutex_enter(&mpt->m_mutex);
13984 15704          target = ptgt->m_devhdl;
13985 15705          sas_wwn = ptgt->m_addr.mta_wwn;
13986 15706          devinfo = ptgt->m_deviceinfo;
13987 15707          phy = ptgt->m_phynum;
13988 15708          mutex_exit(&mpt->m_mutex);
13989 15709  
↓ open down ↓ 73 lines elided ↑ open up ↑
14063 15783                          return (DDI_FAILURE);
14064 15784                  }
14065 15785          }
14066 15786          scsi_hba_nodename_compatible_get(inq, NULL,
14067 15787              inq->inq_dtype, NULL, &nodename, &compatible, &ncompatible);
14068 15788  
14069 15789          /*
14070 15790           * if nodename can't be determined then print a message and skip it
14071 15791           */
14072 15792          if (nodename == NULL) {
14073      -                mptsas_log(mpt, CE_WARN, "mptsas driver found no compatible "
     15793 +                mptsas_log(mpt, CE_WARN, "found no compatible "
14074 15794                      "driver for target%d lun %d dtype:0x%02x", target, lun,
14075 15795                      inq->inq_dtype);
14076 15796                  return (DDI_FAILURE);
14077 15797          }
14078 15798  
14079      -        wwn_str = kmem_zalloc(MPTSAS_WWN_STRLEN, KM_SLEEP);
14080 15799          /* The property is needed by MPAPI */
14081 15800          (void) sprintf(wwn_str, "%016"PRIx64, sas_wwn);
14082 15801  
14083 15802          lun_addr = kmem_zalloc(SCSI_MAXNAMELEN, KM_SLEEP);
14084 15803          if (guid) {
14085 15804                  (void) sprintf(lun_addr, "w%s,%x", wwn_str, lun);
14086 15805                  (void) sprintf(wwn_str, "w%016"PRIx64, sas_wwn);
14087 15806          } else {
14088 15807                  (void) sprintf(lun_addr, "p%x,%x", phy, lun);
14089 15808                  (void) sprintf(wwn_str, "p%x", phy);
14090 15809          }
14091 15810  
14092 15811          mdi_rtn = mdi_pi_alloc_compatible(pdip, nodename,
14093 15812              guid, lun_addr, compatible, ncompatible,
14094 15813              0, pip);
14095 15814          if (mdi_rtn == MDI_SUCCESS) {
14096 15815  
14097 15816                  if (mdi_prop_update_string(*pip, MDI_GUID,
14098 15817                      guid) != DDI_SUCCESS) {
14099      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15818 +                        mptsas_log(mpt, CE_WARN, "unable to "
14100 15819                              "create prop for target %d lun %d (MDI_GUID)",
14101 15820                              target, lun);
14102 15821                          mdi_rtn = MDI_FAILURE;
14103 15822                          goto virt_create_done;
14104 15823                  }
14105 15824  
14106 15825                  if (mdi_prop_update_int(*pip, LUN_PROP,
14107 15826                      lun) != DDI_SUCCESS) {
14108      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15827 +                        mptsas_log(mpt, CE_WARN, "unable to "
14109 15828                              "create prop for target %d lun %d (LUN_PROP)",
14110 15829                              target, lun);
14111 15830                          mdi_rtn = MDI_FAILURE;
14112 15831                          goto virt_create_done;
14113 15832                  }
14114 15833                  lun64 = (int64_t)lun;
14115 15834                  if (mdi_prop_update_int64(*pip, LUN64_PROP,
14116 15835                      lun64) != DDI_SUCCESS) {
14117      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15836 +                        mptsas_log(mpt, CE_WARN, "unable to "
14118 15837                              "create prop for target %d (LUN64_PROP)",
14119 15838                              target);
14120 15839                          mdi_rtn = MDI_FAILURE;
14121 15840                          goto virt_create_done;
14122 15841                  }
14123 15842                  if (mdi_prop_update_string_array(*pip, "compatible",
14124 15843                      compatible, ncompatible) !=
14125 15844                      DDI_PROP_SUCCESS) {
14126      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15845 +                        mptsas_log(mpt, CE_WARN, "unable to "
14127 15846                              "create prop for target %d lun %d (COMPATIBLE)",
14128 15847                              target, lun);
14129 15848                          mdi_rtn = MDI_FAILURE;
14130 15849                          goto virt_create_done;
14131 15850                  }
14132 15851                  if (sas_wwn && (mdi_prop_update_string(*pip,
14133 15852                      SCSI_ADDR_PROP_TARGET_PORT, wwn_str) != DDI_PROP_SUCCESS)) {
14134      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15853 +                        mptsas_log(mpt, CE_WARN, "unable to "
14135 15854                              "create prop for target %d lun %d "
14136 15855                              "(target-port)", target, lun);
14137 15856                          mdi_rtn = MDI_FAILURE;
14138 15857                          goto virt_create_done;
14139 15858                  } else if ((sas_wwn == 0) && (mdi_prop_update_int(*pip,
14140 15859                      "sata-phy", phy) != DDI_PROP_SUCCESS)) {
14141 15860                          /*
14142 15861                           * Direct attached SATA device without DeviceName
14143 15862                           */
14144      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15863 +                        mptsas_log(mpt, CE_WARN, "unable to "
14145 15864                              "create prop for SAS target %d lun %d "
14146 15865                              "(sata-phy)", target, lun);
14147 15866                          mdi_rtn = MDI_FAILURE;
14148 15867                          goto virt_create_done;
14149 15868                  }
14150 15869                  mutex_enter(&mpt->m_mutex);
14151 15870  
14152 15871                  page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
14153 15872                      MPI2_SAS_DEVICE_PGAD_FORM_MASK) |
14154 15873                      (uint32_t)ptgt->m_devhdl;
14155 15874                  rval = mptsas_get_sas_device_page0(mpt, page_address,
14156 15875                      &dev_hdl, &dev_sas_wwn, &dev_info, &physport,
14157      -                    &phy_id, &pdev_hdl, &bay_num, &enclosure);
     15876 +                    &phy_id, &pdev_hdl, &bay_num, &enclosure, &io_flags);
14158 15877                  if (rval != DDI_SUCCESS) {
14159 15878                          mutex_exit(&mpt->m_mutex);
14160 15879                          mptsas_log(mpt, CE_WARN, "mptsas unable to get "
14161 15880                              "parent device for handle %d", page_address);
14162 15881                          mdi_rtn = MDI_FAILURE;
14163 15882                          goto virt_create_done;
14164 15883                  }
14165 15884  
14166 15885                  page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
14167 15886                      MPI2_SAS_DEVICE_PGAD_FORM_MASK) | (uint32_t)pdev_hdl;
14168 15887                  rval = mptsas_get_sas_device_page0(mpt, page_address,
14169 15888                      &dev_hdl, &pdev_sas_wwn, &pdev_info, &physport,
14170      -                    &phy_id, &pdev_hdl, &bay_num, &enclosure);
     15889 +                    &phy_id, &pdev_hdl, &bay_num, &enclosure, &io_flags);
14171 15890                  if (rval != DDI_SUCCESS) {
14172 15891                          mutex_exit(&mpt->m_mutex);
14173 15892                          mptsas_log(mpt, CE_WARN, "mptsas unable to get"
14174 15893                              "device info for handle %d", page_address);
14175 15894                          mdi_rtn = MDI_FAILURE;
14176 15895                          goto virt_create_done;
14177 15896                  }
14178 15897  
14179 15898                  mutex_exit(&mpt->m_mutex);
14180 15899  
↓ open down ↓ 26 lines elided ↑ open up ↑
14207 15926                                  goto virt_create_done;
14208 15927                          }
14209 15928  
14210 15929                          (void) sprintf(pdev_wwn_str, "w%016"PRIx64,
14211 15930                              mpt->un.m_base_wwid);
14212 15931                  }
14213 15932  
14214 15933                  if (mdi_prop_update_string(*pip,
14215 15934                      SCSI_ADDR_PROP_ATTACHED_PORT, pdev_wwn_str) !=
14216 15935                      DDI_PROP_SUCCESS) {
14217      -                        mptsas_log(mpt, CE_WARN, "mptsas unable to create "
     15936 +                        mptsas_log(mpt, CE_WARN, "unable to create "
14218 15937                              "property for iport attached-port %s (sas_wwn)",
14219      -                            attached_wwn_str);
     15938 +                            pdev_wwn_str);
14220 15939                          mdi_rtn = MDI_FAILURE;
14221 15940                          goto virt_create_done;
14222 15941                  }
14223 15942  
14224 15943  
14225 15944                  if (inq->inq_dtype == 0) {
14226 15945                          component = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
14227 15946                          /*
14228 15947                           * set obp path for pathinfo
14229 15948                           */
14230 15949                          (void) snprintf(component, MAXPATHLEN,
14231 15950                              "disk@%s", lun_addr);
14232 15951  
14233 15952                          if (mdi_pi_pathname_obp_set(*pip, component) !=
14234 15953                              DDI_SUCCESS) {
14235      -                                mptsas_log(mpt, CE_WARN, "mpt_sas driver "
     15954 +                                mptsas_log(mpt, CE_WARN,
14236 15955                                      "unable to set obp-path for object %s",
14237 15956                                      component);
14238 15957                                  mdi_rtn = MDI_FAILURE;
14239 15958                                  goto virt_create_done;
14240 15959                          }
14241 15960                  }
14242 15961  
14243 15962                  *lun_dip = MDI_PI(*pip)->pi_client->ct_dip;
14244 15963                  if (devinfo & (MPI2_SAS_DEVICE_INFO_SATA_DEVICE |
14245 15964                      MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE)) {
14246 15965                          if ((ndi_prop_update_int(DDI_DEV_T_NONE, *lun_dip,
14247 15966                              "pm-capable", 1)) !=
14248 15967                              DDI_PROP_SUCCESS) {
14249      -                                mptsas_log(mpt, CE_WARN, "mptsas driver"
     15968 +                                mptsas_log(mpt, CE_WARN,
14250 15969                                      "failed to create pm-capable "
14251 15970                                      "property, target %d", target);
14252 15971                                  mdi_rtn = MDI_FAILURE;
14253 15972                                  goto virt_create_done;
14254 15973                          }
14255 15974                  }
14256 15975                  /*
14257 15976                   * Create the phy-num property
14258 15977                   */
14259 15978                  if (mdi_prop_update_int(*pip, "phy-num",
14260 15979                      ptgt->m_phynum) != DDI_SUCCESS) {
14261      -                        mptsas_log(mpt, CE_WARN, "mptsas driver unable to "
     15980 +                        mptsas_log(mpt, CE_WARN, "unable to "
14262 15981                              "create phy-num property for target %d lun %d",
14263 15982                              target, lun);
14264 15983                          mdi_rtn = MDI_FAILURE;
14265 15984                          goto virt_create_done;
14266 15985                  }
14267 15986                  NDBG20(("new path:%s onlining,", MDI_PI(*pip)->pi_addr));
14268 15987                  mdi_rtn = mdi_pi_online(*pip, 0);
14269 15988                  if (mdi_rtn == MDI_SUCCESS) {
14270 15989                          mutex_enter(&mpt->m_mutex);
14271 15990                          ptgt->m_led_status = 0;
↓ open down ↓ 8 lines elided ↑ open up ↑
14280 15999                          (void) mdi_pi_free(*pip, 0);
14281 16000                          *pip = NULL;
14282 16001                          *lun_dip = NULL;
14283 16002                  }
14284 16003          }
14285 16004  
14286 16005          scsi_hba_nodename_compatible_free(nodename, compatible);
14287 16006          if (lun_addr != NULL) {
14288 16007                  kmem_free(lun_addr, SCSI_MAXNAMELEN);
14289 16008          }
14290      -        if (wwn_str != NULL) {
14291      -                kmem_free(wwn_str, MPTSAS_WWN_STRLEN);
14292      -        }
14293 16009          if (component != NULL) {
14294 16010                  kmem_free(component, MAXPATHLEN);
14295 16011          }
14296 16012  
14297 16013          return ((mdi_rtn == MDI_SUCCESS) ? DDI_SUCCESS : DDI_FAILURE);
14298 16014  }
14299 16015  
14300 16016  static int
14301 16017  mptsas_create_phys_lun(dev_info_t *pdip, struct scsi_inquiry *inq,
14302 16018      char *guid, dev_info_t **lun_dip, mptsas_target_t *ptgt, int lun)
14303 16019  {
14304 16020          int                     target;
14305 16021          int                     rval;
14306 16022          int                     ndi_rtn = NDI_FAILURE;
14307 16023          uint64_t                be_sas_wwn;
14308 16024          char                    *nodename = NULL;
14309 16025          char                    **compatible = NULL;
14310 16026          int                     ncompatible = 0;
14311 16027          int                     instance = 0;
14312 16028          mptsas_t                *mpt = DIP2MPT(pdip);
14313      -        char                    *wwn_str = NULL;
14314      -        char                    *component = NULL;
14315      -        char                    *attached_wwn_str = NULL;
     16029 +        char                    wwn_str[MPTSAS_WWN_STRLEN];
     16030 +        char                    component[MAXPATHLEN];
14316 16031          uint8_t                 phy = 0xFF;
14317 16032          uint64_t                sas_wwn;
14318 16033          uint32_t                devinfo;
14319 16034          uint16_t                dev_hdl;
14320 16035          uint16_t                pdev_hdl;
14321 16036          uint64_t                pdev_sas_wwn;
14322 16037          uint64_t                dev_sas_wwn;
14323 16038          uint32_t                pdev_info;
14324 16039          uint8_t                 physport;
14325 16040          uint8_t                 phy_id;
14326 16041          uint32_t                page_address;
14327      -        uint16_t                bay_num, enclosure;
     16042 +        uint16_t                bay_num, enclosure, io_flags;
14328 16043          char                    pdev_wwn_str[MPTSAS_WWN_STRLEN];
14329 16044          uint32_t                dev_info;
14330 16045          int64_t                 lun64 = 0;
14331 16046  
14332 16047          mutex_enter(&mpt->m_mutex);
14333 16048          target = ptgt->m_devhdl;
14334 16049          sas_wwn = ptgt->m_addr.mta_wwn;
14335 16050          devinfo = ptgt->m_deviceinfo;
14336 16051          phy = ptgt->m_phynum;
14337 16052          mutex_exit(&mpt->m_mutex);
↓ open down ↓ 50 lines elided ↑ open up ↑
14388 16103                          ndi_rtn = NDI_FAILURE;
14389 16104                          goto phys_create_done;
14390 16105                  }
14391 16106  
14392 16107                  /*
14393 16108                   * We need the SAS WWN for non-multipath devices, so
14394 16109                   * we'll use the same property as that multipathing
14395 16110                   * devices need to present for MPAPI. If we don't have
14396 16111                   * a WWN (e.g. parallel SCSI), don't create the prop.
14397 16112                   */
14398      -                wwn_str = kmem_zalloc(MPTSAS_WWN_STRLEN, KM_SLEEP);
14399 16113                  (void) sprintf(wwn_str, "w%016"PRIx64, sas_wwn);
14400 16114                  if (sas_wwn && ndi_prop_update_string(DDI_DEV_T_NONE,
14401 16115                      *lun_dip, SCSI_ADDR_PROP_TARGET_PORT, wwn_str)
14402 16116                      != DDI_PROP_SUCCESS) {
14403 16117                          mptsas_log(mpt, CE_WARN, "mptsas unable to "
14404 16118                              "create property for SAS target %d lun %d "
14405 16119                              "(target-port)", target, lun);
14406 16120                          ndi_rtn = NDI_FAILURE;
14407 16121                          goto phys_create_done;
14408 16122                  }
↓ open down ↓ 45 lines elided ↑ open up ↑
14454 16168                          goto phys_raid_lun;
14455 16169  
14456 16170                  mutex_enter(&mpt->m_mutex);
14457 16171  
14458 16172                  page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
14459 16173                      MPI2_SAS_DEVICE_PGAD_FORM_MASK) |
14460 16174                      (uint32_t)ptgt->m_devhdl;
14461 16175                  rval = mptsas_get_sas_device_page0(mpt, page_address,
14462 16176                      &dev_hdl, &dev_sas_wwn, &dev_info,
14463 16177                      &physport, &phy_id, &pdev_hdl,
14464      -                    &bay_num, &enclosure);
     16178 +                    &bay_num, &enclosure, &io_flags);
14465 16179                  if (rval != DDI_SUCCESS) {
14466 16180                          mutex_exit(&mpt->m_mutex);
14467 16181                          mptsas_log(mpt, CE_WARN, "mptsas unable to get"
14468 16182                              "parent device for handle %d.", page_address);
14469 16183                          ndi_rtn = NDI_FAILURE;
14470 16184                          goto phys_create_done;
14471 16185                  }
14472 16186  
14473 16187                  page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
14474 16188                      MPI2_SAS_DEVICE_PGAD_FORM_MASK) | (uint32_t)pdev_hdl;
14475 16189                  rval = mptsas_get_sas_device_page0(mpt, page_address,
14476      -                    &dev_hdl, &pdev_sas_wwn, &pdev_info,
14477      -                    &physport, &phy_id, &pdev_hdl, &bay_num, &enclosure);
     16190 +                    &dev_hdl, &pdev_sas_wwn, &pdev_info, &physport,
     16191 +                    &phy_id, &pdev_hdl, &bay_num, &enclosure, &io_flags);
14478 16192                  if (rval != DDI_SUCCESS) {
14479 16193                          mutex_exit(&mpt->m_mutex);
14480 16194                          mptsas_log(mpt, CE_WARN, "mptsas unable to create "
14481 16195                              "device for handle %d.", page_address);
14482 16196                          ndi_rtn = NDI_FAILURE;
14483 16197                          goto phys_create_done;
14484 16198                  }
14485 16199  
14486 16200                  mutex_exit(&mpt->m_mutex);
14487 16201  
↓ open down ↓ 29 lines elided ↑ open up ↑
14517 16231                          (void) sprintf(pdev_wwn_str, "w%016"PRIx64,
14518 16232                              mpt->un.m_base_wwid);
14519 16233                  }
14520 16234  
14521 16235                  if (ndi_prop_update_string(DDI_DEV_T_NONE,
14522 16236                      *lun_dip, SCSI_ADDR_PROP_ATTACHED_PORT, pdev_wwn_str) !=
14523 16237                      DDI_PROP_SUCCESS) {
14524 16238                          mptsas_log(mpt, CE_WARN,
14525 16239                              "mptsas unable to create "
14526 16240                              "property for iport attached-port %s (sas_wwn)",
14527      -                            attached_wwn_str);
     16241 +                            pdev_wwn_str);
14528 16242                          ndi_rtn = NDI_FAILURE;
14529 16243                          goto phys_create_done;
14530 16244                  }
14531 16245  
14532 16246                  if (IS_SATA_DEVICE(dev_info)) {
14533 16247                          if (ndi_prop_update_string(DDI_DEV_T_NONE,
14534 16248                              *lun_dip, MPTSAS_VARIANT, "sata") !=
14535 16249                              DDI_PROP_SUCCESS) {
14536 16250                                  mptsas_log(mpt, CE_WARN,
14537 16251                                      "mptsas unable to create "
↓ open down ↓ 18 lines elided ↑ open up ↑
14556 16270  phys_raid_lun:
14557 16271                  /*
14558 16272                   * if this is a SAS controller, and the target is a SATA
14559 16273                   * drive, set the 'pm-capable' property for sd and if on
14560 16274                   * an OPL platform, also check if this is an ATAPI
14561 16275                   * device.
14562 16276                   */
14563 16277                  instance = ddi_get_instance(mpt->m_dip);
14564 16278                  if (devinfo & (MPI2_SAS_DEVICE_INFO_SATA_DEVICE |
14565 16279                      MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE)) {
14566      -                        NDBG2(("mptsas%d: creating pm-capable property, "
     16280 +                        NDBG2(("mptsas3%d: creating pm-capable property, "
14567 16281                              "target %d", instance, target));
14568 16282  
14569 16283                          if ((ndi_prop_update_int(DDI_DEV_T_NONE,
14570 16284                              *lun_dip, "pm-capable", 1)) !=
14571 16285                              DDI_PROP_SUCCESS) {
14572 16286                                  mptsas_log(mpt, CE_WARN, "mptsas "
14573 16287                                      "failed to create pm-capable "
14574 16288                                      "property, target %d", target);
14575 16289                                  ndi_rtn = NDI_FAILURE;
14576 16290                                  goto phys_create_done;
14577 16291                          }
14578 16292  
14579 16293                  }
14580 16294  
14581 16295                  if ((inq->inq_dtype == 0) || (inq->inq_dtype == 5)) {
14582 16296                          /*
14583 16297                           * add 'obp-path' properties for devinfo
14584 16298                           */
14585 16299                          bzero(wwn_str, sizeof (wwn_str));
14586 16300                          (void) sprintf(wwn_str, "%016"PRIx64, sas_wwn);
14587      -                        component = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
14588 16301                          if (guid) {
14589 16302                                  (void) snprintf(component, MAXPATHLEN,
14590 16303                                      "disk@w%s,%x", wwn_str, lun);
14591 16304                          } else {
14592 16305                                  (void) snprintf(component, MAXPATHLEN,
14593 16306                                      "disk@p%x,%x", phy, lun);
14594 16307                          }
14595 16308                          if (ddi_pathname_obp_set(*lun_dip, component)
14596 16309                              != DDI_SUCCESS) {
14597 16310                                  mptsas_log(mpt, CE_WARN, "mpt_sas driver "
↓ open down ↓ 3 lines elided ↑ open up ↑
14601 16314                                  goto phys_create_done;
14602 16315                          }
14603 16316                  }
14604 16317                  /*
14605 16318                   * Create the phy-num property for non-raid disk
14606 16319                   */
14607 16320                  if (ptgt->m_addr.mta_phymask != 0) {
14608 16321                          if (ndi_prop_update_int(DDI_DEV_T_NONE,
14609 16322                              *lun_dip, "phy-num", ptgt->m_phynum) !=
14610 16323                              DDI_PROP_SUCCESS) {
14611      -                                mptsas_log(mpt, CE_WARN, "mptsas driver "
     16324 +                                mptsas_log(mpt, CE_WARN,
14612 16325                                      "failed to create phy-num property for "
14613 16326                                      "target %d", target);
14614 16327                                  ndi_rtn = NDI_FAILURE;
14615 16328                                  goto phys_create_done;
14616 16329                          }
14617 16330                  }
14618 16331  phys_create_done:
14619 16332                  /*
14620 16333                   * If props were setup ok, online the lun
14621 16334                   */
↓ open down ↓ 7 lines elided ↑ open up ↑
14629 16342                          mutex_enter(&mpt->m_mutex);
14630 16343                          ptgt->m_led_status = 0;
14631 16344                          (void) mptsas_flush_led_status(mpt, ptgt);
14632 16345                          mutex_exit(&mpt->m_mutex);
14633 16346                  }
14634 16347  
14635 16348                  /*
14636 16349                   * If success set rtn flag, else unwire alloc'd lun
14637 16350                   */
14638 16351                  if (ndi_rtn != NDI_SUCCESS) {
14639      -                        NDBG12(("mptsas driver unable to online "
     16352 +                        NDBG12(("unable to online "
14640 16353                              "target %d lun %d", target, lun));
14641 16354                          ndi_prop_remove_all(*lun_dip);
14642 16355                          (void) ndi_devi_free(*lun_dip);
14643 16356                          *lun_dip = NULL;
14644 16357                  }
14645 16358          }
14646 16359  
14647 16360          scsi_hba_nodename_compatible_free(nodename, compatible);
14648 16361  
14649      -        if (wwn_str != NULL) {
14650      -                kmem_free(wwn_str, MPTSAS_WWN_STRLEN);
14651      -        }
14652      -        if (component != NULL) {
14653      -                kmem_free(component, MAXPATHLEN);
14654      -        }
14655      -
14656      -
14657 16362          return ((ndi_rtn == NDI_SUCCESS) ? DDI_SUCCESS : DDI_FAILURE);
14658 16363  }
14659 16364  
14660 16365  static int
14661 16366  mptsas_probe_smp(dev_info_t *pdip, uint64_t wwn)
14662 16367  {
14663 16368          mptsas_t        *mpt = DIP2MPT(pdip);
14664 16369          struct smp_device smp_sd;
14665 16370  
14666 16371          /* XXX An HBA driver should not be allocating an smp_device. */
↓ open down ↓ 50 lines elided ↑ open up ↑
14717 16422          uint64_t        smp_sas_wwn;
14718 16423          uint8_t         physport;
14719 16424          uint8_t         phy_id;
14720 16425          uint16_t        pdev_hdl;
14721 16426          uint8_t         numphys = 0;
14722 16427          uint16_t        i = 0;
14723 16428          char            phymask[MPTSAS_MAX_PHYS];
14724 16429          char            *iport = NULL;
14725 16430          mptsas_phymask_t        phy_mask = 0;
14726 16431          uint16_t        attached_devhdl;
14727      -        uint16_t        bay_num, enclosure;
     16432 +        uint16_t        bay_num, enclosure, io_flags;
14728 16433  
14729 16434          (void) sprintf(wwn_str, "%"PRIx64, smp_node->m_addr.mta_wwn);
14730 16435  
14731 16436          /*
14732 16437           * Probe smp device, prevent the node of removed device from being
14733 16438           * configured succesfully
14734 16439           */
14735 16440          if (mptsas_probe_smp(pdip, smp_node->m_addr.mta_wwn) != NDI_SUCCESS) {
14736 16441                  return (DDI_FAILURE);
14737 16442          }
↓ open down ↓ 46 lines elided ↑ open up ↑
14784 16489                              "parent device info for %x", page_address);
14785 16490                          ndi_rtn = NDI_FAILURE;
14786 16491                          goto smp_create_done;
14787 16492                  }
14788 16493  
14789 16494                  smp_node->m_pdevhdl = dev_info.m_pdevhdl;
14790 16495                  page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
14791 16496                      MPI2_SAS_DEVICE_PGAD_FORM_MASK) |
14792 16497                      (uint32_t)dev_info.m_pdevhdl;
14793 16498                  rval = mptsas_get_sas_device_page0(mpt, page_address,
14794      -                    &dev_hdl, &sas_wwn, &smp_node->m_pdevinfo,
14795      -                    &physport, &phy_id, &pdev_hdl, &bay_num, &enclosure);
     16499 +                    &dev_hdl, &sas_wwn, &smp_node->m_pdevinfo, &physport,
     16500 +                    &phy_id, &pdev_hdl, &bay_num, &enclosure, &io_flags);
14796 16501                  if (rval != DDI_SUCCESS) {
14797 16502                          mutex_exit(&mpt->m_mutex);
14798 16503                          mptsas_log(mpt, CE_WARN, "mptsas unable to get "
14799 16504                              "device info for %x", page_address);
14800 16505                          ndi_rtn = NDI_FAILURE;
14801 16506                          goto smp_create_done;
14802 16507                  }
14803 16508  
14804 16509                  page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
14805 16510                      MPI2_SAS_DEVICE_PGAD_FORM_MASK) |
14806 16511                      (uint32_t)dev_info.m_devhdl;
14807 16512                  rval = mptsas_get_sas_device_page0(mpt, page_address,
14808 16513                      &dev_hdl, &smp_sas_wwn, &smp_node->m_deviceinfo,
14809      -                    &physport, &phy_id, &pdev_hdl, &bay_num, &enclosure);
     16514 +                    &physport, &phy_id, &pdev_hdl, &bay_num, &enclosure,
     16515 +                    &io_flags);
14810 16516                  if (rval != DDI_SUCCESS) {
14811 16517                          mutex_exit(&mpt->m_mutex);
14812 16518                          mptsas_log(mpt, CE_WARN, "mptsas unable to get "
14813 16519                              "device info for %x", page_address);
14814 16520                          ndi_rtn = NDI_FAILURE;
14815 16521                          goto smp_create_done;
14816 16522                  }
14817 16523                  mutex_exit(&mpt->m_mutex);
14818 16524  
14819 16525                  /*
↓ open down ↓ 133 lines elided ↑ open up ↑
14953 16659  
14954 16660          return ((ndi_rtn == NDI_SUCCESS) ? DDI_SUCCESS : DDI_FAILURE);
14955 16661  }
14956 16662  
14957 16663  /* smp transport routine */
14958 16664  static int mptsas_smp_start(struct smp_pkt *smp_pkt)
14959 16665  {
14960 16666          uint64_t                        wwn;
14961 16667          Mpi2SmpPassthroughRequest_t     req;
14962 16668          Mpi2SmpPassthroughReply_t       rep;
14963      -        uint32_t                        direction = 0;
     16669 +        uint8_t                         direction = 0;
14964 16670          mptsas_t                        *mpt;
14965 16671          int                             ret;
14966 16672          uint64_t                        tmp64;
14967 16673  
14968 16674          mpt = (mptsas_t *)smp_pkt->smp_pkt_address->
14969 16675              smp_a_hba_tran->smp_tran_hba_private;
14970 16676  
14971 16677          bcopy(smp_pkt->smp_pkt_address->smp_a_wwn, &wwn, SAS_WWN_BYTE_SIZE);
14972 16678          /*
14973 16679           * Need to compose a SMP request message
↓ open down ↓ 283 lines elided ↑ open up ↑
15257 16963          tmp_tgt->m_devhdl = devhdl;
15258 16964          tmp_tgt->m_addr.mta_wwn = wwid;
15259 16965          tmp_tgt->m_deviceinfo = devinfo;
15260 16966          tmp_tgt->m_addr.mta_phymask = phymask;
15261 16967          tmp_tgt->m_phynum = phynum;
15262 16968          /* Initialized the tgt structure */
15263 16969          tmp_tgt->m_qfull_retries = QFULL_RETRIES;
15264 16970          tmp_tgt->m_qfull_retry_interval =
15265 16971              drv_usectohz(QFULL_RETRY_INTERVAL * 1000);
15266 16972          tmp_tgt->m_t_throttle = MAX_THROTTLE;
     16973 +        mutex_init(&tmp_tgt->m_t_mutex, NULL, MUTEX_DRIVER, NULL);
     16974 +        TAILQ_INIT(&tmp_tgt->m_active_cmdq);
15267 16975  
15268 16976          refhash_insert(mpt->m_targets, tmp_tgt);
15269 16977  
15270 16978          return (tmp_tgt);
15271 16979  }
15272 16980  
     16981 +static void
     16982 +mptsas_smp_target_copy(mptsas_smp_t *src, mptsas_smp_t *dst)
     16983 +{
     16984 +        dst->m_devhdl = src->m_devhdl;
     16985 +        dst->m_deviceinfo = src->m_deviceinfo;
     16986 +        dst->m_pdevhdl = src->m_pdevhdl;
     16987 +        dst->m_pdevinfo = src->m_pdevinfo;
     16988 +}
     16989 +
15273 16990  static mptsas_smp_t *
15274 16991  mptsas_smp_alloc(mptsas_t *mpt, mptsas_smp_t *data)
15275 16992  {
15276 16993          mptsas_target_addr_t addr;
15277 16994          mptsas_smp_t *ret_data;
15278 16995  
15279 16996          addr.mta_wwn = data->m_addr.mta_wwn;
15280 16997          addr.mta_phymask = data->m_addr.mta_phymask;
15281 16998          ret_data = refhash_lookup(mpt->m_smp_targets, &addr);
     16999 +        /*
     17000 +         * If there's already a matching SMP target, update its fields
     17001 +         * in place.  Since the address is not changing, it's safe to do
     17002 +         * this.  We cannot just bcopy() here because the structure we've
     17003 +         * been given has invalid hash links.
     17004 +         */
15282 17005          if (ret_data != NULL) {
15283      -                bcopy(data, ret_data, sizeof (mptsas_smp_t)); /* XXX - dupl */
     17006 +                mptsas_smp_target_copy(data, ret_data);
15284 17007                  return (ret_data);
15285 17008          }
15286 17009  
15287 17010          ret_data = kmem_alloc(sizeof (mptsas_smp_t), KM_SLEEP);
15288 17011          bcopy(data, ret_data, sizeof (mptsas_smp_t));
15289 17012          refhash_insert(mpt->m_smp_targets, ret_data);
15290 17013          return (ret_data);
15291 17014  }
15292 17015  
15293 17016  /*
↓ open down ↓ 143 lines elided ↑ open up ↑
15437 17160  {
15438 17161          ddi_dma_cookie_t        new_cookie;
15439 17162          size_t                  alloc_len;
15440 17163          uint_t                  ncookie;
15441 17164  
15442 17165          if (cookiep == NULL)
15443 17166                  cookiep = &new_cookie;
15444 17167  
15445 17168          if (ddi_dma_alloc_handle(mpt->m_dip, &dma_attr, DDI_DMA_SLEEP,
15446 17169              NULL, dma_hdp) != DDI_SUCCESS) {
15447      -                dma_hdp = NULL;
15448 17170                  return (FALSE);
15449 17171          }
15450 17172  
15451 17173          if (ddi_dma_mem_alloc(*dma_hdp, alloc_size, &mpt->m_dev_acc_attr,
15452 17174              DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, dma_memp, &alloc_len,
15453 17175              acc_hdp) != DDI_SUCCESS) {
15454 17176                  ddi_dma_free_handle(dma_hdp);
15455      -                dma_hdp = NULL;
15456 17177                  return (FALSE);
15457 17178          }
15458 17179  
15459 17180          if (ddi_dma_addr_bind_handle(*dma_hdp, NULL, *dma_memp, alloc_len,
15460 17181              (DDI_DMA_RDWR | DDI_DMA_CONSISTENT), DDI_DMA_SLEEP, NULL,
15461 17182              cookiep, &ncookie) != DDI_DMA_MAPPED) {
15462 17183                  (void) ddi_dma_mem_free(acc_hdp);
15463 17184                  ddi_dma_free_handle(dma_hdp);
15464      -                dma_hdp = NULL;
15465 17185                  return (FALSE);
15466 17186          }
15467 17187  
15468 17188          return (TRUE);
15469 17189  }
15470 17190  
15471 17191  void
15472 17192  mptsas_dma_addr_destroy(ddi_dma_handle_t *dma_hdp, ddi_acc_handle_t *acc_hdp)
15473 17193  {
15474 17194          if (*dma_hdp == NULL)
15475 17195                  return;
15476 17196  
15477 17197          (void) ddi_dma_unbind_handle(*dma_hdp);
15478 17198          (void) ddi_dma_mem_free(acc_hdp);
15479 17199          ddi_dma_free_handle(dma_hdp);
15480      -        dma_hdp = NULL;
15481 17200  }
    
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX