1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  25  */
  26 
  27 /*
  28  * AHCI (Advanced Host Controller Interface) SATA HBA Driver
  29  *
  30  * Power Management Support
  31  * ------------------------
  32  *
  33  * At the moment, the ahci driver only implements suspend/resume to
  34  * support Suspend to RAM on X86 feature. Device power management isn't
  35  * implemented, link power management is disabled, and hot plug isn't
  36  * allowed during the period from suspend to resume.
  37  *
  38  * For s/r support, the ahci driver only need to implement DDI_SUSPEND
  39  * and DDI_RESUME entries, and don't need to take care of new requests
  40  * sent down after suspend because the target driver (sd) has already
  41  * handled these conditions, and blocked these requests. For the detailed
  42  * information, please check with sdopen, sdclose and sdioctl routines.
  43  *
  44  */
  45 
  46 #include <sys/note.h>
  47 #include <sys/scsi/scsi.h>
  48 #include <sys/pci.h>
  49 #include <sys/disp.h>
  50 #include <sys/sata/sata_hba.h>
  51 #include <sys/sata/adapters/ahci/ahcireg.h>
  52 #include <sys/sata/adapters/ahci/ahcivar.h>
  53 
  54 /*
  55  * FMA header files
  56  */
  57 #include <sys/ddifm.h>
  58 #include <sys/fm/protocol.h>
  59 #include <sys/fm/util.h>
  60 #include <sys/fm/io/ddi.h>
  61 
  62 /*
  63  * This is the string displayed by modinfo, etc.
  64  */
  65 static char ahci_ident[] = "ahci driver";
  66 
  67 /*
  68  * Function prototypes for driver entry points
  69  */
  70 static  int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
  71 static  int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
  72 static  int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
  73 static  int ahci_quiesce(dev_info_t *);
  74 
  75 /*
  76  * Function prototypes for SATA Framework interfaces
  77  */
  78 static  int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
  79 static  int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
  80 
  81 static  int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
  82 static  int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
  83 static  int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
  84 static  int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
  85 static  int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
  86 static  int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
  87 #if defined(__lock_lint)
  88 static  int ahci_selftest(dev_info_t *, sata_device_t *);
  89 #endif
  90 
  91 /*
  92  * FMA Prototypes
  93  */
  94 static  void ahci_fm_init(ahci_ctl_t *);
  95 static  void ahci_fm_fini(ahci_ctl_t *);
  96 static  int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
  97 int     ahci_check_acc_handle(ddi_acc_handle_t);
  98 int     ahci_check_dma_handle(ddi_dma_handle_t);
  99 void    ahci_fm_ereport(ahci_ctl_t *, char *);
 100 static  int ahci_check_all_handle(ahci_ctl_t *);
 101 static  int ahci_check_ctl_handle(ahci_ctl_t *);
 102 static  int ahci_check_port_handle(ahci_ctl_t *, int);
 103 static  int ahci_check_slot_handle(ahci_port_t *, int);
 104 
 105 /*
 106  * Local function prototypes
 107  */
 108 static  int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
 109 static  int ahci_alloc_ports_state(ahci_ctl_t *);
 110 static  void ahci_dealloc_ports_state(ahci_ctl_t *);
 111 static  int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
 112 static  void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
 113 static  int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
 114 static  void ahci_dealloc_rcvd_fis(ahci_port_t *);
 115 static  int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
 116 static  void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
 117 static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
 118 static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
 119 static  void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
 120 static  void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
 121 
 122 static  int ahci_initialize_controller(ahci_ctl_t *);
 123 static  void ahci_uninitialize_controller(ahci_ctl_t *);
 124 static  int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 125 static  int ahci_config_space_init(ahci_ctl_t *);
 126 static  void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
 127 
 128 static  void ahci_drain_ports_taskq(ahci_ctl_t *);
 129 static  int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
 130     uint8_t);
 131 static  int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
 132 static  int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
 133 static  int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
 134     sata_device_t *);
 135 static  int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
 136     sata_pmult_gscr_t *);
 137 static  int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
 138     sata_device_t *);
 139 static  int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 140 static  int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 141 static  int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
 142     sata_device_t *);
 143 
 144 static  void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
 145 static  int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
 146 static  void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
 147     ahci_addr_t *);
 148 static  void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
 149 static  int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
 150     ahci_addr_t *, sata_pkt_t *);
 151 static  int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
 152     ahci_addr_t *, sata_pkt_t *);
 153 static  int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
 154     ahci_addr_t *, int);
 155 static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
 156 static  void ahci_copy_ncq_err_page(sata_cmd_t *,
 157     struct sata_ncq_error_recovery_page *);
 158 static  void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
 159 static  void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
 160 static  void ahci_flush_doneq(ahci_port_t *);
 161 
 162 static  int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 163 static  int ahci_hba_reset(ahci_ctl_t *);
 164 static  int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 165 static  int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 166 static  void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
 167 static  int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 168     ahci_addr_t *);
 169 static  int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 170     ahci_addr_t *);
 171 static  int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 172     ahci_addr_t *);
 173 static  int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
 174 static  int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
 175     uint8_t);
 176 static  int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
 177     uint8_t, int, int *);
 178 static  void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
 179     uint32_t, uint32_t, uint32_t, uint32_t);
 180 static  uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
 181 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
 182     uint8_t, sata_pkt_t *);
 183 static  void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
 184     ahci_addr_t *, uint32_t);
 185 static  void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
 186     uint8_t, uint32_t);
 187 static  void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
 188     uint8_t, uint32_t);
 189 static  void ahci_events_handler(void *);
 190 static  void ahci_watchdog_handler(ahci_ctl_t *);
 191 
 192 static  uint_t ahci_intr(caddr_t, caddr_t);
 193 static  void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
 194 static  int ahci_add_intrs(ahci_ctl_t *, int);
 195 static  void ahci_rem_intrs(ahci_ctl_t *);
 196 static  void ahci_enable_all_intrs(ahci_ctl_t *);
 197 static  void ahci_disable_all_intrs(ahci_ctl_t *);
 198 static  void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
 199 static  void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
 200 
 201 static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
 202 static  int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
 203 static  int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 204 static  int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
 205 static  int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
 206 static  int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
 207     ahci_port_t *, uint8_t);
 208 static  int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
 209 static  int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
 210     uint8_t, uint32_t);
 211 static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
 212     uint8_t, uint32_t);
 213 static  int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
 214 
 215 static  void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
 216 static  int ahci_get_num_implemented_ports(uint32_t);
 217 static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
 218 static  void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
 219 static  void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
 220 #if AHCI_DEBUG
 221 static  void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
 222 #endif
 223 
 224 
 225 /*
 226  * DMA attributes for the data buffer
 227  *
 228  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 229  * does not support 64-bit addressing
 230  */
 231 static ddi_dma_attr_t buffer_dma_attr = {
 232         DMA_ATTR_V0,            /* dma_attr_version */
 233         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 234         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 235         0x3fffffull,            /* dma_attr_count_max i.e. for one cookie */
 236         0x2ull,                 /* dma_attr_align: word aligned */
 237         1,                      /* dma_attr_burstsizes */
 238         1,                      /* dma_attr_minxfer */
 239         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 240         0xffffffffull,          /* dma_attr_seg */
 241         AHCI_PRDT_NUMBER,       /* dma_attr_sgllen */
 242         512,                    /* dma_attr_granular */
 243         0,                      /* dma_attr_flags */
 244 };
 245 
 246 /*
 247  * DMA attributes for the rcvd FIS
 248  *
 249  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 250  * does not support 64-bit addressing
 251  */
 252 static ddi_dma_attr_t rcvd_fis_dma_attr = {
 253         DMA_ATTR_V0,            /* dma_attr_version */
 254         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 255         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 256         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 257         0x100ull,               /* dma_attr_align: 256-byte aligned */
 258         1,                      /* dma_attr_burstsizes */
 259         1,                      /* dma_attr_minxfer */
 260         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 261         0xffffffffull,          /* dma_attr_seg */
 262         1,                      /* dma_attr_sgllen */
 263         1,                      /* dma_attr_granular */
 264         0,                      /* dma_attr_flags */
 265 };
 266 
 267 /*
 268  * DMA attributes for the command list
 269  *
 270  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 271  * does not support 64-bit addressing
 272  */
 273 static ddi_dma_attr_t cmd_list_dma_attr = {
 274         DMA_ATTR_V0,            /* dma_attr_version */
 275         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 276         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 277         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 278         0x400ull,               /* dma_attr_align: 1K-byte aligned */
 279         1,                      /* dma_attr_burstsizes */
 280         1,                      /* dma_attr_minxfer */
 281         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 282         0xffffffffull,          /* dma_attr_seg */
 283         1,                      /* dma_attr_sgllen */
 284         1,                      /* dma_attr_granular */
 285         0,                      /* dma_attr_flags */
 286 };
 287 
 288 /*
 289  * DMA attributes for cmd tables
 290  *
 291  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 292  * does not support 64-bit addressing
 293  */
 294 static ddi_dma_attr_t cmd_table_dma_attr = {
 295         DMA_ATTR_V0,            /* dma_attr_version */
 296         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 297         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 298         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 299         0x80ull,                /* dma_attr_align: 128-byte aligned */
 300         1,                      /* dma_attr_burstsizes */
 301         1,                      /* dma_attr_minxfer */
 302         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 303         0xffffffffull,          /* dma_attr_seg */
 304         1,                      /* dma_attr_sgllen */
 305         1,                      /* dma_attr_granular */
 306         0,                      /* dma_attr_flags */
 307 };
 308 
 309 
 310 /* Device access attributes */
 311 static ddi_device_acc_attr_t accattr = {
 312         DDI_DEVICE_ATTR_V1,
 313         DDI_STRUCTURE_LE_ACC,
 314         DDI_STRICTORDER_ACC,
 315         DDI_DEFAULT_ACC
 316 };
 317 
 318 
 319 static struct dev_ops ahcictl_dev_ops = {
 320         DEVO_REV,               /* devo_rev */
 321         0,                      /* refcnt  */
 322         ahci_getinfo,           /* info */
 323         nulldev,                /* identify */
 324         nulldev,                /* probe */
 325         ahci_attach,            /* attach */
 326         ahci_detach,            /* detach */
 327         nodev,                  /* no reset */
 328         (struct cb_ops *)0,     /* driver operations */
 329         NULL,                   /* bus operations */
 330         NULL,                   /* power */
 331         ahci_quiesce,           /* quiesce */
 332 };
 333 
 334 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
 335         SATA_TRAN_HOTPLUG_OPS_REV_1,
 336         ahci_tran_hotplug_port_activate,
 337         ahci_tran_hotplug_port_deactivate
 338 };
 339 
 340 extern struct mod_ops mod_driverops;
 341 
 342 static  struct modldrv modldrv = {
 343         &mod_driverops,             /* driverops */
 344         ahci_ident,             /* short description */
 345         &ahcictl_dev_ops,   /* driver ops */
 346 };
 347 
 348 static  struct modlinkage modlinkage = {
 349         MODREV_1,
 350         &modldrv,
 351         NULL
 352 };
 353 
 354 /* The following variables are watchdog handler related */
 355 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
 356 static clock_t ahci_watchdog_tick;
 357 
 358 /*
 359  * This static variable indicates the size of command table,
 360  * and it's changeable with prdt number, which ahci_dma_prdt_number
 361  * indicates.
 362  */
 363 static size_t ahci_cmd_table_size;
 364 
 365 /*
 366  * The below global variables are tunable via /etc/system
 367  *
 368  *      ahci_dma_prdt_number
 369  *      ahci_msi_enabled
 370  *      ahci_buf_64bit_dma
 371  *      ahci_commu_64bit_dma
 372  */
 373 
 374 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
 375 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
 376 
 377 /* AHCI MSI is tunable */
 378 boolean_t ahci_msi_enabled = B_TRUE;
 379 
 380 /*
 381  * 64-bit dma addressing for data buffer is tunable
 382  *
 383  * The variable controls only the below value:
 384  *      DBAU (upper 32-bits physical address of data block)
 385  */
 386 boolean_t ahci_buf_64bit_dma = B_TRUE;
 387 
 388 /*
 389  * 64-bit dma addressing for communication system descriptors is tunable
 390  *
 391  * The variable controls the below three values:
 392  *
 393  *      PxCLBU (upper 32-bits for the command list base physical address)
 394  *      PxFBU (upper 32-bits for the received FIS base physical address)
 395  *      CTBAU (upper 32-bits of command table base)
 396  */
 397 boolean_t ahci_commu_64bit_dma = B_TRUE;
 398 
 399 /*
 400  * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
 401  * chipset. If the users want to have a try with 64-bit dma, please change
 402  * the below variable value to enable it.
 403  */
 404 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
 405 
 406 /*
 407  * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
 408  * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
 409  * please change the below value to enable it.
 410  */
 411 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
 412 
 413 
 414 /*
 415  * End of global tunable variable definition
 416  */
 417 
 418 #if AHCI_DEBUG
 419 uint32_t ahci_debug_flags = 0;
 420 #else
 421 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
 422 #endif
 423 
 424 
 425 #if AHCI_DEBUG
 426 /* The following is needed for ahci_log() */
 427 static kmutex_t ahci_log_mutex;
 428 static char ahci_log_buf[512];
 429 #endif
 430 
 431 /* Opaque state pointer initialized by ddi_soft_state_init() */
 432 static void *ahci_statep = NULL;
 433 
 434 /*
 435  *  ahci module initialization.
 436  */
 437 int
 438 _init(void)
 439 {
 440         int     ret;
 441 
 442         ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
 443         if (ret != 0) {
 444                 goto err_out;
 445         }
 446 
 447 #if AHCI_DEBUG
 448         mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
 449 #endif
 450 
 451         if ((ret = sata_hba_init(&modlinkage)) != 0) {
 452 #if AHCI_DEBUG
 453                 mutex_destroy(&ahci_log_mutex);
 454 #endif
 455                 ddi_soft_state_fini(&ahci_statep);
 456                 goto err_out;
 457         }
 458 
 459         /* watchdog tick */
 460         ahci_watchdog_tick = drv_usectohz(
 461             (clock_t)ahci_watchdog_timeout * 1000000);
 462 
 463         ret = mod_install(&modlinkage);
 464         if (ret != 0) {
 465                 sata_hba_fini(&modlinkage);
 466 #if AHCI_DEBUG
 467                 mutex_destroy(&ahci_log_mutex);
 468 #endif
 469                 ddi_soft_state_fini(&ahci_statep);
 470                 goto err_out;
 471         }
 472 
 473         return (ret);
 474 
 475 err_out:
 476         cmn_err(CE_WARN, "!ahci: Module init failed");
 477         return (ret);
 478 }
 479 
 480 /*
 481  * ahci module uninitialize.
 482  */
 483 int
 484 _fini(void)
 485 {
 486         int     ret;
 487 
 488         ret = mod_remove(&modlinkage);
 489         if (ret != 0) {
 490                 return (ret);
 491         }
 492 
 493         /* Remove the resources allocated in _init(). */
 494         sata_hba_fini(&modlinkage);
 495 #if AHCI_DEBUG
 496         mutex_destroy(&ahci_log_mutex);
 497 #endif
 498         ddi_soft_state_fini(&ahci_statep);
 499 
 500         return (ret);
 501 }
 502 
 503 /*
 504  * _info entry point
 505  */
 506 int
 507 _info(struct modinfo *modinfop)
 508 {
 509         return (mod_info(&modlinkage, modinfop));
 510 }
 511 
 512 /*
 513  * The attach entry point for dev_ops.
 514  */
 515 static int
 516 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 517 {
 518         ahci_ctl_t *ahci_ctlp = NULL;
 519         int instance = ddi_get_instance(dip);
 520         int status;
 521         int attach_state;
 522         uint32_t cap_status, ahci_version;
 523         uint32_t ghc_control;
 524         int intr_types;
 525         int i;
 526         pci_regspec_t *regs;
 527         int regs_length;
 528         int rnumber;
 529 #if AHCI_DEBUG
 530         int speed;
 531 #endif
 532 
 533         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
 534             NULL);
 535 
 536         switch (cmd) {
 537         case DDI_ATTACH:
 538                 break;
 539 
 540         case DDI_RESUME:
 541 
 542                 /*
 543                  * During DDI_RESUME, the hardware state of the device
 544                  * (power may have been removed from the device) must be
 545                  * restored, allow pending requests to continue, and
 546                  * service new requests.
 547                  */
 548                 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
 549                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 550 
 551                 /*
 552                  * GHC.AE must be set to 1 before any other AHCI register
 553                  * is accessed
 554                  */
 555                 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 556                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
 557                 ghc_control |= AHCI_HBA_GHC_AE;
 558                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
 559                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
 560 
 561                 /* Restart watch thread */
 562                 if (ahci_ctlp->ahcictl_timeout_id == 0)
 563                         ahci_ctlp->ahcictl_timeout_id = timeout(
 564                             (void (*)(void *))ahci_watchdog_handler,
 565                             (caddr_t)ahci_ctlp, ahci_watchdog_tick);
 566 
 567                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 568 
 569                 /*
 570                  * Re-initialize the controller and enable the interrupts and
 571                  * restart all the ports.
 572                  *
 573                  * Note that so far we don't support hot-plug during
 574                  * suspend/resume.
 575                  */
 576                 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
 577                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
 578                             "Failed to initialize the controller "
 579                             "during DDI_RESUME", NULL);
 580                         return (DDI_FAILURE);
 581                 }
 582 
 583                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 584                 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
 585                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 586 
 587                 return (DDI_SUCCESS);
 588 
 589         default:
 590                 return (DDI_FAILURE);
 591         }
 592 
 593         attach_state = AHCI_ATTACH_STATE_NONE;
 594 
 595         /* Allocate soft state */
 596         status = ddi_soft_state_zalloc(ahci_statep, instance);
 597         if (status != DDI_SUCCESS) {
 598                 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
 599                     instance);
 600                 goto err_out;
 601         }
 602 
 603         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
 604         ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
 605         ahci_ctlp->ahcictl_dip = dip;
 606 
 607         /* Initialize the cport/port mapping */
 608         for (i = 0; i < AHCI_MAX_PORTS; i++) {
 609                 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
 610                 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
 611         }
 612 
 613         attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
 614 
 615         /* Initialize FMA properties */
 616         ahci_fm_init(ahci_ctlp);
 617 
 618         attach_state |= AHCI_ATTACH_STATE_FMA;
 619 
 620         /*
 621          * Now map the AHCI base address; which includes global
 622          * registers and port control registers
 623          *
 624          * According to the spec, the AHCI Base Address is BAR5,
 625          * but BAR0-BAR4 are optional, so we need to check which
 626          * rnumber is used for BAR5.
 627          */
 628 
 629         /*
 630          * search through DDI "reg" property for the AHCI register set
 631          */
 632         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
 633             DDI_PROP_DONTPASS, "reg", (int **)&regs,
 634             (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
 635                 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
 636                     instance);
 637                 goto err_out;
 638         }
 639 
 640         /* AHCI Base Address is located at 0x24 offset */
 641         for (rnumber = 0; rnumber < regs_length; ++rnumber) {
 642                 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
 643                     == AHCI_PCI_RNUM)
 644                         break;
 645         }
 646 
 647         ddi_prop_free(regs);
 648 
 649         if (rnumber == regs_length) {
 650                 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
 651                     instance);
 652                 goto err_out;
 653         }
 654 
 655         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
 656 
 657         status = ddi_regs_map_setup(dip,
 658             rnumber,
 659             (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
 660             0,
 661             0,
 662             &accattr,
 663             &ahci_ctlp->ahcictl_ahci_acc_handle);
 664         if (status != DDI_SUCCESS) {
 665                 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
 666                     instance);
 667                 goto err_out;
 668         }
 669 
 670         attach_state |= AHCI_ATTACH_STATE_REG_MAP;
 671 
 672         /*
 673          * GHC.AE must be set to 1 before any other AHCI register
 674          * is accessed
 675          */
 676         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 677             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
 678         ghc_control |= AHCI_HBA_GHC_AE;
 679         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
 680             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
 681 
 682         /* Get the AHCI version information */
 683         ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 684             (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
 685 
 686         cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
 687             (ahci_version & 0xffff0000) >> 16,
 688             ((ahci_version & 0x0000ff00) >> 4 |
 689             (ahci_version & 0x000000ff)));
 690 
 691         /* We don't support controllers whose versions are lower than 1.0 */
 692         if (!(ahci_version & 0xffff0000)) {
 693                 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
 694                     "than version 1.0", instance);
 695                 goto err_out;
 696         }
 697 
 698         /* Get the HBA capabilities information */
 699         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 700             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
 701 
 702         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
 703             cap_status);
 704 
 705         /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
 706         if (ahci_version >= 0x00010200) {
 707                 uint32_t cap2_status;
 708 
 709                 /* Get the HBA capabilities extended information */
 710                 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 711                     (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
 712 
 713                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 714                     "hba capabilities extended = 0x%x", cap2_status);
 715         }
 716 
 717 #if AHCI_DEBUG
 718         /* Get the interface speed supported by the HBA */
 719         speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
 720         if (speed == 0x01) {
 721                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 722                     "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
 723         } else if (speed == 0x10) {
 724                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 725                     "hba interface speed support: Gen 2 (3 Gbps)", NULL);
 726         } else if (speed == 0x11) {
 727                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 728                     "hba interface speed support: Gen 3 (6 Gbps)", NULL);
 729         }
 730 #endif
 731 
 732         /* Get the number of command slots supported by the HBA */
 733         ahci_ctlp->ahcictl_num_cmd_slots =
 734             ((cap_status & AHCI_HBA_CAP_NCS) >>
 735             AHCI_HBA_CAP_NCS_SHIFT) + 1;
 736 
 737         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
 738             ahci_ctlp->ahcictl_num_cmd_slots);
 739 
 740         /* Get the bit map which indicates ports implemented by the HBA */
 741         ahci_ctlp->ahcictl_ports_implemented =
 742             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 743             (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
 744 
 745         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
 746             ahci_ctlp->ahcictl_ports_implemented);
 747 
 748         /* Max port number implemented */
 749         ahci_ctlp->ahcictl_num_ports =
 750             ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
 751 
 752         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
 753             (cap_status & AHCI_HBA_CAP_NP) + 1);
 754 
 755         /* Get the number of implemented ports by the HBA */
 756         ahci_ctlp->ahcictl_num_implemented_ports =
 757             ahci_get_num_implemented_ports(
 758             ahci_ctlp->ahcictl_ports_implemented);
 759 
 760         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 761             "hba number of implemented ports: %d",
 762             ahci_ctlp->ahcictl_num_implemented_ports);
 763 
 764         /* Check whether HBA supports 64bit DMA addressing */
 765         if (!(cap_status & AHCI_HBA_CAP_S64A)) {
 766                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
 767                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
 768                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 769                     "hba does not support 64-bit addressing", NULL);
 770         }
 771 
 772         /* Checking for the support of Port Multiplier */
 773         if (cap_status & AHCI_HBA_CAP_SPM) {
 774                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
 775                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 776                     "hba supports port multiplier (CBSS)", NULL);
 777 
 778                 /* Support FIS-based switching ? */
 779                 if (cap_status & AHCI_HBA_CAP_FBSS) {
 780                         ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
 781                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 782                             "hba supports FIS-based switching (FBSS)", NULL);
 783                 }
 784         }
 785 
 786         /* Checking for Support Command List Override */
 787         if (cap_status & AHCI_HBA_CAP_SCLO) {
 788                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
 789                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 790                     "hba supports command list override.", NULL);
 791         }
 792 
 793         /* Checking for Asynchronous Notification */
 794         if (cap_status & AHCI_HBA_CAP_SSNTF) {
 795                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
 796                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 797                     "hba supports asynchronous notification.", NULL);
 798         }
 799 
 800         if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
 801             != DDI_SUCCESS) {
 802                 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
 803                     instance);
 804                 goto err_out;
 805         }
 806 
 807         attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
 808 
 809         /*
 810          * Check the pci configuration space, and set caps. We also
 811          * handle the hardware defect in this function.
 812          *
 813          * For example, force ATI SB600 to use 32-bit dma addressing
 814          * since it doesn't support 64-bit dma though its CAP register
 815          * declares it support.
 816          */
 817         if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
 818                 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
 819                     instance);
 820                 goto err_out;
 821         }
 822 
 823         /*
 824          * Disable the whole controller interrupts before adding
 825          * interrupt handlers(s).
 826          */
 827         ahci_disable_all_intrs(ahci_ctlp);
 828 
 829         /* Get supported interrupt types */
 830         if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
 831                 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
 832                     instance);
 833                 goto err_out;
 834         }
 835 
 836         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 837             "ddi_intr_get_supported_types() returned: 0x%x",
 838             intr_types);
 839 
 840         if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
 841                 /*
 842                  * Try MSI first, but fall back to FIXED if failed
 843                  */
 844                 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
 845                     DDI_SUCCESS) {
 846                         ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
 847                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 848                             "Using MSI interrupt type", NULL);
 849                         goto intr_done;
 850                 }
 851 
 852                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 853                     "MSI registration failed, "
 854                     "trying FIXED interrupts", NULL);
 855         }
 856 
 857         if (intr_types & DDI_INTR_TYPE_FIXED) {
 858                 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
 859                     DDI_SUCCESS) {
 860                         ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
 861                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 862                             "Using FIXED interrupt type", NULL);
 863                         goto intr_done;
 864                 }
 865 
 866                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 867                     "FIXED interrupt registration failed", NULL);
 868         }
 869 
 870         cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
 871 
 872         goto err_out;
 873 
 874 intr_done:
 875 
 876         attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
 877 
 878         /* Initialize the controller mutex */
 879         mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
 880             (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
 881 
 882         attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
 883 
 884         if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
 885                 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
 886         } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
 887                 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
 888         }
 889 
 890         ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
 891             (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
 892             sizeof (ahci_prdt_item_t));
 893 
 894         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 895             "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
 896             " ahci_cmd_table_size is 0x%x",
 897             ahci_dma_prdt_number, ahci_cmd_table_size);
 898 
 899         if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
 900                 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
 901                     ahci_dma_prdt_number;
 902 
 903         ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
 904         ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
 905         ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
 906         ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
 907 
 908         /*
 909          * enable 64bit dma for data buffer for SB600 if
 910          * sb600_buf_64bit_dma_disable is B_FALSE
 911          */
 912         if ((ahci_buf_64bit_dma == B_FALSE) ||
 913             ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
 914             !(sb600_buf_64bit_dma_disable == B_FALSE &&
 915             ahci_ctlp->ahcictl_venid == 0x1002 &&
 916             ahci_ctlp->ahcictl_devid == 0x4380))) {
 917                 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
 918                     0xffffffffull;
 919         }
 920 
 921         /*
 922          * enable 64bit dma for command buffer for SB600/700/710/800
 923          * if sbxxx_commu_64bit_dma_disable is B_FALSE
 924          */
 925         if ((ahci_commu_64bit_dma == B_FALSE) ||
 926             ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
 927             !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
 928             ahci_ctlp->ahcictl_venid == 0x1002 &&
 929             (ahci_ctlp->ahcictl_devid == 0x4380 ||
 930             ahci_ctlp->ahcictl_devid == 0x4391)))) {
 931                 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
 932                     0xffffffffull;
 933                 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
 934                     0xffffffffull;
 935                 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
 936                     0xffffffffull;
 937         }
 938 
 939         /* Allocate the ports structure */
 940         status = ahci_alloc_ports_state(ahci_ctlp);
 941         if (status != AHCI_SUCCESS) {
 942                 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
 943                     instance);
 944                 goto err_out;
 945         }
 946 
 947         attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
 948 
 949         /*
 950          * Initialize the controller and ports.
 951          */
 952         status = ahci_initialize_controller(ahci_ctlp);
 953         if (status != AHCI_SUCCESS) {
 954                 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
 955                     instance);
 956                 goto err_out;
 957         }
 958 
 959         attach_state |= AHCI_ATTACH_STATE_HW_INIT;
 960 
 961         /* Start one thread to check packet timeouts */
 962         ahci_ctlp->ahcictl_timeout_id = timeout(
 963             (void (*)(void *))ahci_watchdog_handler,
 964             (caddr_t)ahci_ctlp, ahci_watchdog_tick);
 965 
 966         attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
 967 
 968         if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
 969                 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
 970                     instance);
 971                 goto err_out;
 972         }
 973 
 974         /* Check all handles at the end of the attach operation. */
 975         if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
 976                 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
 977                     instance);
 978                 goto err_out;
 979         }
 980 
 981         ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
 982 
 983         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
 984 
 985         return (DDI_SUCCESS);
 986 
 987 err_out:
 988         /* FMA message */
 989         ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
 990         ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
 991 
 992         if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
 993                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 994                 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
 995                 ahci_ctlp->ahcictl_timeout_id = 0;
 996                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 997         }
 998 
 999         if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1000                 ahci_uninitialize_controller(ahci_ctlp);
1001         }
1002 
1003         if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1004                 ahci_dealloc_ports_state(ahci_ctlp);
1005         }
1006 
1007         if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1008                 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1009         }
1010 
1011         if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1012                 ahci_rem_intrs(ahci_ctlp);
1013         }
1014 
1015         if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1016                 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1017         }
1018 
1019         if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1020                 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1021         }
1022 
1023         if (attach_state & AHCI_ATTACH_STATE_FMA) {
1024                 ahci_fm_fini(ahci_ctlp);
1025         }
1026 
1027         if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1028                 ddi_soft_state_free(ahci_statep, instance);
1029         }
1030 
1031         return (DDI_FAILURE);
1032 }
1033 
1034 /*
1035  * The detach entry point for dev_ops.
1036  */
1037 static int
1038 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1039 {
1040         ahci_ctl_t *ahci_ctlp;
1041         int instance;
1042         int ret;
1043 
1044         instance = ddi_get_instance(dip);
1045         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1046 
1047         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1048 
1049         switch (cmd) {
1050         case DDI_DETACH:
1051 
1052                 /* disable the interrupts for an uninterrupted detach */
1053                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1054                 ahci_disable_all_intrs(ahci_ctlp);
1055                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1056 
1057                 /* unregister from the sata framework. */
1058                 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1059                 if (ret != AHCI_SUCCESS) {
1060                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1061                         ahci_enable_all_intrs(ahci_ctlp);
1062                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1063                         return (DDI_FAILURE);
1064                 }
1065 
1066                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1067 
1068                 /* stop the watchdog handler */
1069                 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1070                 ahci_ctlp->ahcictl_timeout_id = 0;
1071 
1072                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1073 
1074                 /* uninitialize the controller */
1075                 ahci_uninitialize_controller(ahci_ctlp);
1076 
1077                 /* remove the interrupts */
1078                 ahci_rem_intrs(ahci_ctlp);
1079 
1080                 /* deallocate the ports structures */
1081                 ahci_dealloc_ports_state(ahci_ctlp);
1082 
1083                 /* destroy mutex */
1084                 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1085 
1086                 /* teardown the pci config */
1087                 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1088 
1089                 /* remove the reg maps. */
1090                 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1091 
1092                 /* release fma resource */
1093                 ahci_fm_fini(ahci_ctlp);
1094 
1095                 /* free the soft state. */
1096                 ddi_soft_state_free(ahci_statep, instance);
1097 
1098                 return (DDI_SUCCESS);
1099 
1100         case DDI_SUSPEND:
1101 
1102                 /*
1103                  * The steps associated with suspension must include putting
1104                  * the underlying device into a quiescent state so that it
1105                  * will not generate interrupts or modify or access memory.
1106                  */
1107                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1108                 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1109                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1110                         return (DDI_SUCCESS);
1111                 }
1112 
1113                 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1114 
1115                 /* stop the watchdog handler */
1116                 if (ahci_ctlp->ahcictl_timeout_id) {
1117                         (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1118                         ahci_ctlp->ahcictl_timeout_id = 0;
1119                 }
1120 
1121                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1122 
1123                 /*
1124                  * drain the taskq
1125                  */
1126                 ahci_drain_ports_taskq(ahci_ctlp);
1127 
1128                 /*
1129                  * Disable the interrupts and stop all the ports.
1130                  */
1131                 ahci_uninitialize_controller(ahci_ctlp);
1132 
1133                 return (DDI_SUCCESS);
1134 
1135         default:
1136                 return (DDI_FAILURE);
1137         }
1138 }
1139 
1140 /*
1141  * The info entry point for dev_ops.
1142  *
1143  */
1144 static int
1145 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1146                     void *arg, void **result)
1147 {
1148 #ifndef __lock_lint
1149         _NOTE(ARGUNUSED(dip))
1150 #endif /* __lock_lint */
1151 
1152         ahci_ctl_t *ahci_ctlp;
1153         int instance;
1154         dev_t dev;
1155 
1156         dev = (dev_t)arg;
1157         instance = getminor(dev);
1158 
1159         switch (infocmd) {
1160                 case DDI_INFO_DEVT2DEVINFO:
1161                         ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
1162                         if (ahci_ctlp != NULL) {
1163                                 *result = ahci_ctlp->ahcictl_dip;
1164                                 return (DDI_SUCCESS);
1165                         } else {
1166                                 *result = NULL;
1167                                 return (DDI_FAILURE);
1168                         }
1169                 case DDI_INFO_DEVT2INSTANCE:
1170                         *(int *)result = instance;
1171                         break;
1172                 default:
1173                         break;
1174         }
1175 
1176         return (DDI_SUCCESS);
1177 }
1178 
1179 /*
1180  * Registers the ahci with sata framework.
1181  */
1182 static int
1183 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1184 {
1185         struct  sata_hba_tran   *sata_hba_tran;
1186 
1187         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1188             "ahci_register_sata_hba_tran enter", NULL);
1189 
1190         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1191 
1192         /* Allocate memory for the sata_hba_tran  */
1193         sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1194 
1195         sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1196         sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1197         sata_hba_tran->sata_tran_hba_dma_attr =
1198             &ahci_ctlp->ahcictl_buffer_dma_attr;
1199 
1200         /* Report the number of implemented ports */
1201         sata_hba_tran->sata_tran_hba_num_cports =
1202             ahci_ctlp->ahcictl_num_implemented_ports;
1203 
1204         /* Support ATAPI device */
1205         sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1206 
1207         /* Get the data transfer capability for PIO command by the HBA */
1208         if (cap_status & AHCI_HBA_CAP_PMD) {
1209                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1210                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1211                     "DRQ block data transfer for PIO command protocol", NULL);
1212         }
1213 
1214         /*
1215          * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1216          * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1217          * QUEUED (EXT), and SERVICE commands). Queued operations are
1218          * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1219          * QUEUED commands when the HBA and device support native command
1220          * queuing(NCQ).
1221          *
1222          * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1223          * CAP register of the HBA indicates NCQ is supported.
1224          *
1225          * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1226          * set because the previous register content of PxCI can be re-written
1227          * in the register write.
1228          */
1229         if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1230             !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1231                 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1232                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1233                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1234                     "Command Queuing", NULL);
1235         }
1236 
1237         /* Support port multiplier? */
1238         if (cap_status & AHCI_HBA_CAP_SPM) {
1239                 sata_hba_tran->sata_tran_hba_features_support |=
1240                     SATA_CTLF_PORT_MULTIPLIER;
1241 
1242                 /* Support FIS-based switching for port multiplier? */
1243                 if (cap_status & AHCI_HBA_CAP_FBSS) {
1244                         sata_hba_tran->sata_tran_hba_features_support |=
1245                             SATA_CTLF_PMULT_FBS;
1246                 }
1247         }
1248 
1249         /* Report the number of command slots */
1250         sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1251 
1252         sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1253         sata_hba_tran->sata_tran_start = ahci_tran_start;
1254         sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1255         sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1256         sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1257 #ifdef __lock_lint
1258         sata_hba_tran->sata_tran_selftest = ahci_selftest;
1259 #endif
1260         /*
1261          * When SATA framework adds support for pwrmgt the
1262          * pwrmgt_ops needs to be updated
1263          */
1264         sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1265         sata_hba_tran->sata_tran_ioctl = NULL;
1266 
1267         ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1268 
1269         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1270 
1271         /* Attach it to SATA framework */
1272         if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1273             != DDI_SUCCESS) {
1274                 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1275                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1276                 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1277                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1278                 return (AHCI_FAILURE);
1279         }
1280 
1281         return (AHCI_SUCCESS);
1282 }
1283 
1284 /*
1285  * Unregisters the ahci with sata framework.
1286  */
1287 static int
1288 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1289 {
1290         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1291             "ahci_unregister_sata_hba_tran enter", NULL);
1292 
1293         /* Detach from the SATA framework. */
1294         if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1295             DDI_SUCCESS) {
1296                 return (AHCI_FAILURE);
1297         }
1298 
1299         /* Deallocate sata_hba_tran. */
1300         kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1301             sizeof (sata_hba_tran_t));
1302 
1303         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1304         ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1305         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1306 
1307         return (AHCI_SUCCESS);
1308 }
1309 
1310 #define SET_PORTSTR(str, addrp)                                         \
1311         if (AHCI_ADDR_IS_PORT(addrp))                                   \
1312                 (void) sprintf((str), "%d", (addrp)->aa_port);               \
1313         else if (AHCI_ADDR_IS_PMULT(addrp))                             \
1314                 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port);       \
1315         else                                                            \
1316                 (void) sprintf((str), "%d:%d", (addrp)->aa_port,     \
1317                     (addrp)->aa_pmport);
1318 
1319 /*
1320  * ahci_tran_probe_port is called by SATA framework. It returns port state,
1321  * port status registers and an attached device type via sata_device
1322  * structure.
1323  *
1324  * We return the cached information from a previous hardware probe. The
1325  * actual hardware probing itself was done either from within
1326  * ahci_initialize_controller() during the driver attach or from a phy
1327  * ready change interrupt handler.
1328  */
1329 static int
1330 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1331 {
1332         ahci_ctl_t *ahci_ctlp;
1333         ahci_port_t *ahci_portp;
1334         ahci_addr_t addr, pmult_addr;
1335         uint8_t cport = sd->satadev_addr.cport;
1336         char portstr[10];
1337         uint8_t device_type;
1338         uint32_t port_state;
1339         uint8_t port;
1340         int rval = SATA_SUCCESS, rval_init;
1341 
1342         ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1343         port = ahci_ctlp->ahcictl_cport_to_port[cport];
1344 
1345         ahci_portp = ahci_ctlp->ahcictl_ports[port];
1346 
1347         mutex_enter(&ahci_portp->ahciport_mutex);
1348 
1349         ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1350         ASSERT(AHCI_ADDR_IS_VALID(&addr));
1351         SET_PORTSTR(portstr, &addr);
1352 
1353         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1354             "ahci_tran_probe_port enter: port %s", portstr);
1355 
1356         if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1357             (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1358             ahci_portp->ahciport_pmult_info == NULL)) {
1359                 /* port mutliplier is removed. */
1360                 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1361                     "ahci_tran_probe_port: "
1362                     "pmult is removed from port %s", portstr);
1363                 mutex_exit(&ahci_portp->ahciport_mutex);
1364                 return (SATA_FAILURE);
1365         }
1366 
1367         /*
1368          * The sata_device may refer to
1369          * 1. A controller port.
1370          *    A controller port should be ready here.
1371          * 2. A port multiplier.
1372          *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1373          *    it and register the port multiplier to the framework.
1374          *    SATA_ADDR_PMULT - check the status of all its device ports.
1375          * 3. A port multiplier port.
1376          *    If it has not been initialized, initialized it.
1377          *
1378          * A port multiplier or a port multiplier port may require some
1379          * initialization because we cannot do these time-consuming jobs in an
1380          * interrupt context.
1381          */
1382         if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1383                 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1384                 /* Initialize registers on a port multiplier */
1385                 rval_init = ahci_initialize_pmult(ahci_ctlp,
1386                     ahci_portp, &pmult_addr, sd);
1387                 if (rval_init != AHCI_SUCCESS) {
1388                         AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1389                             "ahci_tran_probe_port: "
1390                             "pmult initialization failed.", NULL);
1391                         mutex_exit(&ahci_portp->ahciport_mutex);
1392                         return (SATA_FAILURE);
1393                 }
1394         } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1395                 /* Check pmports hotplug events */
1396                 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1397         } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1398             SATA_ADDR_DPMPORT)) {
1399                 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1400                     &addr, sd) != AHCI_SUCCESS) {
1401                         rval = SATA_FAILURE;
1402                         goto out;
1403                 }
1404         }
1405 
1406         /* Update port state and device type */
1407         port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1408 
1409         switch (port_state) {
1410 
1411         case SATA_PSTATE_FAILED:
1412                 sd->satadev_state = SATA_PSTATE_FAILED;
1413                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1414                     "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1415                 goto out;
1416 
1417         case SATA_PSTATE_SHUTDOWN:
1418                 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1419                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1420                     "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1421                 goto out;
1422 
1423         case SATA_PSTATE_PWROFF:
1424                 sd->satadev_state = SATA_PSTATE_PWROFF;
1425                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1426                     "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1427                 goto out;
1428 
1429         case SATA_PSTATE_PWRON:
1430                 sd->satadev_state = SATA_PSTATE_PWRON;
1431                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1432                     "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1433                 break;
1434 
1435         default:
1436                 sd->satadev_state = port_state;
1437                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1438                     "ahci_tran_probe_port: port %s PORT NORMAL %x",
1439                     portstr, port_state);
1440                 break;
1441         }
1442 
1443         device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1444 
1445         switch (device_type) {
1446 
1447         case SATA_DTYPE_ATADISK:
1448                 sd->satadev_type = SATA_DTYPE_ATADISK;
1449                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1450                     "ahci_tran_probe_port: port %s DISK found", portstr);
1451                 break;
1452 
1453         case SATA_DTYPE_ATAPI:
1454                 /*
1455                  * HBA driver only knows it's an ATAPI device, and don't know
1456                  * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1457                  * type need to be determined by checking IDENTIFY PACKET
1458                  * DEVICE data
1459                  */
1460                 sd->satadev_type = SATA_DTYPE_ATAPI;
1461                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1462                     "ahci_tran_probe_port: port %s ATAPI found", portstr);
1463                 break;
1464 
1465         case SATA_DTYPE_PMULT:
1466                 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1467                 sd->satadev_type = SATA_DTYPE_PMULT;
1468 
1469                 /* Update the number of pmports. */
1470                 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1471                 sd->satadev_add_info = ahci_portp->
1472                     ahciport_pmult_info->ahcipmi_num_dev_ports;
1473 
1474                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1475                     "ahci_tran_probe_port: port %s Port Multiplier found",
1476                     portstr);
1477                 break;
1478 
1479         case SATA_DTYPE_UNKNOWN:
1480                 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1481                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1482                     "ahci_tran_probe_port: port %s Unknown device found",
1483                     portstr);
1484                 break;
1485 
1486         default:
1487                 /* we don't support any other device types */
1488                 sd->satadev_type = SATA_DTYPE_NONE;
1489                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1490                     "ahci_tran_probe_port: port %s No device found", portstr);
1491                 break;
1492         }
1493 
1494 out:
1495         /* Register update only fails while probing a pmult/pmport */
1496         if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1497                 ahci_update_sata_registers(ahci_ctlp, port, sd);
1498         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1499                 if (port_state & SATA_STATE_READY)
1500                         if (ahci_update_pmult_pscr(ahci_ctlp,
1501                             &addr, sd) != AHCI_SUCCESS)
1502                                 rval = SATA_FAILURE;
1503         }
1504 
1505         /* Check handles for the sata registers access */
1506         if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1507             (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1508                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1509                     DDI_SERVICE_UNAFFECTED);
1510                 rval = SATA_FAILURE;
1511         }
1512 
1513         mutex_exit(&ahci_portp->ahciport_mutex);
1514         return (rval);
1515 }
1516 
1517 /*
1518  * There are four operation modes in sata framework:
1519  * SATA_OPMODE_INTERRUPTS
1520  * SATA_OPMODE_POLLING
1521  * SATA_OPMODE_ASYNCH
1522  * SATA_OPMODE_SYNCH
1523  *
1524  * Their combined meanings as following:
1525  *
1526  * SATA_OPMODE_SYNCH
1527  * The command has to be completed before sata_tran_start functions returns.
1528  * Either interrupts or polling could be used - it's up to the driver.
1529  * Mode used currently for internal, sata-module initiated operations.
1530  *
1531  * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1532  * It is the same as the one above.
1533  *
1534  * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1535  * The command has to be completed before sata_tran_start function returns.
1536  * No interrupt used, polling only. This should be the mode used for scsi
1537  * packets with FLAG_NOINTR.
1538  *
1539  * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1540  * The command may be queued (callback function specified). Interrupts could
1541  * be used. It's normal operation mode.
1542  */
1543 /*
1544  * Called by sata framework to transport a sata packet down stream.
1545  */
1546 static int
1547 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1548 {
1549         ahci_ctl_t *ahci_ctlp;
1550         ahci_port_t *ahci_portp;
1551         ahci_addr_t addr;
1552         uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1553         uint8_t port;
1554         char portstr[10];
1555 
1556         ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1557         port = ahci_ctlp->ahcictl_cport_to_port[cport];
1558 
1559         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1560             "ahci_tran_start enter: cport %d satapkt 0x%p",
1561             cport, (void *)spkt);
1562 
1563         ahci_portp = ahci_ctlp->ahcictl_ports[port];
1564 
1565         mutex_enter(&ahci_portp->ahciport_mutex);
1566         ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1567         SET_PORTSTR(portstr, &addr);
1568 
1569         /* Sanity check */
1570         if (AHCI_ADDR_IS_PMPORT(&addr)) {
1571                 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1572                     ahci_portp->ahciport_pmult_info == NULL) {
1573 
1574                         spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1575                         spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1576                         spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1577                         ahci_update_sata_registers(ahci_ctlp, port,
1578                             &spkt->satapkt_device);
1579                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1580                             "ahci_tran_start returning PORT_ERROR while "
1581                             "pmult removed: port: %s", portstr);
1582                         mutex_exit(&ahci_portp->ahciport_mutex);
1583                         return (SATA_TRAN_PORT_ERROR);
1584                 }
1585 
1586                 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1587                     SATA_STATE_READY)) {
1588                         if (!ddi_in_panic() ||
1589                             ahci_initialize_pmport(ahci_ctlp,
1590                             ahci_portp, &addr) != AHCI_SUCCESS) {
1591                                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1592                                 spkt->satapkt_device.satadev_type =
1593                                     AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1594                                 spkt->satapkt_device.satadev_state =
1595                                     AHCIPORT_GET_STATE(ahci_portp, &addr);
1596                                 ahci_update_sata_registers(ahci_ctlp, port,
1597                                     &spkt->satapkt_device);
1598                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1599                                     "ahci_tran_start returning PORT_ERROR "
1600                                     "while sub-link is not initialized "
1601                                     "at port: %s", portstr);
1602                                 mutex_exit(&ahci_portp->ahciport_mutex);
1603                                 return (SATA_TRAN_PORT_ERROR);
1604                         }
1605                 }
1606         }
1607 
1608         if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1609             AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1610             AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1611                 /*
1612                  * In case the target driver would send the packet before
1613                  * sata framework can have the opportunity to process those
1614                  * event reports.
1615                  */
1616                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1617                 spkt->satapkt_device.satadev_state =
1618                     ahci_portp->ahciport_port_state;
1619                 ahci_update_sata_registers(ahci_ctlp, port,
1620                     &spkt->satapkt_device);
1621                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1622                     "ahci_tran_start returning PORT_ERROR while "
1623                     "port in FAILED/SHUTDOWN/PWROFF state: "
1624                     "port: %s", portstr);
1625                 mutex_exit(&ahci_portp->ahciport_mutex);
1626                 return (SATA_TRAN_PORT_ERROR);
1627         }
1628 
1629         if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1630                 /*
1631                  * ahci_intr_phyrdy_change() may have rendered it to
1632                  * SATA_DTYPE_NONE.
1633                  */
1634                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1635                 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1636                 spkt->satapkt_device.satadev_state =
1637                     ahci_portp->ahciport_port_state;
1638                 ahci_update_sata_registers(ahci_ctlp, port,
1639                     &spkt->satapkt_device);
1640                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1641                     "ahci_tran_start returning PORT_ERROR while "
1642                     "no device attached: port: %s", portstr);
1643                 mutex_exit(&ahci_portp->ahciport_mutex);
1644                 return (SATA_TRAN_PORT_ERROR);
1645         }
1646 
1647         /* R/W PMULT command will occupy the whole HBA port */
1648         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1649                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1650                     "ahci_tran_start returning BUSY while "
1651                     "executing READ/WRITE PORT-MULT command: "
1652                     "port: %s", portstr);
1653                 spkt->satapkt_reason = SATA_PKT_BUSY;
1654                 mutex_exit(&ahci_portp->ahciport_mutex);
1655                 return (SATA_TRAN_BUSY);
1656         }
1657 
1658         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1659                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1660                     "ahci_tran_start returning BUSY while "
1661                     "hot-plug in progress: port: %s", portstr);
1662                 spkt->satapkt_reason = SATA_PKT_BUSY;
1663                 mutex_exit(&ahci_portp->ahciport_mutex);
1664                 return (SATA_TRAN_BUSY);
1665         }
1666 
1667         /*
1668          * SATA HBA driver should remember that a device was reset and it
1669          * is supposed to reject any packets which do not specify either
1670          * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1671          *
1672          * This is to prevent a race condition when a device was arbitrarily
1673          * reset by the HBA driver (and lost it's setting) and a target
1674          * driver sending some commands to a device before the sata framework
1675          * has a chance to restore the device setting (such as cache enable/
1676          * disable or other resettable stuff).
1677          */
1678         /*
1679          * It is unnecessary to use specific flags to indicate
1680          * reset_in_progress for a pmport. While mopping, all command will be
1681          * mopped so that the entire HBA port is being dealt as a single
1682          * object.
1683          */
1684         if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1685                 ahci_portp->ahciport_reset_in_progress = 0;
1686                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1687                     "ahci_tran_start [CLEAR] the "
1688                     "reset_in_progress for port: %d", port);
1689         }
1690 
1691         if (ahci_portp->ahciport_reset_in_progress &&
1692             ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1693             ! ddi_in_panic()) {
1694                 spkt->satapkt_reason = SATA_PKT_BUSY;
1695                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1696                     "ahci_tran_start returning BUSY while "
1697                     "reset in progress: port: %d", port);
1698                 mutex_exit(&ahci_portp->ahciport_mutex);
1699                 return (SATA_TRAN_BUSY);
1700         }
1701 
1702 #ifdef AHCI_DEBUG
1703         if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1704                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1705                     "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1706                     spkt, port);
1707         }
1708 #endif
1709 
1710         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1711                 spkt->satapkt_reason = SATA_PKT_BUSY;
1712                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1713                     "ahci_tran_start returning BUSY while "
1714                     "mopping in progress: port: %d", port);
1715                 mutex_exit(&ahci_portp->ahciport_mutex);
1716                 return (SATA_TRAN_BUSY);
1717         }
1718 
1719         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1720                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1721                     DDI_SERVICE_UNAFFECTED);
1722                 mutex_exit(&ahci_portp->ahciport_mutex);
1723                 return (SATA_TRAN_BUSY);
1724         }
1725 
1726         if (spkt->satapkt_op_mode &
1727             (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1728                 /*
1729                  * If a SYNC command to be executed in interrupt context,
1730                  * bounce it back to sata module.
1731                  */
1732                 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1733                     servicing_interrupt()) {
1734                         spkt->satapkt_reason = SATA_PKT_BUSY;
1735                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1736                             "ahci_tran_start returning BUSY while "
1737                             "sending SYNC mode under interrupt context: "
1738                             "port : %d", port);
1739                         mutex_exit(&ahci_portp->ahciport_mutex);
1740                         return (SATA_TRAN_BUSY);
1741                 }
1742 
1743                 /* We need to do the sync start now */
1744                 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1745                     spkt) == AHCI_FAILURE) {
1746                         goto fail_out;
1747                 }
1748         } else {
1749                 /* Async start, using interrupt */
1750                 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1751                     == AHCI_FAILURE) {
1752                         spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1753                         goto fail_out;
1754                 }
1755         }
1756 
1757         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1758             "sata tran accepted: port %s", portstr);
1759 
1760         mutex_exit(&ahci_portp->ahciport_mutex);
1761         return (SATA_TRAN_ACCEPTED);
1762 
1763 fail_out:
1764         /*
1765          * Failed to deliver packet to the controller.
1766          * Check if it's caused by invalid handles.
1767          */
1768         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1769             ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1770                 spkt->satapkt_device.satadev_type =
1771                     AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1772                 spkt->satapkt_device.satadev_state =
1773                     AHCIPORT_GET_STATE(ahci_portp, &addr);
1774                 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1775                 mutex_exit(&ahci_portp->ahciport_mutex);
1776                 return (SATA_TRAN_PORT_ERROR);
1777         }
1778 
1779         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1780             "return QUEUE_FULL: port %d", port);
1781         mutex_exit(&ahci_portp->ahciport_mutex);
1782         return (SATA_TRAN_QUEUE_FULL);
1783 }
1784 
1785 /*
1786  * SATA_OPMODE_SYNCH flag is set
1787  *
1788  * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1789  * without interrupt, otherwise we can still use the interrupt.
1790  */
1791 static int
1792 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1793     ahci_addr_t *addrp, sata_pkt_t *spkt)
1794 {
1795         int pkt_timeout_ticks;
1796         uint32_t timeout_tags;
1797         int rval;
1798         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1799         uint8_t port = addrp->aa_port;
1800 
1801         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1802 
1803         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1804             "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1805 
1806         if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1807                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1808                 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1809                     addrp, spkt)) == AHCI_FAILURE) {
1810                         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1811                         return (rval);
1812                 }
1813 
1814                 pkt_timeout_ticks =
1815                     drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1816 
1817                 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1818                         mutex_exit(&ahci_portp->ahciport_mutex);
1819 
1820                         /* Simulate the interrupt */
1821                         ahci_port_intr(ahci_ctlp, ahci_portp, port);
1822 
1823                         drv_usecwait(AHCI_10MS_USECS);
1824 
1825                         mutex_enter(&ahci_portp->ahciport_mutex);
1826                         pkt_timeout_ticks -= AHCI_10MS_TICKS;
1827                         if (pkt_timeout_ticks < 0) {
1828                                 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1829                                     "port %d satapkt 0x%p timed out\n",
1830                                     instance, port, (void *)spkt);
1831                                 timeout_tags = (0x1 << rval);
1832                                 mutex_exit(&ahci_portp->ahciport_mutex);
1833                                 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1834                                     port, timeout_tags);
1835                                 mutex_enter(&ahci_portp->ahciport_mutex);
1836                         }
1837                 }
1838                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1839                 return (AHCI_SUCCESS);
1840 
1841         } else {
1842                 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1843                     addrp, spkt)) == AHCI_FAILURE)
1844                         return (rval);
1845 
1846 #if AHCI_DEBUG
1847                 /*
1848                  * Note that the driver always uses the slot 0 to deliver
1849                  * REQUEST SENSE or READ LOG EXT command
1850                  */
1851                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1852                         ASSERT(rval == 0);
1853 #endif
1854 
1855                 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1856                         cv_wait(&ahci_portp->ahciport_cv,
1857                             &ahci_portp->ahciport_mutex);
1858 
1859                 return (AHCI_SUCCESS);
1860         }
1861 }
1862 
1863 /*
1864  * Searches for and claims a free command slot.
1865  *
1866  * Returns value:
1867  *
1868  * AHCI_FAILURE returned only if
1869  *      1. No empty slot left
1870  *      2. Non-queued command requested while queued command(s) is outstanding
1871  *      3. Queued command requested while non-queued command(s) is outstanding
1872  *      4. HBA doesn't support multiple-use of command list while already a
1873  *         non-queued command is oustanding
1874  *      5. Queued command requested while some queued command(s) has been
1875  *         outstanding on a different port multiplier port. (AHCI spec 1.2,
1876  *         9.1.2)
1877  *
1878  * claimed slot number returned if succeeded
1879  *
1880  * NOTE: it will always return slot 0 for following commands to simplify the
1881  * algorithm.
1882  *      1. REQUEST SENSE or READ LOG EXT command during error recovery process
1883  *      2. READ/WRITE PORTMULT command
1884  */
1885 static int
1886 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1887     ahci_addr_t *addrp, int command_type)
1888 {
1889         uint32_t port_cmd_issue;
1890         uint32_t free_slots;
1891         int slot;
1892 
1893         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1894 
1895         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1896             "ahciport_pending_tags = 0x%x "
1897             "ahciport_pending_ncq_tags = 0x%x",
1898             ahci_portp->ahciport_pending_tags,
1899             ahci_portp->ahciport_pending_ncq_tags);
1900 
1901         /*
1902          * According to the AHCI spec, system software is responsible to
1903          * ensure that queued and non-queued commands are not mixed in
1904          * the command list.
1905          */
1906         if (command_type == AHCI_NON_NCQ_CMD) {
1907                 /* Non-NCQ command request */
1908                 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1909                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1910                             "ahci_claim_free_slot: there is still pending "
1911                             "queued command(s) in the command list, "
1912                             "so no available slot for the non-queued "
1913                             "command", NULL);
1914                         return (AHCI_FAILURE);
1915                 }
1916                 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1917                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1918                             "ahci_claim_free_slot: there is still pending "
1919                             "read/write port-mult command(s) in command list, "
1920                             "so no available slot for the non-queued command",
1921                             NULL);
1922                         return (AHCI_FAILURE);
1923                 }
1924                 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
1925                     NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1926                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1927                             "ahci_claim_free_slot: HBA cannot support multiple-"
1928                             "use of the command list for non-queued commands",
1929                             NULL);
1930                         return (AHCI_FAILURE);
1931                 }
1932                 free_slots = (~ahci_portp->ahciport_pending_tags) &
1933                     AHCI_SLOT_MASK(ahci_ctlp);
1934         } else if (command_type == AHCI_NCQ_CMD) {
1935                 /* NCQ command request */
1936                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1937                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1938                             "ahci_claim_free_slot: there is still pending "
1939                             "non-queued command(s) in the command list, "
1940                             "so no available slot for the queued command",
1941                             NULL);
1942                         return (AHCI_FAILURE);
1943                 }
1944 
1945                 /*
1946                  * NCQ commands cannot be sent to different port multiplier
1947                  * ports in Command-Based Switching mode
1948                  */
1949                 /*
1950                  * NOTE: In Command-Based Switching mode, AHCI controller
1951                  * usually reports a 'Handshake Error' when multiple NCQ
1952                  * commands are outstanding simultaneously.
1953                  */
1954                 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
1955                         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1956                         if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
1957                             NCQ_CMD_IN_PROGRESS(ahci_portp) &&
1958                             AHCIPORT_NCQ_PMPORT(ahci_portp) !=
1959                             addrp->aa_pmport) {
1960                                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1961                                     "ahci_claim_free_slot: there is still "
1962                                     "pending queued command(s) in the "
1963                                     "command list for another Port Multiplier "
1964                                     "port, so no available slot.", NULL);
1965                                 return (AHCI_FAILURE);
1966                         }
1967                 }
1968 
1969                 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
1970                     AHCI_NCQ_SLOT_MASK(ahci_portp);
1971         } else if (command_type == AHCI_ERR_RETRI_CMD) {
1972                 /* Error retrieval command request */
1973                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1974                     "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
1975                     "SENSE or READ LOG EXT command", NULL);
1976                 slot = 0;
1977                 goto out;
1978         } else if (command_type == AHCI_RDWR_PMULT_CMD) {
1979                 /*
1980                  * An extra check on PxCI. Sometimes PxCI bits may not be
1981                  * cleared during hot-plug or error recovery process.
1982                  */
1983                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1984                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
1985 
1986                 if (port_cmd_issue != 0) {
1987                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1988                             "ahci_claim_free_slot: there is still pending "
1989                             "command(s) in command list (0x%x/0x%x, PxCI %x),"
1990                             "so no available slot for R/W PMULT command.",
1991                             NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
1992                             NCQ_CMD_IN_PROGRESS(ahci_portp),
1993                             port_cmd_issue);
1994                         return (AHCI_FAILURE);
1995                 }
1996 
1997                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1998                     "ahci_claim_free_slot: slot 0 is allocated for "
1999                     "READ/WRITE PORTMULT command", NULL);
2000                 slot = 0;
2001                 goto out;
2002         }
2003 
2004         slot = ddi_ffs(free_slots) - 1;
2005         if (slot == -1) {
2006                 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2007                     "ahci_claim_free_slot: no empty slots", NULL);
2008                 return (AHCI_FAILURE);
2009         }
2010 
2011         /*
2012          * According to the AHCI spec, to allow a simple mechanism for the
2013          * HBA to map command list slots to queue entries, software must
2014          * match the tag number it uses to the slot it is placing the command
2015          * in. For example, if a queued command is placed in slot 5, the tag
2016          * for that command must be 5.
2017          */
2018         if (command_type == AHCI_NCQ_CMD) {
2019                 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2020                 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2021                         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2022                         AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2023                 }
2024         }
2025 
2026         ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2027 
2028 out:
2029         AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2030             "ahci_claim_free_slot: found slot: 0x%x", slot);
2031 
2032         return (slot);
2033 }
2034 
2035 /*
2036  * Builds the Command Table for the sata packet and delivers it to controller.
2037  *
2038  * Returns:
2039  *      slot number if we can obtain a slot successfully
2040  *      otherwise, return AHCI_FAILURE
2041  */
2042 static int
2043 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2044     ahci_addr_t *addrp, sata_pkt_t *spkt)
2045 {
2046         int cmd_slot;
2047         sata_cmd_t *scmd;
2048         ahci_fis_h2d_register_t *h2d_register_fisp;
2049         ahci_cmd_table_t *cmd_table;
2050         ahci_cmd_header_t *cmd_header;
2051         int ncookies;
2052         int i;
2053         int command_type = AHCI_NON_NCQ_CMD;
2054         int ncq_qdepth;
2055         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2056         uint8_t port, pmport;
2057 #if AHCI_DEBUG
2058         uint32_t *ptr;
2059         uint8_t *ptr2;
2060 #endif
2061 
2062         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2063 
2064         port = addrp->aa_port;
2065         pmport = addrp->aa_pmport;
2066 
2067         spkt->satapkt_reason = SATA_PKT_BUSY;
2068 
2069         scmd = &spkt->satapkt_cmd;
2070 
2071         /* Check if the command is a NCQ command */
2072         if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2073             scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2074                 command_type = AHCI_NCQ_CMD;
2075 
2076                 /*
2077                  * When NCQ is support, system software must determine the
2078                  * maximum tag allowed by the device and the HBA, and it
2079                  * must use a value not beyond of the lower bound of the two.
2080                  *
2081                  * Sata module is going to calculate the qdepth and send
2082                  * down to HBA driver via sata_cmd.
2083                  */
2084                 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2085 
2086                 /*
2087                  * At the moment, the driver doesn't support the dynamic
2088                  * setting of the maximum ncq depth, and the value can be
2089                  * set either during the attach or after hot-plug insertion.
2090                  */
2091                 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2092                         ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2093                         AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2094                             "ahci_deliver_satapkt: port %d the max tags for "
2095                             "NCQ command is %d", port, ncq_qdepth);
2096                 } else {
2097                         if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2098                                 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2099                                     " port %d the max tag for NCQ command is "
2100                                     "requested to change from %d to %d, at the"
2101                                     " moment the driver doesn't support the "
2102                                     "dynamic change so it's going to "
2103                                     "still use the previous tag value",
2104                                     instance, port,
2105                                     ahci_portp->ahciport_max_ncq_tags,
2106                                     ncq_qdepth);
2107                         }
2108                 }
2109         }
2110 
2111         /* Check if the command is an error retrieval command */
2112         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2113                 command_type = AHCI_ERR_RETRI_CMD;
2114 
2115         /* Check if the command is an read/write pmult command */
2116         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2117                 command_type = AHCI_RDWR_PMULT_CMD;
2118 
2119         /* Check if there is an empty command slot */
2120         cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2121             addrp, command_type);
2122         if (cmd_slot == AHCI_FAILURE) {
2123                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2124                 return (AHCI_FAILURE);
2125         }
2126 
2127         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2128             "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2129             "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2130             cmd_slot, port, (void *)spkt);
2131 
2132         cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2133         bzero((void *)cmd_table, ahci_cmd_table_size);
2134 
2135         /* For data transfer operations, it is the H2D Register FIS */
2136         h2d_register_fisp =
2137             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2138 
2139         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2140 
2141         /*
2142          * PMP field only make sense when target is a port multiplier or a
2143          * device behind a port multiplier. Otherwise should set it to 0.
2144          */
2145         if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2146                 SET_FIS_PMP(h2d_register_fisp, pmport);
2147 
2148         SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2149         SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2150         SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2151         SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2152 
2153         switch (scmd->satacmd_addr_type) {
2154 
2155         case 0:
2156                 /*
2157                  * satacmd_addr_type will be 0 for the commands below:
2158                  *      ATAPI command
2159                  *      SATAC_IDLE_IM
2160                  *      SATAC_STANDBY_IM
2161                  *      SATAC_DOWNLOAD_MICROCODE
2162                  *      SATAC_FLUSH_CACHE
2163                  *      SATAC_SET_FEATURES
2164                  *      SATAC_SMART
2165                  *      SATAC_ID_PACKET_DEVICE
2166                  *      SATAC_ID_DEVICE
2167                  *      SATAC_READ_PORTMULT
2168                  *      SATAC_WRITE_PORTMULT
2169                  */
2170                 /* FALLTHRU */
2171 
2172         case ATA_ADDR_LBA:
2173                 /* FALLTHRU */
2174 
2175         case ATA_ADDR_LBA28:
2176                 /* LBA[7:0] */
2177                 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2178 
2179                 /* LBA[15:8] */
2180                 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2181 
2182                 /* LBA[23:16] */
2183                 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2184 
2185                 /* LBA [27:24] (also called dev_head) */
2186                 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2187 
2188                 break;
2189 
2190         case ATA_ADDR_LBA48:
2191                 /* LBA[7:0] */
2192                 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2193 
2194                 /* LBA[15:8] */
2195                 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2196 
2197                 /* LBA[23:16] */
2198                 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2199 
2200                 /* LBA [31:24] */
2201                 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2202                     scmd->satacmd_lba_low_msb);
2203 
2204                 /* LBA [39:32] */
2205                 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2206                     scmd->satacmd_lba_mid_msb);
2207 
2208                 /* LBA [47:40] */
2209                 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2210                     scmd->satacmd_lba_high_msb);
2211 
2212                 /* Set dev_head */
2213                 SET_FIS_DEV_HEAD(h2d_register_fisp,
2214                     scmd->satacmd_device_reg);
2215 
2216                 /* Set the extended sector count and features */
2217                 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2218                     scmd->satacmd_sec_count_msb);
2219                 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2220                     scmd->satacmd_features_reg_ext);
2221                 break;
2222         }
2223 
2224         /*
2225          * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2226          * filled into features field, and sector count 8:15 is filled into
2227          * features (exp) field. The hba driver doesn't need to anything
2228          * special with regard to this, since sata framework has already
2229          * done so.
2230          *
2231          * However the driver needs to make sure TAG is filled into sector
2232          * field.
2233          */
2234         if (command_type == AHCI_NCQ_CMD) {
2235                 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2236                     (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2237         }
2238 
2239         ncookies = scmd->satacmd_num_dma_cookies;
2240         AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2241             "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2242             ncookies, ahci_dma_prdt_number);
2243 
2244         ASSERT(ncookies <= ahci_dma_prdt_number);
2245         ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2246 
2247         /* *** now fill the scatter gather list ******* */
2248         for (i = 0; i < ncookies; i++) {
2249                 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2250                     scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2251                 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2252                     scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2253                 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2254                     scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2255                 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2256                     scmd->satacmd_dma_cookie_list[i].dmac_size;
2257         }
2258 
2259         AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2260             "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2261             ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2262 
2263         /* The ACMD field is filled in for ATAPI command */
2264         if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2265                 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2266                     SATA_ATAPI_MAX_CDB_LEN);
2267         }
2268 
2269         /* Set Command Header in Command List */
2270         cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2271         BZERO_DESCR_INFO(cmd_header);
2272         BZERO_PRD_BYTE_COUNT(cmd_header);
2273 
2274         /* Set the number of entries in the PRD table */
2275         SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2276 
2277         /* Set the length of the command in the CFIS area */
2278         SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2279 
2280         /*
2281          * PMP field only make sense when target is a port multiplier or a
2282          * device behind a port multiplier. Otherwise should set it to 0.
2283          */
2284         if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2285                 SET_PORT_MULTI_PORT(cmd_header, pmport);
2286 
2287         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2288             "sata_data_direction = 0x%x",
2289             scmd->satacmd_flags.sata_data_direction);
2290 
2291         /* Set A bit if it is an ATAPI command */
2292         if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2293                 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2294 
2295         /* Set W bit if data is going to the device */
2296         if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2297                 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2298 
2299         /*
2300          * Set the prefetchable bit - this bit is only valid if the PRDTL
2301          * field is non-zero or the ATAPI 'A' bit is set in the command
2302          * header. This bit cannot be set when using native command
2303          * queuing commands or when using FIS-based switching with a Port
2304          * multiplier.
2305          */
2306         if (command_type != AHCI_NCQ_CMD)
2307                 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2308 
2309         /*
2310          * Now remember the sata packet in ahciport_slot_pkts[].
2311          * Error retrieval command and r/w port multiplier command will
2312          * be stored specifically for each port.
2313          */
2314         if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2315             !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2316                 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2317 
2318         /*
2319          * Keep the timeout value
2320          */
2321         ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2322 
2323         /*
2324          * If the intial timout is less than 1 tick, then make it longer by
2325          * 1 tick to avoid immediate timeout
2326          */
2327         if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2328             ahci_watchdog_timeout)
2329                 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2330                     ahci_watchdog_timeout;
2331 
2332 #if AHCI_DEBUG
2333         if (ahci_debug_flags & AHCIDBG_ATACMD &&
2334             scmd->satacmd_cmd_reg != SATAC_PACKET ||
2335             ahci_debug_flags & AHCIDBG_ATAPICMD &&
2336             scmd->satacmd_cmd_reg == SATAC_PACKET) {
2337 
2338                 /* Dump the command header and table */
2339                 ahci_log(ahci_ctlp, CE_WARN, "\n");
2340                 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2341                     "0x%p cmd_reg 0x%x port %d", spkt,
2342                     scmd->satacmd_cmd_reg, port);
2343                 ptr = (uint32_t *)cmd_header;
2344                 ahci_log(ahci_ctlp, CE_WARN,
2345                     "  Command Header:%8x %8x %8x %8x",
2346                     ptr[0], ptr[1], ptr[2], ptr[3]);
2347 
2348                 /* Dump the H2D register FIS */
2349                 ptr = (uint32_t *)h2d_register_fisp;
2350                 ahci_log(ahci_ctlp, CE_WARN,
2351                     "  Command FIS:   %8x %8x %8x %8x",
2352                     ptr[0], ptr[1], ptr[2], ptr[3]);
2353 
2354                 /* Dump the ACMD register FIS */
2355                 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2356                 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2357                         if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2358                                 ahci_log(ahci_ctlp, CE_WARN,
2359                                     "  ATAPI command: %2x %2x %2x %2x "
2360                                     "%2x %2x %2x %2x",
2361                                     ptr2[8 * i], ptr2[8 * i + 1],
2362                                     ptr2[8 * i + 2], ptr2[8 * i + 3],
2363                                     ptr2[8 * i + 4], ptr2[8 * i + 5],
2364                                     ptr2[8 * i + 6], ptr2[8 * i + 7]);
2365 
2366                 /* Dump the PRDT */
2367                 for (i = 0; i < ncookies; i++) {
2368                         ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2369                         ahci_log(ahci_ctlp, CE_WARN,
2370                             "  Cookie %d:      %8x %8x %8x %8x",
2371                             i, ptr[0], ptr[1], ptr[2], ptr[3]);
2372                 }
2373         }
2374 #endif
2375 
2376         (void) ddi_dma_sync(
2377             ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2378             0,
2379             ahci_cmd_table_size,
2380             DDI_DMA_SYNC_FORDEV);
2381 
2382         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2383             cmd_slot * sizeof (ahci_cmd_header_t),
2384             sizeof (ahci_cmd_header_t),
2385             DDI_DMA_SYNC_FORDEV);
2386 
2387         if ((ahci_check_dma_handle(ahci_portp->
2388             ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2389             ahci_check_dma_handle(ahci_portp->
2390             ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2391                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2392                     DDI_SERVICE_UNAFFECTED);
2393                 return (AHCI_FAILURE);
2394         }
2395 
2396         /* Set the corresponding bit in the PxSACT.DS for queued command */
2397         if (command_type == AHCI_NCQ_CMD) {
2398                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2399                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2400                     (0x1 << cmd_slot));
2401         }
2402 
2403         /* Indicate to the HBA that a command is active. */
2404         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2405             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2406             (0x1 << cmd_slot));
2407 
2408         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2409             "exit: port %d", port);
2410 
2411         /* Make sure the command is started by the PxSACT/PxCI */
2412         if (ahci_check_acc_handle(ahci_ctlp->
2413             ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2414                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2415                     DDI_SERVICE_UNAFFECTED);
2416                 return (AHCI_FAILURE);
2417         }
2418 
2419         return (cmd_slot);
2420 }
2421 
2422 /*
2423  * Called by the sata framework to abort the previously sent packet(s).
2424  *
2425  * Reset device to abort commands.
2426  */
2427 static int
2428 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2429 {
2430         ahci_ctl_t *ahci_ctlp;
2431         ahci_port_t *ahci_portp;
2432         uint32_t slot_status = 0;
2433         uint32_t aborted_tags = 0;
2434         uint32_t finished_tags = 0;
2435         uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2436         uint8_t port;
2437         int tmp_slot;
2438         int instance = ddi_get_instance(dip);
2439 
2440         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2441         port = ahci_ctlp->ahcictl_cport_to_port[cport];
2442 
2443         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2444             "ahci_tran_abort enter: port %d", port);
2445 
2446         ahci_portp = ahci_ctlp->ahcictl_ports[port];
2447         mutex_enter(&ahci_portp->ahciport_mutex);
2448 
2449         /*
2450          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2451          * commands are being mopped, therefore there is nothing else to do
2452          */
2453         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2454                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2455                     "ahci_tran_abort: port %d is in "
2456                     "mopping process, so just return directly ", port);
2457                 mutex_exit(&ahci_portp->ahciport_mutex);
2458                 return (SATA_SUCCESS);
2459         }
2460 
2461         /*
2462          * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2463          * command is being executed so no other commands is outstanding,
2464          * nothing to do.
2465          */
2466         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2467                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2468                     "ahci_tran_abort: port %d is reading/writing "
2469                     "port multiplier, so just return directly ", port);
2470                 mutex_exit(&ahci_portp->ahciport_mutex);
2471                 return (SATA_SUCCESS);
2472         }
2473 
2474         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2475             ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2476             ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2477                 /*
2478                  * In case the targer driver would send the request before
2479                  * sata framework can have the opportunity to process those
2480                  * event reports.
2481                  */
2482                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2483                 spkt->satapkt_device.satadev_state =
2484                     ahci_portp->ahciport_port_state;
2485                 ahci_update_sata_registers(ahci_ctlp, port,
2486                     &spkt->satapkt_device);
2487                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2488                     "ahci_tran_abort returning SATA_FAILURE while "
2489                     "port in FAILED/SHUTDOWN/PWROFF state: "
2490                     "port: %d", port);
2491                 mutex_exit(&ahci_portp->ahciport_mutex);
2492                 return (SATA_FAILURE);
2493         }
2494 
2495         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2496                 /*
2497                  * ahci_intr_phyrdy_change() may have rendered it to
2498                  * AHCI_PORT_TYPE_NODEV.
2499                  */
2500                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2501                 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2502                 spkt->satapkt_device.satadev_state =
2503                     ahci_portp->ahciport_port_state;
2504                 ahci_update_sata_registers(ahci_ctlp, port,
2505                     &spkt->satapkt_device);
2506                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2507                     "ahci_tran_abort returning SATA_FAILURE while "
2508                     "no device attached: port: %d", port);
2509                 mutex_exit(&ahci_portp->ahciport_mutex);
2510                 return (SATA_FAILURE);
2511         }
2512 
2513         if (flag == SATA_ABORT_ALL_PACKETS) {
2514                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2515                         aborted_tags = ahci_portp->ahciport_pending_tags;
2516                 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2517                         aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2518 
2519                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2520                     instance, port);
2521         } else {
2522                 aborted_tags = 0xffffffff;
2523                 /*
2524                  * Aborting one specific packet, first search the
2525                  * ahciport_slot_pkts[] list for matching spkt.
2526                  */
2527                 for (tmp_slot = 0;
2528                     tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2529                         if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2530                                 aborted_tags = (0x1 << tmp_slot);
2531                                 break;
2532                         }
2533                 }
2534 
2535                 if (aborted_tags == 0xffffffff) {
2536                         /* request packet is not on the pending list */
2537                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2538                             "Cannot find the aborting pkt 0x%p on the "
2539                             "pending list", (void *)spkt);
2540                         ahci_update_sata_registers(ahci_ctlp, port,
2541                             &spkt->satapkt_device);
2542                         mutex_exit(&ahci_portp->ahciport_mutex);
2543                         return (SATA_FAILURE);
2544                 }
2545                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2546                     instance, port, (void *)spkt);
2547         }
2548 
2549         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2550                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2551                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2552         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2553                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2554                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2555 
2556         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2557         ahci_portp->ahciport_mop_in_progress++;
2558 
2559         /*
2560          * To abort the packet(s), first we are trying to clear PxCMD.ST
2561          * to stop the port, and if the port can be stopped
2562          * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2563          * then we just send back the aborted packet(s) with ABORTED flag
2564          * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2565          * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2566          * perform a COMRESET.
2567          */
2568         (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2569             ahci_portp, port, NULL, NULL);
2570 
2571         /*
2572          * Compute which have finished and which need to be retried.
2573          *
2574          * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2575          * minus the slot_status. The aborted_tags has to be deducted by
2576          * finished_tags since we can't possibly abort a tag which had finished
2577          * already.
2578          */
2579         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2580                 finished_tags = ahci_portp->ahciport_pending_tags &
2581                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2582         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2583                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2584                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2585 
2586         aborted_tags &= ~finished_tags;
2587 
2588         ahci_mop_commands(ahci_ctlp,
2589             ahci_portp,
2590             slot_status,
2591             0, /* failed tags */
2592             0, /* timeout tags */
2593             aborted_tags,
2594             0); /* reset tags */
2595 
2596         ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2597         mutex_exit(&ahci_portp->ahciport_mutex);
2598 
2599         return (SATA_SUCCESS);
2600 }
2601 
2602 /*
2603  * Used to do device reset and reject all the pending packets on a device
2604  * during the reset operation.
2605  *
2606  * NOTE: ONLY called by ahci_tran_reset_dport
2607  */
2608 static int
2609 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2610     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2611 {
2612         uint32_t slot_status = 0;
2613         uint32_t reset_tags = 0;
2614         uint32_t finished_tags = 0;
2615         uint8_t port = addrp->aa_port;
2616         sata_device_t sdevice;
2617         int ret;
2618 
2619         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2620 
2621         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2622             "ahci_reset_device_reject_pkts on port: %d", port);
2623 
2624         /*
2625          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2626          * commands are being mopped, therefore there is nothing else to do
2627          */
2628         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2629                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2630                     "ahci_reset_device_reject_pkts: port %d is in "
2631                     "mopping process, so return directly ", port);
2632                 return (SATA_SUCCESS);
2633         }
2634 
2635         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2636                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2637                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2638                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2639         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2640                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2641                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2642                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2643         }
2644 
2645         if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2646             != AHCI_SUCCESS) {
2647                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2648                     "Try to do a port reset after software "
2649                     "reset failed", port);
2650                 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2651                 if (ret != AHCI_SUCCESS) {
2652                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2653                             "ahci_reset_device_reject_pkts: port %d "
2654                             "failed", port);
2655                         return (SATA_FAILURE);
2656                 }
2657         }
2658         /* Set the reset in progress flag */
2659         ahci_portp->ahciport_reset_in_progress = 1;
2660 
2661         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2662         ahci_portp->ahciport_mop_in_progress++;
2663 
2664         /* Indicate to the framework that a reset has happened */
2665         bzero((void *)&sdevice, sizeof (sata_device_t));
2666         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2667         sdevice.satadev_addr.pmport = 0;
2668         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2669         sdevice.satadev_state = SATA_DSTATE_RESET |
2670             SATA_DSTATE_PWR_ACTIVE;
2671         mutex_exit(&ahci_portp->ahciport_mutex);
2672         sata_hba_event_notify(
2673             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2674             &sdevice,
2675             SATA_EVNT_DEVICE_RESET);
2676         mutex_enter(&ahci_portp->ahciport_mutex);
2677 
2678         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2679             "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2680 
2681         /* Next try to mop the pending commands */
2682         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2683                 finished_tags = ahci_portp->ahciport_pending_tags &
2684                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2685         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2686                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2687                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2688 
2689         reset_tags &= ~finished_tags;
2690 
2691         ahci_mop_commands(ahci_ctlp,
2692             ahci_portp,
2693             slot_status,
2694             0, /* failed tags */
2695             0, /* timeout tags */
2696             0, /* aborted tags */
2697             reset_tags); /* reset tags */
2698 
2699         return (SATA_SUCCESS);
2700 }
2701 
2702 /*
2703  * Used to do device reset and reject all the pending packets on a device
2704  * during the reset operation.
2705  *
2706  * NOTE: ONLY called by ahci_tran_reset_dport
2707  */
2708 static int
2709 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2710     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2711 {
2712         uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2713         uint8_t port = addrp->aa_port;
2714         uint8_t pmport = addrp->aa_pmport;
2715         sata_device_t sdevice;
2716 
2717         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2718 
2719         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2720             "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2721 
2722         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2723                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2724                     "ahci_reset_pmdevice_reject_pkts: port %d is in "
2725                     "mopping process, so return directly ", port);
2726                 return (SATA_SUCCESS);
2727         }
2728 
2729         /* Checking for outstanding commands */
2730         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2731                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2732                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2733                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2734         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2735                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2736                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2737                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2738         }
2739 
2740         /* Issue SOFTWARE reset command. */
2741         if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2742             != AHCI_SUCCESS) {
2743                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2744                     "Try to do a port reset after software "
2745                     "reset failed", port);
2746                 return (SATA_FAILURE);
2747         }
2748 
2749         /* Set the reset in progress flag */
2750         ahci_portp->ahciport_reset_in_progress = 1;
2751 
2752         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2753         ahci_portp->ahciport_mop_in_progress++;
2754 
2755         /* Indicate to the framework that a reset has happened */
2756         bzero((void *)&sdevice, sizeof (sata_device_t));
2757         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2758         sdevice.satadev_addr.pmport = pmport;
2759         if (AHCI_ADDR_IS_PMULT(addrp))
2760                 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2761         else
2762                 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2763         sdevice.satadev_state = SATA_DSTATE_RESET |
2764             SATA_DSTATE_PWR_ACTIVE;
2765         mutex_exit(&ahci_portp->ahciport_mutex);
2766         sata_hba_event_notify(
2767             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2768             &sdevice,
2769             SATA_EVNT_DEVICE_RESET);
2770         mutex_enter(&ahci_portp->ahciport_mutex);
2771 
2772         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2773             "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2774             port, pmport);
2775 
2776         /* Next try to mop the pending commands */
2777         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2778                 finished_tags = ahci_portp->ahciport_pending_tags &
2779                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2780         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2781                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2782                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2783         reset_tags &= ~finished_tags;
2784 
2785         AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2786             "reset_tags = %x, finished_tags = %x, slot_status = %x",
2787             reset_tags, finished_tags, slot_status);
2788 
2789         /*
2790          * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2791          * try to reset a single device behind a port multiplier will
2792          * terminate all the commands on that HBA port. We need mop these
2793          * commands as well.
2794          */
2795         ahci_mop_commands(ahci_ctlp,
2796             ahci_portp,
2797             slot_status,
2798             0, /* failed tags */
2799             0, /* timeout tags */
2800             0, /* aborted tags */
2801             reset_tags); /* reset tags */
2802 
2803         return (SATA_SUCCESS);
2804 }
2805 
2806 /*
2807  * Used to do port reset and reject all the pending packets on a port during
2808  * the reset operation.
2809  */
2810 static int
2811 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2812     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2813 {
2814         uint32_t slot_status = 0;
2815         uint32_t reset_tags = 0;
2816         uint32_t finished_tags = 0;
2817         uint8_t port = addrp->aa_port;
2818 
2819         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2820 
2821         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2822             "ahci_reset_port_reject_pkts at port: %d", port);
2823 
2824         /*
2825          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2826          * commands are being mopped, therefore there is nothing else to do
2827          */
2828         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2829                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2830                     "ahci_reset_port_reject_pkts: port %d is in "
2831                     "mopping process, so return directly ", port);
2832                 return (SATA_SUCCESS);
2833         }
2834 
2835         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2836         ahci_portp->ahciport_mop_in_progress++;
2837 
2838         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2839                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2840                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2841                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2842         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2843                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2844                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2845                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2846         }
2847 
2848         if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2849             ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2850             NULL) != AHCI_SUCCESS) {
2851 
2852                 /* Clear mop flag */
2853                 ahci_portp->ahciport_mop_in_progress--;
2854                 if (ahci_portp->ahciport_mop_in_progress == 0)
2855                         ahci_portp->ahciport_flags &=
2856                             ~AHCI_PORT_FLAG_MOPPING;
2857                 return (SATA_FAILURE);
2858         }
2859 
2860         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2861                 finished_tags = ahci_portp->ahciport_pending_tags &
2862                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2863         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2864                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2865                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2866 
2867         reset_tags &= ~finished_tags;
2868 
2869         ahci_mop_commands(ahci_ctlp,
2870             ahci_portp,
2871             slot_status,
2872             0, /* failed tags */
2873             0, /* timeout tags */
2874             0, /* aborted tags */
2875             reset_tags); /* reset tags */
2876 
2877         return (SATA_SUCCESS);
2878 }
2879 
2880 /*
2881  * Used to do hba reset and reject all the pending packets on all ports
2882  * during the reset operation.
2883  */
2884 static int
2885 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2886 {
2887         ahci_port_t *ahci_portp;
2888         uint32_t slot_status[AHCI_MAX_PORTS];
2889         uint32_t reset_tags[AHCI_MAX_PORTS];
2890         uint32_t finished_tags[AHCI_MAX_PORTS];
2891         int port;
2892         int ret = SATA_SUCCESS;
2893 
2894         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2895             "ahci_reset_hba_reject_pkts enter", NULL);
2896 
2897         bzero(slot_status, sizeof (slot_status));
2898         bzero(reset_tags, sizeof (reset_tags));
2899         bzero(finished_tags, sizeof (finished_tags));
2900 
2901         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2902                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2903                         continue;
2904                 }
2905 
2906                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2907 
2908                 mutex_enter(&ahci_portp->ahciport_mutex);
2909                 ahci_portp->ahciport_reset_in_progress = 1;
2910                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2911                         slot_status[port] = ddi_get32(
2912                             ahci_ctlp->ahcictl_ahci_acc_handle,
2913                             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2914                         reset_tags[port] = slot_status[port] &
2915                             AHCI_SLOT_MASK(ahci_ctlp);
2916                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2917                             "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2918                             port, reset_tags[port],
2919                             ahci_portp->ahciport_pending_tags);
2920                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2921                         slot_status[port] = ddi_get32(
2922                             ahci_ctlp->ahcictl_ahci_acc_handle,
2923                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2924                         reset_tags[port] = slot_status[port] &
2925                             AHCI_NCQ_SLOT_MASK(ahci_portp);
2926                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2927                             "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2928                             port, reset_tags[port],
2929                             ahci_portp->ahciport_pending_tags);
2930                 }
2931                 mutex_exit(&ahci_portp->ahciport_mutex);
2932         }
2933 
2934         if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
2935                 ret = SATA_FAILURE;
2936         }
2937 
2938         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2939                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2940                         continue;
2941                 }
2942 
2943                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2944 
2945                 mutex_enter(&ahci_portp->ahciport_mutex);
2946                 /*
2947                  * To prevent recursive enter to ahci_mop_commands, we need
2948                  * check AHCI_PORT_FLAG_MOPPING flag.
2949                  */
2950                 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2951                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2952                             "ahci_reset_hba_reject_pkts: port %d is in "
2953                             "mopping process, so return directly ", port);
2954                         mutex_exit(&ahci_portp->ahciport_mutex);
2955                         continue;
2956                 }
2957 
2958                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2959                 ahci_portp->ahciport_mop_in_progress++;
2960 
2961                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2962                         finished_tags[port]  =
2963                             ahci_portp->ahciport_pending_tags &
2964                             ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
2965                 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2966                         finished_tags[port] =
2967                             ahci_portp->ahciport_pending_ncq_tags &
2968                             ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
2969 
2970                 reset_tags[port] &= ~finished_tags[port];
2971 
2972                 ahci_mop_commands(ahci_ctlp,
2973                     ahci_portp,
2974                     slot_status[port],
2975                     0, /* failed tags */
2976                     0, /* timeout tags */
2977                     0, /* aborted tags */
2978                     reset_tags[port]); /* reset tags */
2979                 mutex_exit(&ahci_portp->ahciport_mutex);
2980         }
2981 out:
2982         return (ret);
2983 }
2984 
2985 /*
2986  * Called by sata framework to reset a port(s) or device.
2987  */
2988 static int
2989 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
2990 {
2991         ahci_ctl_t *ahci_ctlp;
2992         ahci_port_t *ahci_portp;
2993         ahci_addr_t addr;
2994         uint8_t cport = sd->satadev_addr.cport;
2995         uint8_t pmport = sd->satadev_addr.pmport;
2996         uint8_t port;
2997         int ret = SATA_SUCCESS;
2998         int instance = ddi_get_instance(dip);
2999 
3000         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3001         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3002         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3003 
3004         ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3005 
3006         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3007             "ahci_tran_reset_dport enter: cport %d", cport);
3008 
3009         switch (sd->satadev_addr.qual) {
3010         case SATA_ADDR_PMPORT:
3011                 /*
3012                  * If we want to issue a COMRESET on a pmport, we need to
3013                  * reject the outstanding commands on that pmport. According
3014                  * to AHCI spec, PxCI register could only be cleared by
3015                  * clearing PxCMD.ST, which will halt the controller port - as
3016                  * well as other pmports.
3017                  *
3018                  * Therefore we directly reset the controller port for
3019                  * simplicity. ahci_tran_probe_port() will handle reset stuff
3020                  * like initializing the given pmport.
3021                  */
3022                 /* FALLTHRU */
3023         case SATA_ADDR_CPORT:
3024                 /* Port reset */
3025                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3026                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3027                     "port %d reset port", instance, port);
3028 
3029                 mutex_enter(&ahci_portp->ahciport_mutex);
3030                 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3031                 mutex_exit(&ahci_portp->ahciport_mutex);
3032 
3033                 break;
3034 
3035         case SATA_ADDR_DPMPORT:
3036                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3037                     "port %d:%d reset device", instance, port, pmport);
3038                 /* FALLTHRU */
3039         case SATA_ADDR_DCPORT:
3040                 /* Device reset */
3041                 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3042                         cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3043                             "port %d reset device", instance, port);
3044 
3045                 mutex_enter(&ahci_portp->ahciport_mutex);
3046                 /*
3047                  * software reset request must be sent to SATA_PMULT_HOSTPORT
3048                  * if target is a port multiplier:
3049                  */
3050                 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3051                     ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3052                         AHCI_ADDR_SET_PMULT(&addr, port);
3053 
3054                 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3055                     ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3056                     ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3057                         /*
3058                          * In case the targer driver would send the request
3059                          * before sata framework can have the opportunity to
3060                          * process those event reports.
3061                          */
3062                         sd->satadev_state = ahci_portp->ahciport_port_state;
3063                         ahci_update_sata_registers(ahci_ctlp, port, sd);
3064                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3065                             "ahci_tran_reset_dport returning SATA_FAILURE "
3066                             "while port in FAILED/SHUTDOWN/PWROFF state: "
3067                             "port: %d", port);
3068                         mutex_exit(&ahci_portp->ahciport_mutex);
3069                         ret = SATA_FAILURE;
3070                         break;
3071                 }
3072 
3073                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3074                     SATA_DTYPE_NONE) {
3075                         /*
3076                          * ahci_intr_phyrdy_change() may have rendered it to
3077                          * AHCI_PORT_TYPE_NODEV.
3078                          */
3079                         sd->satadev_type = SATA_DTYPE_NONE;
3080                         sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3081                             &addr);
3082                         ahci_update_sata_registers(ahci_ctlp, port, sd);
3083                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3084                             "ahci_tran_reset_dport returning SATA_FAILURE "
3085                             "while no device attached: port: %d", port);
3086                         mutex_exit(&ahci_portp->ahciport_mutex);
3087                         ret = SATA_FAILURE;
3088                         break;
3089                 }
3090 
3091                 if (AHCI_ADDR_IS_PORT(&addr)) {
3092                         ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3093                             ahci_portp, &addr);
3094                 } else {
3095                         ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3096                             ahci_portp, &addr);
3097                 }
3098 
3099                 mutex_exit(&ahci_portp->ahciport_mutex);
3100                 break;
3101 
3102         case SATA_ADDR_CNTRL:
3103                 /* Reset the whole controller */
3104                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3105                     "reset the whole hba", instance);
3106                 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3107                 break;
3108 
3109         default:
3110                 ret = SATA_FAILURE;
3111         }
3112 
3113         return (ret);
3114 }
3115 
3116 /*
3117  * Called by sata framework to activate a port as part of hotplug.
3118  * (cfgadm -c connect satax/y)
3119  * Support port multiplier.
3120  */
3121 static int
3122 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3123 {
3124         ahci_ctl_t *ahci_ctlp;
3125         ahci_port_t *ahci_portp;
3126         ahci_addr_t addr;
3127         uint8_t cport = satadev->satadev_addr.cport;
3128         uint8_t pmport = satadev->satadev_addr.pmport;
3129         uint8_t port;
3130         int instance = ddi_get_instance(dip);
3131 
3132         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3133         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3134 
3135         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3136             "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3137 
3138         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3139 
3140         mutex_enter(&ahci_portp->ahciport_mutex);
3141         ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3142         ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3143 
3144         if (AHCI_ADDR_IS_PORT(&addr)) {
3145                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3146                     instance, port);
3147 
3148                 /* Enable the interrupts on the port */
3149                 ahci_enable_port_intrs(ahci_ctlp, port);
3150 
3151                 /*
3152                  * Reset the port so that the PHY communication would be
3153                  * re-established.  But this reset is an internal operation
3154                  * and the sata module doesn't need to know about it.
3155                  * Moreover, the port with a device attached will be started
3156                  * too.
3157                  */
3158                 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3159                     ahci_portp, port,
3160                     AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3161                     NULL);
3162 
3163                 /*
3164                  * Need to check the link status and device status of the port
3165                  * and consider raising power if the port was in D3 state
3166                  */
3167                 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3168                 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3169                 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3170         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3171                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3172                     instance, port, pmport);
3173                 /* AHCI_ADDR_PMPORT */
3174                 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3175                 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3176                     ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3177         }
3178 
3179         satadev->satadev_state = ahci_portp->ahciport_port_state;
3180 
3181         ahci_update_sata_registers(ahci_ctlp, port, satadev);
3182 
3183         mutex_exit(&ahci_portp->ahciport_mutex);
3184         return (SATA_SUCCESS);
3185 }
3186 
3187 /*
3188  * Called by sata framework to deactivate a port as part of hotplug.
3189  * (cfgadm -c disconnect satax/y)
3190  * Support port multiplier.
3191  */
3192 static int
3193 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3194 {
3195         ahci_ctl_t *ahci_ctlp;
3196         ahci_port_t *ahci_portp;
3197         ahci_addr_t addr;
3198         uint8_t cport = satadev->satadev_addr.cport;
3199         uint8_t pmport = satadev->satadev_addr.pmport;
3200         uint8_t port;
3201         uint32_t port_scontrol;
3202         int instance = ddi_get_instance(dip);
3203 
3204         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3205         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3206 
3207         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3208             "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3209 
3210         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3211         mutex_enter(&ahci_portp->ahciport_mutex);
3212         ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3213         ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3214 
3215         if (AHCI_ADDR_IS_PORT(&addr)) {
3216                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3217                     instance, port);
3218 
3219                 /* Disable the interrupts on the port */
3220                 ahci_disable_port_intrs(ahci_ctlp, port);
3221 
3222                 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3223 
3224                         /* First to abort all the pending commands */
3225                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3226 
3227                         /* Then stop the port */
3228                         (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3229                             ahci_portp, port);
3230                 }
3231 
3232                 /* Next put the PHY offline */
3233                 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3234                     (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3235                 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3236                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3237                     AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3238         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3239                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3240                     instance, port, pmport);
3241 
3242                 ahci_disable_port_intrs(ahci_ctlp, port);
3243                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3244                     != SATA_DTYPE_NONE)
3245                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3246 
3247                 /* Re-enable the interrupts for the other pmports */
3248                 ahci_enable_port_intrs(ahci_ctlp, port);
3249         }
3250 
3251         /* Update port state */
3252         AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3253         satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3254 
3255         ahci_update_sata_registers(ahci_ctlp, port, satadev);
3256 
3257         mutex_exit(&ahci_portp->ahciport_mutex);
3258         return (SATA_SUCCESS);
3259 }
3260 
3261 /*
3262  * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3263  * when a device is unplugged or a port is deactivated.
3264  */
3265 static void
3266 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3267     ahci_port_t *ahci_portp, uint8_t port)
3268 {
3269         uint32_t slot_status = 0;
3270         uint32_t abort_tags = 0;
3271 
3272         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3273 
3274         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3275             "ahci_reject_all_abort_pkts at port: %d", port);
3276 
3277         /* Read/write port multiplier command takes highest priority */
3278         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3279                 slot_status = 0x1;
3280                 abort_tags = 0x1;
3281                 goto out;
3282         }
3283 
3284         /*
3285          * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3286          * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3287          * to get the error data, if yes when the device is removed, the
3288          * command needs to be aborted too.
3289          */
3290         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3291                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3292                         slot_status = 0x1;
3293                         abort_tags = 0x1;
3294                         goto out;
3295                 } else {
3296                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3297                             "ahci_reject_all_abort_pkts return directly "
3298                             "port %d no needs to reject any outstanding "
3299                             "commands", port);
3300                         return;
3301                 }
3302         }
3303 
3304         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3305                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3306                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3307                 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3308         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3309                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3310                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3311                 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3312         }
3313 
3314 out:
3315         /* No need to do mop when there is no outstanding commands */
3316         if (slot_status != 0) {
3317                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3318                 ahci_portp->ahciport_mop_in_progress++;
3319 
3320                 ahci_mop_commands(ahci_ctlp,
3321                     ahci_portp,
3322                     slot_status,
3323                     0, /* failed tags */
3324                     0, /* timeout tags */
3325                     abort_tags, /* aborting tags */
3326                     0); /* reset tags */
3327         }
3328 }
3329 
3330 #if defined(__lock_lint)
3331 static int
3332 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3333 {
3334         return (SATA_SUCCESS);
3335 }
3336 #endif
3337 
3338 /*
3339  * Initialize fma capabilities and register with IO fault services.
3340  */
3341 static void
3342 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3343 {
3344         /*
3345          * Need to change iblock to priority for new MSI intr
3346          */
3347         ddi_iblock_cookie_t fm_ibc;
3348 
3349         ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3350             ahci_ctlp->ahcictl_dip,
3351             DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3352             DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3353             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3354 
3355         /* Only register with IO Fault Services if we have some capability */
3356         if (ahci_ctlp->ahcictl_fm_cap) {
3357                 /* Adjust access and dma attributes for FMA */
3358                 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3359                 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3360                 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3361                 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3362                 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3363 
3364                 /*
3365                  * Register capabilities with IO Fault Services.
3366                  * ahcictl_fm_cap will be updated to indicate
3367                  * capabilities actually supported (not requested.)
3368                  */
3369                 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3370                     &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3371 
3372                 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3373                         cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3374                             ddi_get_instance(ahci_ctlp->ahcictl_dip));
3375                         return;
3376                 }
3377                 /*
3378                  * Initialize pci ereport capabilities if ereport
3379                  * capable (should always be.)
3380                  */
3381                 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3382                     DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3383                         pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3384                 }
3385 
3386                 /*
3387                  * Register error callback if error callback capable.
3388                  */
3389                 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3390                         ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3391                             ahci_fm_error_cb, (void *) ahci_ctlp);
3392                 }
3393 
3394                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3395                     "ahci_fm_fini: fma enabled.", NULL);
3396         }
3397 }
3398 
3399 /*
3400  * Releases fma capabilities and un-registers with IO fault services.
3401  */
3402 static void
3403 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3404 {
3405         /* Only unregister FMA capabilities if registered */
3406         if (ahci_ctlp->ahcictl_fm_cap) {
3407                 /*
3408                  * Un-register error callback if error callback capable.
3409                  */
3410                 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3411                         ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3412                 }
3413 
3414                 /*
3415                  * Release any resources allocated by pci_ereport_setup()
3416                  */
3417                 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3418                     DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3419                         pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3420                 }
3421 
3422                 /* Unregister from IO Fault Services */
3423                 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3424 
3425                 /* Adjust access and dma attributes for FMA */
3426                 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3427                 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3428                 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3429                 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3430                 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3431 
3432                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3433                     "ahci_fm_fini: fma disabled.", NULL);
3434         }
3435 }
3436 
3437 /*ARGSUSED*/
3438 static int
3439 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3440 {
3441         /*
3442          * as the driver can always deal with an error in any dma or
3443          * access handle, we can just return the fme_status value.
3444          */
3445         pci_ereport_post(dip, err, NULL);
3446         return (err->fme_status);
3447 }
3448 
3449 int
3450 ahci_check_acc_handle(ddi_acc_handle_t handle)
3451 {
3452         ddi_fm_error_t de;
3453 
3454         ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3455         return (de.fme_status);
3456 }
3457 
3458 int
3459 ahci_check_dma_handle(ddi_dma_handle_t handle)
3460 {
3461         ddi_fm_error_t de;
3462 
3463         ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3464         return (de.fme_status);
3465 }
3466 
3467 /*
3468  * Generate an ereport
3469  */
3470 void
3471 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3472 {
3473         uint64_t ena;
3474         char buf[FM_MAX_CLASS];
3475 
3476         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3477         ena = fm_ena_generate(0, FM_ENA_FMT1);
3478         if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3479                 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3480                     DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3481                     FM_EREPORT_VERSION, NULL);
3482         }
3483 }
3484 
3485 /*
3486  * Check if all handles are correctly allocated.
3487  */
3488 static int
3489 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3490 {
3491         int port;
3492 
3493         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3494                 return (DDI_FAILURE);
3495         }
3496 
3497         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3498                 ahci_port_t *ahci_portp;
3499 
3500                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3501                         continue;
3502 
3503                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3504 
3505                 mutex_enter(&ahci_portp->ahciport_mutex);
3506                 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3507                         mutex_exit(&ahci_portp->ahciport_mutex);
3508                         return (DDI_FAILURE);
3509                 }
3510                 mutex_exit(&ahci_portp->ahciport_mutex);
3511         }
3512 
3513         return (DDI_SUCCESS);
3514 }
3515 
3516 /*
3517  * Check the access handles for the controller. Note that
3518  * ahcictl_pci_conf_handle is only used in attach process.
3519  */
3520 static int
3521 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3522 {
3523         if ((ahci_check_acc_handle(ahci_ctlp->
3524             ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3525             (ahci_check_acc_handle(ahci_ctlp->
3526             ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3527                 return (DDI_FAILURE);
3528         }
3529         return (DDI_SUCCESS);
3530 }
3531 
3532 /*
3533  * Check the DMA handles and the access handles of a controller port.
3534  */
3535 static int
3536 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3537 {
3538         ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3539         int slot;
3540 
3541         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3542 
3543         if ((ahci_check_dma_handle(ahci_portp->
3544             ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3545             (ahci_check_dma_handle(ahci_portp->
3546             ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3547             (ahci_check_acc_handle(ahci_portp->
3548             ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3549             (ahci_check_acc_handle(ahci_portp->
3550             ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3551                 return (DDI_FAILURE);
3552         }
3553         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3554                 if (ahci_check_slot_handle(ahci_portp, slot)
3555                     != DDI_SUCCESS) {
3556                         return (DDI_FAILURE);
3557                 }
3558         }
3559         return (DDI_SUCCESS);
3560 }
3561 
3562 /*
3563  * Check the DMA handles and the access handles of a cmd table slot.
3564  */
3565 static int
3566 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3567 {
3568         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3569 
3570         if ((ahci_check_acc_handle(ahci_portp->
3571             ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3572             (ahci_check_dma_handle(ahci_portp->
3573             ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3574                 return (DDI_FAILURE);
3575         }
3576         return (DDI_SUCCESS);
3577 }
3578 
3579 /*
3580  * Allocate the ports structure, only called by ahci_attach
3581  */
3582 static int
3583 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3584 {
3585         int port, cport = 0;
3586 
3587         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3588             "ahci_alloc_ports_state enter", NULL);
3589 
3590         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3591 
3592         /* Allocate structures only for the implemented ports */
3593         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3594                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3595                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3596                             "hba port %d not implemented", port);
3597                         continue;
3598                 }
3599 
3600                 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3601                 ahci_ctlp->ahcictl_port_to_cport[port] =
3602                     (uint8_t)cport++;
3603 
3604                 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3605                         goto err_out;
3606                 }
3607         }
3608 
3609         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3610         return (AHCI_SUCCESS);
3611 
3612 err_out:
3613         for (port--; port >= 0; port--) {
3614                 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3615                         ahci_dealloc_port_state(ahci_ctlp, port);
3616                 }
3617         }
3618 
3619         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3620         return (AHCI_FAILURE);
3621 }
3622 
3623 /*
3624  * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3625  */
3626 static void
3627 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3628 {
3629         int port;
3630 
3631         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3632         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3633                 /* if this port is implemented by the HBA */
3634                 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3635                         ahci_dealloc_port_state(ahci_ctlp, port);
3636         }
3637         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3638 }
3639 
3640 /*
3641  * Drain the taskq.
3642  */
3643 static void
3644 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3645 {
3646         ahci_port_t *ahci_portp;
3647         int port;
3648 
3649         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3650                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3651                         continue;
3652                 }
3653 
3654                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3655 
3656                 mutex_enter(&ahci_portp->ahciport_mutex);
3657                 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3658                 mutex_exit(&ahci_portp->ahciport_mutex);
3659         }
3660 }
3661 
3662 /*
3663  * Initialize the controller and all ports. And then try to start the ports
3664  * if there are devices attached.
3665  *
3666  * This routine can be called from three seperate cases: DDI_ATTACH,
3667  * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3668  * other two cases; device signature probing are attempted only during
3669  * DDI_ATTACH case.
3670  */
3671 static int
3672 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3673 {
3674         ahci_port_t *ahci_portp;
3675         ahci_addr_t addr;
3676         int port;
3677 
3678         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3679             "ahci_initialize_controller enter", NULL);
3680 
3681         /* Disable the whole controller interrupts */
3682         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3683         ahci_disable_all_intrs(ahci_ctlp);
3684         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3685 
3686         /* Initialize the implemented ports and structures */
3687         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3688                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3689                         continue;
3690                 }
3691 
3692                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3693                 mutex_enter(&ahci_portp->ahciport_mutex);
3694 
3695                 /*
3696                  * Ensure that the controller is not in the running state
3697                  * by checking every implemented port's PxCMD register
3698                  */
3699                 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3700 
3701                 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3702                     != AHCI_SUCCESS) {
3703                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3704                             "ahci_initialize_controller: failed to "
3705                             "initialize port %d", port);
3706                         /*
3707                          * Set the port state to SATA_PSTATE_FAILED if
3708                          * failed to initialize it.
3709                          */
3710                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3711                 }
3712 
3713                 mutex_exit(&ahci_portp->ahciport_mutex);
3714         }
3715 
3716         /* Enable the whole controller interrupts */
3717         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3718         ahci_enable_all_intrs(ahci_ctlp);
3719         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3720 
3721         return (AHCI_SUCCESS);
3722 }
3723 
3724 /*
3725  * Reverse of ahci_initialize_controller()
3726  *
3727  * We only need to stop the ports and disable the interrupt.
3728  */
3729 static void
3730 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3731 {
3732         ahci_port_t *ahci_portp;
3733         int port;
3734 
3735         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3736             "ahci_uninitialize_controller enter", NULL);
3737 
3738         /* disable all the interrupts. */
3739         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3740         ahci_disable_all_intrs(ahci_ctlp);
3741         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3742 
3743         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3744                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3745                         continue;
3746                 }
3747 
3748                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3749 
3750                 /* Stop the port by clearing PxCMD.ST */
3751                 mutex_enter(&ahci_portp->ahciport_mutex);
3752 
3753                 /*
3754                  * Here we must disable the port interrupt because
3755                  * ahci_disable_all_intrs only clear GHC.IE, and IS
3756                  * register will be still set if PxIE is enabled.
3757                  * When ahci shares one IRQ with other drivers, the
3758                  * intr handler may claim the intr mistakenly.
3759                  */
3760                 ahci_disable_port_intrs(ahci_ctlp, port);
3761                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3762                     ahci_portp, port);
3763                 mutex_exit(&ahci_portp->ahciport_mutex);
3764         }
3765 }
3766 
3767 /*
3768  * ahci_alloc_pmult()
3769  * 1. Setting HBA port registers which are necessary for a port multiplier.
3770  *    (Set PxCMD.PMA while PxCMD.ST is '0')
3771  * 2. Allocate ahci_pmult_info structure.
3772  *
3773  * NOTE: Must stop port before the function is called.
3774  */
3775 static void
3776 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3777 {
3778         uint32_t port_cmd_status;
3779         uint8_t port = ahci_portp->ahciport_port_num;
3780 
3781         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3782 
3783         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3784             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3785 
3786         /* The port must have been stopped before. */
3787         ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3788 
3789         if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3790                 /* set PMA bit */
3791                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3792                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3793                     port_cmd_status|AHCI_CMD_STATUS_PMA);
3794 
3795                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3796                     "ahci_alloc_pmult: "
3797                     "PxCMD.PMA bit set at port %d.", port);
3798         }
3799 
3800         /* Allocate port multiplier information structure */
3801         if (ahci_portp->ahciport_pmult_info == NULL) {
3802                 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3803                     kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3804         }
3805 
3806         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3807 }
3808 
3809 /*
3810  * ahci_dealloc_pmult()
3811  * 1. Clearing related registers when a port multiplier is detached.
3812  *    (Clear PxCMD.PMA while PxCMD.ST is '0')
3813  * 2. Deallocate ahci_pmult_info structure.
3814  *
3815  * NOTE: Must stop port before the function is called.
3816  */
3817 static void
3818 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3819 {
3820         uint32_t port_cmd_status;
3821         uint8_t port = ahci_portp->ahciport_port_num;
3822 
3823         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3824 
3825         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3826             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3827 
3828         if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3829                 /* Clear PMA bit */
3830                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3831                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3832                     (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3833 
3834                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3835                     "ahci_dealloc_pmult: "
3836                     "PxCMD.PMA bit cleared at port %d.", port);
3837         }
3838 
3839         /* Release port multiplier information structure */
3840         if (ahci_portp->ahciport_pmult_info != NULL) {
3841                 kmem_free(ahci_portp->ahciport_pmult_info,
3842                     sizeof (ahci_pmult_info_t));
3843                 ahci_portp->ahciport_pmult_info = NULL;
3844         }
3845 }
3846 
3847 /*
3848  * Staggered Spin-up.
3849  */
3850 static void
3851 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3852 {
3853         uint32_t cap_status;
3854         uint32_t port_cmd_status;
3855 
3856         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3857 
3858         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3859             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3860 
3861         /* Check for staggered spin-up support */
3862         if (!(cap_status & AHCI_HBA_CAP_SSS))
3863                 return;
3864 
3865         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3866             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3867 
3868         /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3869         if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3870                 return;
3871 
3872         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3873 
3874         /* Set PxCMD.SUD */
3875         port_cmd_status |= AHCI_CMD_STATUS_SUD;
3876         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3877             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3878             port_cmd_status);
3879 }
3880 
3881 /*
3882  * The routine is to initialize a port. First put the port in NotRunning
3883  * state, then enable port interrupt and clear Serror register. And under
3884  * AHCI_ATTACH case, find device signature and then try to start the port.
3885  *
3886  * Called by
3887  *    1. ahci_initialize_controller
3888  *    2. ahci_intr_phyrdy_change (hotplug)
3889  */
3890 static int
3891 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3892     ahci_addr_t *addrp)
3893 {
3894         uint32_t port_sstatus, port_task_file, port_cmd_status;
3895         uint8_t port = addrp->aa_port;
3896         boolean_t resuming = B_TRUE;    /*  processing DDI_RESUME */
3897         int ret;
3898 
3899         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3900 
3901         /* AHCI_ADDR_PORT: We've no idea of the attached device here.  */
3902         ASSERT(AHCI_ADDR_IS_PORT(addrp));
3903 
3904         /*
3905          * At the time being, only probe ports/devices and get the types of
3906          * attached devices during DDI_ATTACH. In fact, the device can be
3907          * changed during power state changes, but at the time being, we
3908          * don't support the situation.
3909          */
3910         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
3911                 resuming = B_FALSE;
3912         } else {
3913                 /* check for DDI_RESUME case */
3914                 mutex_exit(&ahci_portp->ahciport_mutex);
3915                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3916                 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
3917                         resuming = B_FALSE;
3918                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3919                 mutex_enter(&ahci_portp->ahciport_mutex);
3920         }
3921 
3922         if (resuming) {
3923                 /*
3924                  * During the resume, we need to set the PxCLB, PxCLBU, PxFB
3925                  * and PxFBU registers in case these registers were cleared
3926                  * during the suspend.
3927                  */
3928                 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
3929                     "ahci_initialize_port: port %d "
3930                     "set PxCLB, PxCLBU, PxFB and PxFBU "
3931                     "during resume", port);
3932 
3933                 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
3934                     AHCI_SUCCESS)
3935                         return (AHCI_FAILURE);
3936         }
3937 
3938         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3939             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3940 
3941         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3942             "ahci_initialize_port: port %d ", port);
3943 
3944         /*
3945          * Check whether the port is in NotRunning state, if not,
3946          * put the port in NotRunning state
3947          */
3948         if (port_cmd_status &
3949             (AHCI_CMD_STATUS_ST |
3950             AHCI_CMD_STATUS_CR |
3951             AHCI_CMD_STATUS_FRE |
3952             AHCI_CMD_STATUS_FR)) {
3953                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3954                     ahci_portp, port);
3955         }
3956 
3957         /* Make sure the drive is spun-up */
3958         ahci_staggered_spin_up(ahci_ctlp, port);
3959 
3960         /* Disable interrupt */
3961         ahci_disable_port_intrs(ahci_ctlp, port);
3962 
3963         /* Device is unknown at first */
3964         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
3965 
3966         /* Disable the interface power management */
3967         ahci_disable_interface_pm(ahci_ctlp, port);
3968 
3969         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3970             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
3971         port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3972             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
3973 
3974         /* Check physcial link status */
3975         if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
3976             SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
3977 
3978             /* Check interface status */
3979             port_task_file & AHCI_TFD_STS_BSY ||
3980             port_task_file & AHCI_TFD_STS_DRQ ||
3981 
3982             /* Check whether port reset must be executed */
3983             ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
3984 
3985             /* Always reset port on RESUME */
3986             resuming != B_FALSE) {
3987 
3988                 /* Something went wrong, we need do some reset things */
3989                 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
3990 
3991                 /* Does port reset succeed on HBA port? */
3992                 if (ret != AHCI_SUCCESS) {
3993                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
3994                             "ahci_initialize_port:"
3995                             "port reset failed at port %d", port);
3996                         return (AHCI_FAILURE);
3997                 }
3998 
3999                 /* Is port failed? */
4000                 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4001                     SATA_PSTATE_FAILED) {
4002                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4003                             "ahci_initialize_port: port %d state 0x%x",
4004                             port, ahci_portp->ahciport_port_state);
4005                         return (AHCI_FAILURE);
4006                 }
4007         }
4008 
4009         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4010         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4011 
4012         /*
4013          * Try to get the device signature if the port is not empty.
4014          */
4015         if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4016             SATA_DTYPE_NONE)
4017                 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4018 
4019         /* Return directly if no device connected */
4020         if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4021                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4022                     "No device connected to port %d", port);
4023                 goto out;
4024         }
4025 
4026         /* If this is a port multiplier, we need do some initialization */
4027         if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4028                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4029                     "Port multiplier found at port %d", port);
4030                 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4031         }
4032 
4033         /* Try to start the port */
4034         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4035             != AHCI_SUCCESS) {
4036                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4037                     "failed to start port %d", port);
4038                 return (AHCI_FAILURE);
4039         }
4040 out:
4041         /* Enable port interrupts */
4042         ahci_enable_port_intrs(ahci_ctlp, port);
4043 
4044         return (AHCI_SUCCESS);
4045 }
4046 
4047 /*
4048  *  Handle hardware defect, and check the capabilities. For example,
4049  *  power management capabilty and MSI capability.
4050  */
4051 static int
4052 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4053 {
4054         ushort_t caps_ptr, cap_count, cap;
4055 #if AHCI_DEBUG
4056         ushort_t pmcap, pmcsr;
4057         ushort_t msimc;
4058 #endif
4059         uint8_t revision;
4060 
4061         ahci_ctlp->ahcictl_venid =
4062             pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4063             PCI_CONF_VENID);
4064 
4065         ahci_ctlp->ahcictl_devid =
4066             pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4067             PCI_CONF_DEVID);
4068 
4069         /*
4070          * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4071          * controllers with 0x00 revision id work on 4-byte aligned buffer,
4072          * which is a bug and was fixed after 0x00 revision id controllers.
4073          *
4074          * Moreover, VT8251 cannot use multiple command slots in the command
4075          * list for non-queued commands because the previous register content
4076          * of PxCI can be re-written in the register write, so a flag will be
4077          * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4078          *
4079          * For VT8251, software reset also has the same defect as the below
4080          * AMD/ATI chipset. That is, software reset will get failed if 0xf
4081          * is filled in pmport field. Therefore, another software reset need
4082          * to be done with 0 filled in pmport field.
4083          */
4084         if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4085                 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4086                     PCI_CONF_REVID);
4087                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4088                     "revision id = 0x%x", revision);
4089                 if (revision == 0x00) {
4090                         ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4091                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4092                             "change ddi_attr_align to 0x4", NULL);
4093                 }
4094 
4095                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4096                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4097                     "VT8251 cannot use multiple command lists for "
4098                     "non-queued commands", NULL);
4099 
4100                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4101         }
4102 
4103         /*
4104          * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4105          * DMA addressing for communication memory descriptors though S64A bit
4106          * of CAP register declares it supports. Even though 64-bit DMA for
4107          * data buffer works on ASUS M2A-VM with newer BIOS, three other
4108          * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4109          * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4110          *
4111          * Due to certain hardware issue, the chipset must do port reset during
4112          * initialization, otherwise, when retrieving device signature,
4113          * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4114          * need to set.
4115          *
4116          * For this chipset software reset will get failure if the pmport of
4117          * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4118          * multiplier is connected to the port. In order to fix the issue,
4119          * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4120          * reset got failure, the driver will try to do another software reset
4121          * with pmport 0.
4122          */
4123         if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4124             ahci_ctlp->ahcictl_devid == 0x4380) {
4125                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4126                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4127                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4128                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4129 
4130                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4131                     "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4132                     "communication memory descriptors though CAP indicates "
4133                     "support, so force it to use 32-bit DMA", NULL);
4134                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4135                     "ATI SB600 need to do a port reset during initialization",
4136                     NULL);
4137                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4138                     "ATI SB600 will get software reset failure if pmport "
4139                     "is set 0xf and no port multiplier is attached", NULL);
4140         }
4141 
4142         /*
4143          * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4144          * vendor ID and device ID (0x1002,0x4391).
4145          *
4146          * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4147          * DMA addressing for communication memory descriptors though S64A bit
4148          * of CAP register declares the support. However, it does support
4149          * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4150          * set for this controller.
4151          *
4152          * SB710 has the same initialization issue as SB600, so it also need
4153          * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4154          *
4155          * SB700 also has the same issue about software reset, and thus
4156          * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4157          */
4158         if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4159             ahci_ctlp->ahcictl_devid == 0x4391) {
4160                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4161                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4162                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4163 
4164                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4165                     "ATI SB700/750 cannot do 64-bit DMA for communication "
4166                     "memory descriptors though CAP indicates support, "
4167                     "so force it to use 32-bit DMA", NULL);
4168                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4169                     "ATI SB710 need to do a port reset during initialization",
4170                     NULL);
4171                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4172                     "ATI SB700 will get software reset failure if pmport "
4173                     "is set 0xf and no port multiplier is attached", NULL);
4174         }
4175 
4176         /*
4177          * Check if capabilities list is supported and if so,
4178          * get initial capabilities pointer and clear bits 0,1.
4179          */
4180         if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4181             PCI_CONF_STAT) & PCI_STAT_CAP) {
4182                 caps_ptr = P2ALIGN(pci_config_get8(
4183                     ahci_ctlp->ahcictl_pci_conf_handle,
4184                     PCI_CONF_CAP_PTR), 4);
4185         } else {
4186                 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4187         }
4188 
4189         /*
4190          * Walk capabilities if supported.
4191          */
4192         for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4193 
4194                 /*
4195                  * Check that we haven't exceeded the maximum number of
4196                  * capabilities and that the pointer is in a valid range.
4197                  */
4198                 if (++cap_count > PCI_CAP_MAX_PTR) {
4199                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4200                             "too many device capabilities", NULL);
4201                         return (AHCI_FAILURE);
4202                 }
4203                 if (caps_ptr < PCI_CAP_PTR_OFF) {
4204                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4205                             "capabilities pointer 0x%x out of range",
4206                             caps_ptr);
4207                         return (AHCI_FAILURE);
4208                 }
4209 
4210                 /*
4211                  * Get next capability and check that it is valid.
4212                  * For now, we only support power management.
4213                  */
4214                 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4215                     caps_ptr);
4216                 switch (cap) {
4217                 case PCI_CAP_ID_PM:
4218 
4219                         /* power management supported */
4220                         ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4221 
4222                         /* Save PMCSR offset */
4223                         ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4224 
4225 #if AHCI_DEBUG
4226                         pmcap = pci_config_get16(
4227                             ahci_ctlp->ahcictl_pci_conf_handle,
4228                             caps_ptr + PCI_PMCAP);
4229                         pmcsr = pci_config_get16(
4230                             ahci_ctlp->ahcictl_pci_conf_handle,
4231                             ahci_ctlp->ahcictl_pmcsr_offset);
4232                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4233                             "Power Management capability found PCI_PMCAP "
4234                             "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4235                         if ((pmcap & 0x3) == 0x3)
4236                                 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4237                                     "PCI Power Management Interface "
4238                                     "spec 1.2 compliant", NULL);
4239 #endif
4240                         break;
4241 
4242                 case PCI_CAP_ID_MSI:
4243 #if AHCI_DEBUG
4244                         msimc = pci_config_get16(
4245                             ahci_ctlp->ahcictl_pci_conf_handle,
4246                             caps_ptr + PCI_MSI_CTRL);
4247                         AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4248                             "Message Signaled Interrupt capability found "
4249                             "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4250 #endif
4251                         AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4252                             "MSI capability found", NULL);
4253                         break;
4254 
4255                 case PCI_CAP_ID_PCIX:
4256                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4257                             "PCI-X capability found", NULL);
4258                         break;
4259 
4260                 case PCI_CAP_ID_PCI_E:
4261                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4262                             "PCI Express capability found", NULL);
4263                         break;
4264 
4265                 case PCI_CAP_ID_MSI_X:
4266                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4267                             "MSI-X capability found", NULL);
4268                         break;
4269 
4270                 case PCI_CAP_ID_SATA:
4271                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4272                             "SATA capability found", NULL);
4273                         break;
4274 
4275                 case PCI_CAP_ID_VS:
4276                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4277                             "Vendor Specific capability found", NULL);
4278                         break;
4279 
4280                 default:
4281                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4282                             "unrecognized capability 0x%x", cap);
4283                         break;
4284                 }
4285 
4286                 /*
4287                  * Get next capabilities pointer and clear bits 0,1.
4288                  */
4289                 caps_ptr = P2ALIGN(pci_config_get8(
4290                     ahci_ctlp->ahcictl_pci_conf_handle,
4291                     (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4292         }
4293 
4294         return (AHCI_SUCCESS);
4295 }
4296 
4297 /*
4298  * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4299  * PORTMULT command. SYNC & POLLING mode is used.
4300  */
4301 static int
4302 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4303     uint8_t regn, uint32_t *pregv, uint8_t type)
4304 {
4305         ahci_port_t *ahci_portp;
4306         ahci_addr_t pmult_addr;
4307         sata_pkt_t *spkt;
4308         sata_cmd_t *scmd;
4309         sata_device_t sata_device;
4310         uint8_t port = addrp->aa_port;
4311         uint8_t pmport = addrp->aa_pmport;
4312         uint8_t cport;
4313         uint32_t intr_mask;
4314         int rval;
4315         char portstr[10];
4316 
4317         SET_PORTSTR(portstr, addrp);
4318         cport = ahci_ctlp->ahcictl_port_to_cport[port];
4319         ahci_portp = ahci_ctlp->ahcictl_ports[port];
4320 
4321         ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4322         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4323 
4324         /* Check the existence of the port multiplier */
4325         if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4326                 return (AHCI_FAILURE);
4327 
4328         /* Request a READ/WRITE PORTMULT sata packet. */
4329         bzero(&sata_device, sizeof (sata_device_t));
4330         sata_device.satadev_addr.cport = cport;
4331         sata_device.satadev_addr.pmport = pmport;
4332         sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4333         sata_device.satadev_rev = SATA_DEVICE_REV;
4334 
4335         /*
4336          * Make sure no command is outstanding here. All R/W PMULT requests
4337          * come from
4338          *
4339          * 1. ahci_attach()
4340          *    The port should be empty.
4341          *
4342          * 2. ahci_tran_probe_port()
4343          *    Any request from SATA framework (via ahci_tran_start) should be
4344          *    rejected if R/W PMULT command is outstanding.
4345          *
4346          *    If we are doing mopping, do not check those flags because no
4347          *    command will be actually outstanding.
4348          *
4349          *    If the port has been occupied by any other commands, the probe
4350          *    function will return a SATA_RETRY. SATA framework will retry
4351          *    later.
4352          */
4353         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4354                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4355                     "R/W PMULT failed: R/W PMULT in progress at port %d.",
4356                     port, ahci_portp->ahciport_flags);
4357                 return (AHCI_FAILURE);
4358         }
4359 
4360         if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4361             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4362             NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4363             NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4364                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4365                     "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4366                     port, ahci_portp->ahciport_flags);
4367                 return (AHCI_FAILURE);
4368         }
4369 
4370         /*
4371          * The port multiplier is gone. This may happen when
4372          * 1. Cutting off the power of an enclosure. The device lose the power
4373          *    before port multiplier.
4374          * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4375          *
4376          * The issued command should be aborted and the following command
4377          * should not be continued.
4378          */
4379         if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4380                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4381                     "READ/WRITE PMULT failed: "
4382                     "port-mult is removed from port %d", port);
4383                 return (AHCI_FAILURE);
4384         }
4385 
4386         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4387 
4388         spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4389             &sata_device, regn, *pregv, type);
4390 
4391         /*
4392          * READ/WRITE PORTMULT command is intended to sent to the control port
4393          * of the port multiplier.
4394          */
4395         AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4396 
4397         ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4398 
4399         /* No interrupt here. Store the interrupt enable mask. */
4400         intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4401             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4402         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4403             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4404 
4405         rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4406 
4407         if (rval == AHCI_SUCCESS &&
4408             spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4409                 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4410                         scmd = &spkt->satapkt_cmd;
4411                         *pregv = scmd->satacmd_lba_high_lsb << 24 |
4412                             scmd->satacmd_lba_mid_lsb << 16 |
4413                             scmd->satacmd_lba_low_lsb << 8 |
4414                             scmd->satacmd_sec_count_lsb;
4415                 }
4416         } else {
4417                 /* Failed or not completed. */
4418                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4419                     "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4420                     type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4421                     AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4422                 rval = AHCI_FAILURE;
4423         }
4424 out:
4425         /* Restore the interrupt mask */
4426         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4427             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4428 
4429         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4430         ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4431         sata_free_rdwr_pmult_pkt(spkt);
4432         return (rval);
4433 }
4434 
4435 static int
4436 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4437     uint8_t regn, uint32_t *pregv)
4438 {
4439         return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4440             SATA_RDWR_PMULT_PKT_TYPE_READ);
4441 }
4442 
4443 static int
4444 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4445     uint8_t regn, uint32_t regv)
4446 {
4447         return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, &regv,
4448             SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4449 }
4450 
4451 #define READ_PMULT(addrp, r, pv, out)                                   \
4452         if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS)   \
4453                 goto out;
4454 
4455 #define WRITE_PMULT(addrp, r, v, out)                                   \
4456         if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS)   \
4457                 goto out;
4458 
4459 /*
4460  * Update sata registers on port multiplier, including GSCR/PSCR registers.
4461  * ahci_update_pmult_gscr()
4462  * ahci_update_pmult_pscr()
4463  */
4464 static int
4465 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4466     sata_pmult_gscr_t *sg)
4467 {
4468         ASSERT(MUTEX_HELD(
4469             &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4470 
4471         READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4472         READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4473         READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4474         READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4475 
4476         return (AHCI_SUCCESS);
4477 
4478 err:    /* R/W PMULT error */
4479         return (AHCI_FAILURE);
4480 }
4481 
4482 static int
4483 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4484     sata_device_t *sd)
4485 {
4486         ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4487         ASSERT(MUTEX_HELD(
4488             &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4489 
4490         READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4491         READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4492         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4493         READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4494 
4495         return (AHCI_SUCCESS);
4496 
4497 err:    /* R/W PMULT error */
4498         return (AHCI_FAILURE);
4499 }
4500 
4501 /*
4502  * ahci_initialize_pmult()
4503  *
4504  * Initialize a port multiplier, including
4505  * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4506  * 2. Redefine MASK register. (SATA Chap 16.?)
4507  */
4508 static int
4509 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4510     ahci_addr_t *addrp, sata_device_t *sd)
4511 {
4512         sata_pmult_gscr_t sg;
4513         uint32_t gscr64;
4514         uint8_t port = addrp->aa_port;
4515 
4516         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4517 
4518         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4519             "[Initialize] Port-multiplier at port %d.", port);
4520 
4521         /*
4522          * Enable features of port multiplier. Currently only
4523          * Asynchronous Notification is enabled.
4524          */
4525         /* Check gscr64 for supported features. */
4526         READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4527 
4528         if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4529                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4530                     "port %d: Port Multiplier supports "
4531                     "Asynchronous Notification.", port);
4532 
4533                 /* Write to gscr96 to enabled features */
4534                 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4535                     SATA_PMULT_CAP_SNOTIF, err);
4536 
4537                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4538                     (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4539                     AHCI_SNOTIF_CLEAR_ALL);
4540                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4541                     "port %d: PMult PxSNTF cleared.", port);
4542 
4543         }
4544 
4545         /*
4546          * Now we need to update gscr33 register to enable hot-plug interrupt
4547          * for sub devices behind port multiplier.
4548          */
4549         WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4550         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4551             "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4552 
4553         /*
4554          * Fetch the number of device ports of the port multiplier
4555          */
4556         if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4557                 return (AHCI_FAILURE);
4558 
4559         /* Register the port multiplier to SATA Framework. */
4560         mutex_exit(&ahci_portp->ahciport_mutex);
4561         sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4562         mutex_enter(&ahci_portp->ahciport_mutex);
4563 
4564         ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4565             sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4566 
4567         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4568             "port %d: pmult sub-port number updated to %x.", port,
4569             ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4570 
4571         /* Till now port-mult is successfully initialized */
4572         ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4573         return (AHCI_SUCCESS);
4574 
4575 err:    /* R/W PMULT error */
4576         return (AHCI_FAILURE);
4577 }
4578 
4579 /*
4580  * Initialize a port multiplier port. According to spec, firstly we need
4581  * issue a COMRESET, then a software reset to get its signature.
4582  *
4583  * NOTE: This function should only be called in ahci_probe_pmport()
4584  */
4585 static int
4586 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4587     ahci_addr_t *addrp)
4588 {
4589         uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4590         uint8_t port = addrp->aa_port;
4591         uint8_t pmport = addrp->aa_pmport;
4592         int ret = AHCI_FAILURE;
4593 
4594         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4595         ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4596 
4597         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4598             "ahci_initialize_pmport: port %d:%d", port, pmport);
4599 
4600         /* Check HBA port state */
4601         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4602                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4603                     "ahci_initialize_pmport:"
4604                     "port %d:%d Port Multiplier is failed.",
4605                     port, pmport);
4606                 return (AHCI_FAILURE);
4607         }
4608 
4609         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4610                 return (AHCI_FAILURE);
4611         }
4612         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4613 
4614         /* Checking for outstanding commands */
4615         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4616                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4617                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4618                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4619         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4620                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4621                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4622                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4623         }
4624 
4625         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4626         ahci_portp->ahciport_mop_in_progress++;
4627 
4628         /* Clear status */
4629         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4630 
4631         /* Firstly assume an unknown device */
4632         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4633 
4634         ahci_disable_port_intrs(ahci_ctlp, port);
4635 
4636         /* port reset is necessary for port multiplier port */
4637         if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4638                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4639                     "ahci_initialize_pmport:"
4640                     "port reset failed at port %d:%d",
4641                     port, pmport);
4642                 goto out;
4643         }
4644 
4645         /* Is port failed? */
4646         if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4647             SATA_PSTATE_FAILED) {
4648                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4649                     "ahci_initialize_pmport: port %d:%d failed. "
4650                     "state = 0x%x", port, pmport,
4651                     ahci_portp->ahciport_port_state);
4652                 goto out;
4653         }
4654 
4655         /* Is there any device attached? */
4656         if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4657             == SATA_DTYPE_NONE) {
4658                 /* Do not waste time on an empty port */
4659                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4660                     "ahci_initialize_pmport: No device is found "
4661                     "at port %d:%d", port, pmport);
4662                 ret = AHCI_SUCCESS;
4663                 goto out;
4664         }
4665 
4666         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4667         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4668             "port %d:%d is ready now.", port, pmport);
4669 
4670         /*
4671          * Till now we can assure a device attached to that HBA port and work
4672          * correctly. Now try to get the device signature. This is an optional
4673          * step. If failed, unknown device is assumed, then SATA module will
4674          * continue to use IDENTIFY DEVICE to get the information of the
4675          * device.
4676          */
4677         ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4678 
4679         ret = AHCI_SUCCESS;
4680 
4681 out:
4682         /* Next try to mop the pending commands */
4683         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4684                 finished_tags = ahci_portp->ahciport_pending_tags &
4685                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4686         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4687                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4688                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4689         reset_tags &= ~finished_tags;
4690 
4691         ahci_mop_commands(ahci_ctlp,
4692             ahci_portp,
4693             slot_status,
4694             0, /* failed tags */
4695             0, /* timeout tags */
4696             0, /* aborted tags */
4697             reset_tags); /* reset tags */
4698 
4699         /* Clear PxSNTF register if supported. */
4700         if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4701                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4702                     (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4703                     AHCI_SNOTIF_CLEAR_ALL);
4704         }
4705 
4706         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4707         ahci_enable_port_intrs(ahci_ctlp, port);
4708         return (ret);
4709 }
4710 
4711 /*
4712  * ahci_probe_pmult()
4713  *
4714  * This function will be called to probe a port multiplier, which will
4715  * handle hotplug events on port multiplier ports.
4716  *
4717  * NOTE: Only called from ahci_tran_probe_port()
4718  */
4719 static int
4720 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4721     ahci_addr_t *addrp)
4722 {
4723         sata_device_t sdevice;
4724         ahci_addr_t pmport_addr;
4725         uint32_t gscr32, port_hotplug_tags;
4726         uint32_t pmport_sstatus;
4727         int dev_exists_now = 0, dev_existed_previously = 0;
4728         uint8_t port = addrp->aa_port;
4729         int npmport;
4730 
4731         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4732 
4733         /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4734         READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4735         port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4736 
4737         do {
4738                 npmport = ddi_ffs(port_hotplug_tags) - 1;
4739                 if (npmport == -1)
4740                         /* no pending hot plug events. */
4741                         return (AHCI_SUCCESS);
4742 
4743                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4744                     "hot-plug event at port %d:%d", port, npmport);
4745 
4746                 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4747 
4748                 /* Check previous device at that port */
4749                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4750                     != SATA_DTYPE_NONE)
4751                         dev_existed_previously = 1;
4752 
4753                 /* PxSStatus tells the presence of device. */
4754                 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4755                     &pmport_sstatus, err);
4756 
4757                 if (SSTATUS_GET_DET(pmport_sstatus) ==
4758                     SSTATUS_DET_DEVPRE_PHYCOM)
4759                         dev_exists_now = 1;
4760 
4761                 /*
4762                  * Clear PxSERR is critical. The transition from 0 to 1 will
4763                  * emit a FIS which generates an asynchronous notification
4764                  * event at controller. If we fail to clear the PxSERR, the
4765                  * Async Notif events will no longer be activated on this
4766                  * pmport.
4767                  */
4768                 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4769                     AHCI_SERROR_CLEAR_ALL, err);
4770 
4771                 bzero((void *)&sdevice, sizeof (sata_device_t));
4772                 sdevice.satadev_addr.cport = ahci_ctlp->
4773                     ahcictl_port_to_cport[port];
4774                 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4775                 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4776                 sdevice.satadev_state = SATA_PSTATE_PWRON;
4777 
4778                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4779                     "[Existence] %d -> %d", dev_existed_previously,
4780                     dev_exists_now);
4781 
4782                 if (dev_exists_now) {
4783                         if (dev_existed_previously) {
4784                                 /* Link (may) not change: Exist -> Exist * */
4785                                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4786                                     "ahci_probe_pmult: port %d:%d "
4787                                     "device link lost/established",
4788                                     port, npmport);
4789 
4790                                 mutex_exit(&ahci_portp->ahciport_mutex);
4791                                 sata_hba_event_notify(
4792                                     ahci_ctlp->ahcictl_sata_hba_tran->
4793                                     sata_tran_hba_dip,
4794                                     &sdevice,
4795                                     SATA_EVNT_LINK_LOST|
4796                                     SATA_EVNT_LINK_ESTABLISHED);
4797                                 mutex_enter(&ahci_portp->ahciport_mutex);
4798                         } else {
4799                                 /* Link change: None -> Exist */
4800                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4801                                     "ahci_probe_pmult: port %d:%d "
4802                                     "device link established", port, npmport);
4803 
4804                                 /* Clear port state */
4805                                 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4806                                     SATA_STATE_UNKNOWN);
4807                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4808                                     "ahci_probe_pmult: port %d "
4809                                     "ahciport_port_state [Cleared].", port);
4810 
4811                                 mutex_exit(&ahci_portp->ahciport_mutex);
4812                                 sata_hba_event_notify(
4813                                     ahci_ctlp->ahcictl_sata_hba_tran->
4814                                     sata_tran_hba_dip,
4815                                     &sdevice,
4816                                     SATA_EVNT_LINK_ESTABLISHED);
4817                                 mutex_enter(&ahci_portp->ahciport_mutex);
4818                         }
4819                 } else { /* No device exists now */
4820                         if (dev_existed_previously) {
4821 
4822                                 /* Link change: Exist -> None */
4823                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4824                                     "ahci_probe_pmult: port %d:%d "
4825                                     "device link lost", port, npmport);
4826 
4827                                 /* An existing device is lost. */
4828                                 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4829                                     SATA_STATE_UNKNOWN);
4830                                 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4831                                     SATA_DTYPE_NONE);
4832 
4833                                 mutex_exit(&ahci_portp->ahciport_mutex);
4834                                 sata_hba_event_notify(
4835                                     ahci_ctlp->ahcictl_sata_hba_tran->
4836                                     sata_tran_hba_dip,
4837                                     &sdevice,
4838                                     SATA_EVNT_LINK_LOST);
4839                                 mutex_enter(&ahci_portp->ahciport_mutex);
4840                         }
4841                 }
4842 
4843                 CLEAR_BIT(port_hotplug_tags, npmport);
4844         } while (port_hotplug_tags != 0);
4845 
4846         return (AHCI_SUCCESS);
4847 
4848 err:    /* R/W PMULT error */
4849         return (AHCI_FAILURE);
4850 }
4851 
4852 /*
4853  * Probe and initialize a port multiplier port.
4854  * A port multiplier port could only be initilaizer here.
4855  */
4856 static int
4857 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4858     ahci_addr_t *addrp, sata_device_t *sd)
4859 {
4860         uint32_t port_state;
4861         uint8_t port = addrp->aa_port;
4862         ahci_addr_t addr_pmult;
4863 
4864         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4865 
4866         /*
4867          * Check the parent - port multiplier first.
4868          */
4869 
4870         /*
4871          * Parent port multiplier might have been removed. This event will be
4872          * ignored and failure.
4873          */
4874         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4875             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4876                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4877                     "ahci_tran_probe_port: "
4878                     "parent device removed, ignore event.", NULL);
4879 
4880                 return (AHCI_FAILURE);
4881         }
4882 
4883         /* The port is ready? */
4884         port_state = ahci_portp->ahciport_port_state;
4885         if (!(port_state & SATA_STATE_READY)) {
4886                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4887                     "ahci_tran_probe_port: "
4888                     "parent port-mult is NOT ready.", NULL);
4889 
4890                 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4891                     ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4892                     AHCI_SUCCESS) {
4893                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4894                             "ahci_tran_probe_port: "
4895                             "restart port-mult failed.", NULL);
4896                         return (AHCI_FAILURE);
4897                 }
4898         }
4899 
4900         /*
4901          * If port-mult is restarted due to some reason, we need
4902          * re-initialized the PMult.
4903          */
4904         if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
4905                 /* Initialize registers on a port multiplier */
4906                 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
4907                 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
4908                     &addr_pmult, sd) != AHCI_SUCCESS)
4909                         return (AHCI_FAILURE);
4910         }
4911 
4912         /*
4913          * Then we check the port-mult port
4914          */
4915         /* Is this pmport initialized? */
4916         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
4917         if (!(port_state & SATA_STATE_READY)) {
4918 
4919                 /* ahci_initialize_pmport() will set READY state */
4920                 if (ahci_initialize_pmport(ahci_ctlp,
4921                     ahci_portp, addrp) != AHCI_SUCCESS)
4922                         return (AHCI_FAILURE);
4923         }
4924 
4925         return (AHCI_SUCCESS);
4926 }
4927 
4928 /*
4929  * AHCI device reset ...; a single device on one of the ports is reset,
4930  * but the HBA and physical communication remain intact. This is the
4931  * least intrusive.
4932  *
4933  * When issuing a software reset sequence, there should not be other
4934  * commands in the command list, so we will first clear and then re-set
4935  * PxCMD.ST to clear PxCI. And before issuing the software reset,
4936  * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
4937  * cleared unless command list override (PxCMD.CLO) is supported.
4938  */
4939 static int
4940 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4941     ahci_addr_t *addrp)
4942 {
4943         ahci_fis_h2d_register_t *h2d_register_fisp;
4944         ahci_cmd_table_t *cmd_table;
4945         ahci_cmd_header_t *cmd_header;
4946         uint32_t port_cmd_status, port_cmd_issue, port_task_file;
4947         int slot, loop_count;
4948         uint8_t port = addrp->aa_port;
4949         uint8_t pmport = addrp->aa_pmport;
4950         int rval = AHCI_FAILURE;
4951 
4952         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4953 
4954         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
4955             "port %d:%d device software resetting (FIS)", port, pmport);
4956 
4957         /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
4958         if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
4959             port) != AHCI_SUCCESS) {
4960                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4961                     "ahci_software_reset: cannot stop HBA port %d.", port);
4962                 goto out;
4963         }
4964 
4965         /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
4966         port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4967             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4968 
4969         if (port_task_file & AHCI_TFD_STS_BSY ||
4970             port_task_file & AHCI_TFD_STS_DRQ) {
4971                 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
4972                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4973                             "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
4974                             "cannot issue a software reset.", port_task_file);
4975                         goto out;
4976                 }
4977 
4978                 /*
4979                  * If HBA Support CLO, as Command List Override (CAP.SCLO is
4980                  * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
4981                  * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
4982                  */
4983                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4984                     "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
4985 
4986                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4987                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4988                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4989                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
4990                     port_cmd_status|AHCI_CMD_STATUS_CLO);
4991 
4992                 /* Waiting till PxCMD.SCLO bit is cleared */
4993                 loop_count = 0;
4994                 do {
4995                         /* Wait for 10 millisec */
4996                         drv_usecwait(AHCI_10MS_USECS);
4997 
4998                         /* We are effectively timing out after 1 sec. */
4999                         if (loop_count++ > 100) {
5000                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5001                                     "SCLO time out. port %d is busy.", port);
5002                                 goto out;
5003                         }
5004 
5005                         port_cmd_status =
5006                             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5007                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5008                 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5009 
5010                 /* Re-check */
5011                 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5012                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5013                 if (port_task_file & AHCI_TFD_STS_BSY ||
5014                     port_task_file & AHCI_TFD_STS_DRQ) {
5015                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5016                             "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5017                             port_task_file);
5018                         goto out;
5019                 }
5020         }
5021 
5022         /* Then start port */
5023         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5024             != AHCI_SUCCESS) {
5025                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5026                     "ahci_software_reset: cannot start AHCI port %d.", port);
5027                 goto out;
5028         }
5029 
5030         /*
5031          * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5032          * ahci_port.ahciport_pending_ncq_tags may fail
5033          * ahci_claim_free_slot(). Actually according to spec, by clearing
5034          * PxCMD.ST there is no command outstanding while executing software
5035          * reseting. Hence we directly use slot 0 instead of
5036          * ahci_claim_free_slot().
5037          */
5038         slot = 0;
5039 
5040         /* Now send the first H2D Register FIS with SRST set to 1 */
5041         cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5042         bzero((void *)cmd_table, ahci_cmd_table_size);
5043 
5044         h2d_register_fisp =
5045             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5046 
5047         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5048         SET_FIS_PMP(h2d_register_fisp, pmport);
5049         SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5050 
5051         /* Set Command Header in Command List */
5052         cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5053         BZERO_DESCR_INFO(cmd_header);
5054         BZERO_PRD_BYTE_COUNT(cmd_header);
5055         SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5056         SET_PORT_MULTI_PORT(cmd_header, pmport);
5057 
5058         SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5059         SET_RESET(cmd_header, 1);
5060         SET_WRITE(cmd_header, 1);
5061 
5062         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5063             0,
5064             ahci_cmd_table_size,
5065             DDI_DMA_SYNC_FORDEV);
5066 
5067         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5068             slot * sizeof (ahci_cmd_header_t),
5069             sizeof (ahci_cmd_header_t),
5070             DDI_DMA_SYNC_FORDEV);
5071 
5072         /* Indicate to the HBA that a command is active. */
5073         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5074             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5075             (0x1 << slot));
5076 
5077         loop_count = 0;
5078 
5079         /* Loop till the first command is finished */
5080         do {
5081                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5082                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5083 
5084                 /* We are effectively timing out after 1 sec. */
5085                 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5086                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5087                             "the first SRST FIS is timed out, "
5088                             "loop_count = %d", loop_count);
5089                         goto out;
5090                 }
5091                 /* Wait for 10 millisec */
5092                 drv_usecwait(AHCI_10MS_USECS);
5093         } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5094 
5095         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5096             "ahci_software_reset: 1st loop count: %d, "
5097             "port_cmd_issue = 0x%x, slot = 0x%x",
5098             loop_count, port_cmd_issue, slot);
5099 
5100         /* According to ATA spec, we need wait at least 5 microsecs here. */
5101         drv_usecwait(AHCI_1MS_USECS);
5102 
5103         /* Now send the second H2D Register FIS with SRST cleard to zero */
5104         cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5105         bzero((void *)cmd_table, ahci_cmd_table_size);
5106 
5107         h2d_register_fisp =
5108             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5109 
5110         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5111         SET_FIS_PMP(h2d_register_fisp, pmport);
5112 
5113         /* Set Command Header in Command List */
5114         cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5115         BZERO_DESCR_INFO(cmd_header);
5116         BZERO_PRD_BYTE_COUNT(cmd_header);
5117         SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5118         SET_PORT_MULTI_PORT(cmd_header, pmport);
5119 
5120         SET_WRITE(cmd_header, 1);
5121 
5122         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5123             0,
5124             ahci_cmd_table_size,
5125             DDI_DMA_SYNC_FORDEV);
5126 
5127         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5128             slot * sizeof (ahci_cmd_header_t),
5129             sizeof (ahci_cmd_header_t),
5130             DDI_DMA_SYNC_FORDEV);
5131 
5132         /* Indicate to the HBA that a command is active. */
5133         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5134             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5135             (0x1 << slot));
5136 
5137         loop_count = 0;
5138 
5139         /* Loop till the second command is finished */
5140         do {
5141                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5142                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5143 
5144                 /* We are effectively timing out after 1 sec. */
5145                 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5146                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5147                             "the second SRST FIS is timed out, "
5148                             "loop_count = %d", loop_count);
5149                         goto out;
5150                 }
5151 
5152                 /* Wait for 10 millisec */
5153                 drv_usecwait(AHCI_10MS_USECS);
5154         } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5155 
5156         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5157             "ahci_software_reset: 2nd loop count: %d, "
5158             "port_cmd_issue = 0x%x, slot = 0x%x",
5159             loop_count, port_cmd_issue, slot);
5160 
5161         if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5162             (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5163                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5164                     DDI_SERVICE_UNAFFECTED);
5165                 goto out;
5166         }
5167 
5168         rval = AHCI_SUCCESS;
5169 out:
5170         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5171             "ahci_software_reset: %s at port %d:%d",
5172             rval == AHCI_SUCCESS ? "succeed" : "failed",
5173             port, pmport);
5174 
5175         return (rval);
5176 }
5177 
5178 /*
5179  * AHCI port reset ...; the physical communication between the HBA and device
5180  * on a port are disabled. This is more intrusive.
5181  *
5182  * When an HBA or port reset occurs, Phy communication is going to
5183  * be re-established with the device through a COMRESET followed by the
5184  * normal out-of-band communication sequence defined in Serial ATA. At
5185  * the end of reset, the device, if working properly, will send a D2H
5186  * Register FIS, which contains the device signature. When the HBA receives
5187  * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5188  * the PxSIG register with the signature.
5189  *
5190  * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5191  * function is called. If not, it is assumed the interface is in hung
5192  * condition.
5193  */
5194 static int
5195 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5196     ahci_addr_t *addrp)
5197 {
5198         ahci_addr_t pmult_addr;
5199         uint32_t port_cmd_status;
5200         uint32_t port_scontrol, port_sstatus;
5201         uint32_t port_task_file;
5202         uint32_t port_state;
5203         uint8_t port = addrp->aa_port;
5204 
5205         int loop_count;
5206         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5207 
5208         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5209 
5210         /* Target is a port multiplier port? */
5211         if (AHCI_ADDR_IS_PMPORT(addrp))
5212                 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5213 
5214         /* Otherwise it must be an HBA port. */
5215         ASSERT(AHCI_ADDR_IS_PORT(addrp));
5216 
5217         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5218             "Port %d port resetting...", port);
5219         ahci_portp->ahciport_port_state = 0;
5220 
5221         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5222             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5223 
5224         /*
5225          * According to the spec, SUD bit should be set here,
5226          * but JMicron JMB363 doesn't follow it, so print
5227          * a debug message.
5228          */
5229         if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5230                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5231                     "ahci_port_reset: port %d SUD bit not set", port);
5232 
5233         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5234             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5235         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5236 
5237         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5238             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5239             port_scontrol);
5240 
5241         /* Enable PxCMD.FRE to read device */
5242         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5243             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5244             port_cmd_status|AHCI_CMD_STATUS_FRE);
5245 
5246         /*
5247          * The port enters P:StartComm state, and the HBA tells the link layer
5248          * to start communication, which involves sending COMRESET to the
5249          * device. And the HBA resets PxTFD.STS to 7Fh.
5250          *
5251          * Give time for COMRESET to percolate, according to the AHCI
5252          * spec, software shall wait at least 1 millisecond before
5253          * clearing PxSCTL.DET
5254          */
5255         drv_usecwait(AHCI_1MS_USECS * 2);
5256 
5257         /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5258         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5259             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5260         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5261         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5262             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5263             port_scontrol);
5264 
5265         /*
5266          * When a COMINIT is received from the device, then the port enters
5267          * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5268          * PxSSTS.DET to 1h to indicate a device is detected but communication
5269          * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5270          * a COMINIT has been received.
5271          */
5272         /*
5273          * The DET field is valid only if IPM field indicates
5274          * that the interface is in active state.
5275          */
5276         loop_count = 0;
5277         for (;;) {
5278                 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5279                     (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5280 
5281                 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5282                         /*
5283                          * If the interface is not active, the DET field
5284                          * is considered not accurate. So we want to
5285                          * continue looping.
5286                          */
5287                         SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5288                 }
5289 
5290                 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5291                         break;
5292 
5293                 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5294                         /*
5295                          * We are effectively timing out after 0.1 sec.
5296                          */
5297                         break;
5298                 }
5299 
5300                 /* Wait for 10 millisec */
5301                 drv_usecwait(AHCI_10MS_USECS);
5302         }
5303 
5304         AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5305             "ahci_port_reset: 1st loop count: %d, "
5306             "port_sstatus = 0x%x port %d",
5307             loop_count, port_sstatus, port);
5308 
5309         if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5310                 /*
5311                  * Either the port is not active or there
5312                  * is no device present.
5313                  */
5314                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5315                 return (AHCI_SUCCESS);
5316         }
5317 
5318         /* Clear port serror register for the port */
5319         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5320             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5321             AHCI_SERROR_CLEAR_ALL);
5322 
5323         /*
5324          * Devices should return a FIS contains its signature to HBA after
5325          * COMINIT signal. Check whether a D2H Register FIS is received by
5326          * polling PxTFD.STS.
5327          */
5328         loop_count = 0;
5329         for (;;) {
5330                 port_task_file =
5331                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5332                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5333 
5334                 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5335                     AHCI_TFD_STS_ERR)) == 0)
5336                         break;
5337 
5338                 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5339                         /*
5340                          * We are effectively timing out after 11 sec.
5341                          */
5342                         cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5343                             "the device hardware has been initialized and "
5344                             "the power-up diagnostics failed",
5345                             instance, port);
5346 
5347                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5348                             "port %d: some or all of BSY, DRQ and ERR in "
5349                             "PxTFD.STS are not clear. We need another "
5350                             "software reset.", port);
5351 
5352                         /* Clear port serror register for the port */
5353                         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5354                             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5355                             AHCI_SERROR_CLEAR_ALL);
5356 
5357                         AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5358 
5359                         /* Try another software reset. */
5360                         if (ahci_software_reset(ahci_ctlp, ahci_portp,
5361                             &pmult_addr) != AHCI_SUCCESS) {
5362                                 AHCIPORT_SET_STATE(ahci_portp, addrp,
5363                                     SATA_PSTATE_FAILED);
5364                                 return (AHCI_FAILURE);
5365                         }
5366                         break;
5367                 }
5368 
5369                 /* Wait for 10 millisec */
5370                 drv_usecwait(AHCI_10MS_USECS);
5371         }
5372 
5373         AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5374             "ahci_port_reset: 2nd loop count: %d, "
5375             "port_task_file = 0x%x port %d",
5376             loop_count, port_task_file, port);
5377 
5378         /* Clear port serror register for the port */
5379         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5380             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5381             AHCI_SERROR_CLEAR_ALL);
5382 
5383         /* Set port as ready */
5384         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5385         AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5386 
5387         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5388             "ahci_port_reset: succeed at port %d.", port);
5389         return (AHCI_SUCCESS);
5390 }
5391 
5392 /*
5393  * COMRESET on a port multiplier port.
5394  *
5395  * NOTE: Only called in ahci_port_reset()
5396  */
5397 static int
5398 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5399     ahci_addr_t *addrp)
5400 {
5401         uint32_t port_scontrol, port_sstatus, port_serror;
5402         uint32_t port_cmd_status, port_intr_status;
5403         uint32_t port_state;
5404         uint8_t port = addrp->aa_port;
5405         uint8_t pmport = addrp->aa_pmport;
5406         int loop_count;
5407         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5408 
5409         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5410             "port %d:%d: pmport resetting", port, pmport);
5411 
5412         /* Initialize pmport state */
5413         AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5414 
5415         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5416         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5417         WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5418 
5419         /* PxCMD.FRE should be set before. */
5420         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5421             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5422         ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5423         if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5424                 return (AHCI_FAILURE);
5425 
5426         /*
5427          * Give time for COMRESET to percolate, according to the AHCI
5428          * spec, software shall wait at least 1 millisecond before
5429          * clearing PxSCTL.DET
5430          */
5431         drv_usecwait(AHCI_1MS_USECS*2);
5432 
5433         /*
5434          * Fetch the SCONTROL again and rewrite the DET part with 0
5435          * This will generate an Asychronous Notification events.
5436          */
5437         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5438         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5439         WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5440 
5441         /*
5442          * The port enters P:StartComm state, and HBA tells link layer to
5443          * start communication, which involves sending COMRESET to device.
5444          * And the HBA resets PxTFD.STS to 7Fh.
5445          *
5446          * When a COMINIT is received from the device, then the port enters
5447          * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5448          * PxSSTS.DET to 1h to indicate a device is detected but communication
5449          * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5450          * a COMINIT has been received.
5451          */
5452         /*
5453          * The DET field is valid only if IPM field indicates
5454          * that the interface is in active state.
5455          */
5456         loop_count = 0;
5457         do {
5458                 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5459 
5460                 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5461                         /*
5462                          * If the interface is not active, the DET field
5463                          * is considered not accurate. So we want to
5464                          * continue looping.
5465                          */
5466                         SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5467                 }
5468 
5469                 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5470                         /*
5471                          * We are effectively timing out after 0.1 sec.
5472                          */
5473                         break;
5474                 }
5475 
5476                 /* Wait for 10 millisec */
5477                 drv_usecwait(AHCI_10MS_USECS);
5478 
5479         } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5480 
5481         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5482             "ahci_pmport_reset: 1st loop count: %d, "
5483             "port_sstatus = 0x%x port %d:%d",
5484             loop_count, port_sstatus, port, pmport);
5485 
5486         if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5487             (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5488                 /*
5489                  * Either the port is not active or there
5490                  * is no device present.
5491                  */
5492                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5493                     "ahci_pmport_reset: "
5494                     "no device attached to port %d:%d",
5495                     port, pmport);
5496                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5497                 return (AHCI_SUCCESS);
5498         }
5499 
5500         /* Now we can make sure there is a device connected to the port */
5501         /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5502         READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5503 
5504         if (!(port_serror & (1 << 26))) {
5505                 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5506                     "COMINIT signal from the device not received port %d:%d",
5507                     instance, port, pmport);
5508 
5509                 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5510                 return (AHCI_FAILURE);
5511         }
5512 
5513         /*
5514          * After clear PxSERR register, we will receive a D2H FIS.
5515          * Normally this FIS will cause a IPMS error according to AHCI spec
5516          * v1.2 because there is no command outstanding for it. So we need
5517          * to ignore this error.
5518          */
5519         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5520         WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5521 
5522         /* Now we need to check the D2H FIS by checking IPMS error. */
5523         loop_count = 0;
5524         do {
5525                 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5526                     (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5527 
5528                 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5529                         /*
5530                          * No D2H FIS received. This is possible according
5531                          * to SATA 2.6 spec.
5532                          */
5533                         cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5534                             "PxIS.IPMS is not set, we need another "
5535                             "software reset.", port, pmport);
5536 
5537                         break;
5538                 }
5539 
5540                 /* Wait for 10 millisec */
5541                 mutex_exit(&ahci_portp->ahciport_mutex);
5542                 delay(AHCI_10MS_TICKS);
5543                 mutex_enter(&ahci_portp->ahciport_mutex);
5544 
5545         } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5546 
5547         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5548             "ahci_pmport_reset: 2st loop count: %d, "
5549             "port_sstatus = 0x%x port %d:%d",
5550             loop_count, port_sstatus, port, pmport);
5551 
5552         /* Clear IPMS */
5553         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5554             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5555             AHCI_INTR_STATUS_IPMS);
5556         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5557 
5558         /* This pmport is now ready for ahci_tran_start() */
5559         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5560         AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5561 
5562         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5563             "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5564         return (AHCI_SUCCESS);
5565 
5566 err:    /* R/W PMULT error */
5567         /* IPMS flags might be set before. */
5568         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5569         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5570             "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5571 
5572         return (AHCI_FAILURE);
5573 }
5574 
5575 /*
5576  * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5577  * This is the most intrusive.
5578  *
5579  * When an HBA reset occurs, Phy communication will be re-established with
5580  * the device through a COMRESET followed by the normal out-of-band
5581  * communication sequence defined in Serial ATA. At the end of reset, the
5582  * device, if working properly, will send a D2H Register FIS, which contains
5583  * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5584  * and PxTFD.ERR register fields, and updates the PxSIG register with the
5585  * signature.
5586  *
5587  * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5588  */
5589 static int
5590 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5591 {
5592         ahci_port_t *ahci_portp;
5593         uint32_t ghc_control;
5594         uint8_t port;
5595         int loop_count;
5596         int rval = AHCI_SUCCESS;
5597 
5598         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5599             NULL);
5600 
5601         mutex_enter(&ahci_ctlp->ahcictl_mutex);
5602 
5603         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5604             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5605 
5606         /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5607         ghc_control |= AHCI_HBA_GHC_HR;
5608         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5609             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5610 
5611         /*
5612          * Wait until HBA Reset complete or timeout
5613          */
5614         loop_count = 0;
5615         do {
5616                 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5617                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5618 
5619                 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5620                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5621                             "ahci hba reset is timing out, "
5622                             "ghc_control = 0x%x", ghc_control);
5623                         /* We are effectively timing out after 1 sec. */
5624                         break;
5625                 }
5626 
5627                 /* Wait for 10 millisec */
5628                 drv_usecwait(AHCI_10MS_USECS);
5629         } while (ghc_control & AHCI_HBA_GHC_HR);
5630 
5631         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5632             "ahci_hba_reset: 1st loop count: %d, "
5633             "ghc_control = 0x%x", loop_count, ghc_control);
5634 
5635         if (ghc_control & AHCI_HBA_GHC_HR) {
5636                 /* The hba is not reset for some reasons */
5637                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5638                     "hba reset failed: HBA in a hung or locked state", NULL);
5639                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5640                 return (AHCI_FAILURE);
5641         }
5642 
5643         /*
5644          * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5645          */
5646         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5647             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5648         ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5649         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5650             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5651 
5652         mutex_exit(&ahci_ctlp->ahcictl_mutex);
5653 
5654         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5655                 /* Only check implemented ports */
5656                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5657                         continue;
5658                 }
5659 
5660                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5661                 mutex_enter(&ahci_portp->ahciport_mutex);
5662 
5663                 /* Make sure the drive is spun-up */
5664                 ahci_staggered_spin_up(ahci_ctlp, port);
5665 
5666                 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5667                     port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5668                     AHCI_SUCCESS) {
5669                         rval = AHCI_FAILURE;
5670                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5671                             "ahci_hba_reset: port %d failed", port);
5672                         /*
5673                          * Set the port state to SATA_PSTATE_FAILED if
5674                          * failed to initialize it.
5675                          */
5676                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5677                 }
5678 
5679                 mutex_exit(&ahci_portp->ahciport_mutex);
5680         }
5681 
5682         return (rval);
5683 }
5684 
5685 /*
5686  * This routine is only called from AHCI_ATTACH or phyrdy change
5687  * case. It first calls software reset, then stop the port and try to
5688  * read PxSIG register to find the type of device attached to the port.
5689  *
5690  * The caller should make sure a valid device exists on specified port and
5691  * physical communication has been established so that the signature could
5692  * be retrieved by software reset.
5693  *
5694  * NOTE: The port interrupts should be disabled before the function is called.
5695  */
5696 static void
5697 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5698     ahci_addr_t *addrp)
5699 {
5700         ahci_addr_t dev_addr;
5701         uint32_t signature;
5702         uint8_t port = addrp->aa_port;
5703         uint8_t pmport = addrp->aa_pmport;
5704         int rval;
5705 
5706         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5707         ASSERT(AHCI_ADDR_IS_VALID(addrp));
5708 
5709         /*
5710          * If the HBA doesn't support port multiplier, then the driver
5711          * doesn't need to bother to check port multiplier device.
5712          *
5713          * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5714          * connected to Silicon Image 4723, to which the two sata drives
5715          * attached can be set with RAID1, RAID0 or Spanning mode.
5716          *
5717          * We found software reset will get failure if port multiplier address
5718          * 0xf is used by software reset, so just ignore the check since
5719          * ICH7 doesn't support port multiplier device at all.
5720          */
5721         if (AHCI_ADDR_IS_PORT(addrp) &&
5722             (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5723                 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5724                     "ahci_find_dev_signature enter: port %d", port);
5725 
5726                 /*
5727                  * NOTE: when the ahci address is a HBA port, we do not know
5728                  * it is a device or a port multiplier that attached. we need
5729                  * try a software reset at port multiplier address (0xf
5730                  * pmport)
5731                  */
5732                 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5733         } else {
5734                 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5735                     "ahci_find_dev_signature enter: port %d:%d",
5736                     port, pmport);
5737                 dev_addr = *addrp;
5738         }
5739 
5740         /* Assume it is unknown. */
5741         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5742 
5743         /* Issue a software reset to get the signature */
5744         rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5745         if (rval != AHCI_SUCCESS) {
5746 
5747                 /*
5748                  * Try to do software reset again with pmport set with 0 if
5749                  * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5750                  * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5751                  */
5752                 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5753                     (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5754                         dev_addr.aa_pmport = 0;
5755                         rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5756                             &dev_addr);
5757                 }
5758 
5759                 if (rval != AHCI_SUCCESS) {
5760                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5761                             "ahci_find_dev_signature: software reset failed "
5762                             "at port %d:%d, cannot get signature.",
5763                             port, pmport);
5764 
5765                         AHCIPORT_SET_STATE(ahci_portp, addrp,
5766                             SATA_PSTATE_FAILED);
5767                         return;
5768                 }
5769         }
5770 
5771         /*
5772          * ahci_software_reset has started the port, so we need manually stop
5773          * the port again.
5774          */
5775         if (AHCI_ADDR_IS_PORT(addrp)) {
5776                 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5777                     ahci_portp, port) != AHCI_SUCCESS) {
5778                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5779                             "ahci_find_dev_signature: cannot stop port %d.",
5780                             port);
5781                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5782                         return;
5783                 }
5784         }
5785 
5786         /* Now we can make sure that a valid signature is received. */
5787         signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5788             (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5789 
5790         if (AHCI_ADDR_IS_PMPORT(addrp)) {
5791                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5792                     "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5793                     signature, port, pmport);
5794         } else {
5795                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5796                     "ahci_find_dev_signature: signature = 0x%x at port %d",
5797                     signature, port);
5798         }
5799 
5800         /* NOTE: Only support ATAPI device at controller port. */
5801         if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5802                 signature = SATA_DTYPE_UNKNOWN;
5803 
5804         switch (signature) {
5805 
5806         case AHCI_SIGNATURE_DISK:
5807                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5808                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5809                     "Disk is found at port: %d", port);
5810                 break;
5811 
5812         case AHCI_SIGNATURE_ATAPI:
5813                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5814                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5815                     "ATAPI device is found at port: %d", port);
5816                 break;
5817 
5818         case AHCI_SIGNATURE_PORT_MULTIPLIER:
5819                 /* Port Multiplier cannot recursively attached. */
5820                 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5821                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5822                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5823                     "Port Multiplier is found at port: %d", port);
5824                 break;
5825 
5826         default:
5827                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5828                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5829                     "Unknown device is found at port: %d", port);
5830         }
5831 }
5832 
5833 /*
5834  * According to the spec, to reliably detect hot plug removals, software
5835  * must disable interface power management. Software should perform the
5836  * following initialization on a port after a device is attached:
5837  *   Set PxSCTL.IPM to 3h to disable interface state transitions
5838  *   Set PxCMD.ALPE to '0' to disable aggressive power management
5839  *   Disable device initiated interface power management by SET FEATURE
5840  *
5841  * We can ignore the last item because by default the feature is disabled
5842  */
5843 static void
5844 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5845 {
5846         uint32_t port_scontrol, port_cmd_status;
5847 
5848         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5849             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5850         SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5851         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5852             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5853 
5854         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5855             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5856         port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5857         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5858             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5859 }
5860 
5861 /*
5862  * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5863  * to 1, then set it firstly.
5864  *
5865  * Each port contains two major DMA engines. One DMA engine walks through
5866  * the command list, and is controlled by PxCMD.ST. The second DMA engine
5867  * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5868  *
5869  * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5870  * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5871  * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5872  *
5873  * Software shall not set PxCMD.ST to '1' unless a functional device is
5874  * present on the port(as determined by PxTFD.STS.BSY = '0',
5875  * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5876  */
5877 static int
5878 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5879 {
5880         uint32_t port_cmd_status;
5881 
5882         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5883 
5884         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5885 
5886         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5887                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5888                     "the state for port %d is 0x%x",
5889                     port, ahci_portp->ahciport_port_state);
5890                 return (AHCI_FAILURE);
5891         }
5892 
5893         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5894                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5895                     "no device is attached at port %d", port);
5896                 return (AHCI_FAILURE);
5897         }
5898 
5899         /* First to set PxCMD.FRE before setting PxCMD.ST. */
5900         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5901             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5902 
5903         if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
5904                 port_cmd_status |= AHCI_CMD_STATUS_FRE;
5905                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5906                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5907                     port_cmd_status);
5908         }
5909 
5910         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5911             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5912 
5913         port_cmd_status |= AHCI_CMD_STATUS_ST;
5914 
5915         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5916             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5917             port_cmd_status);
5918 
5919         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
5920 
5921         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
5922             "PxCMD.ST set to '1' at port %d", port);
5923 
5924         return (AHCI_SUCCESS);
5925 }
5926 
5927 /*
5928  * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
5929  * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
5930  * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
5931  */
5932 static int
5933 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
5934 {
5935         uint8_t port = ahci_portp->ahciport_port_num;
5936         uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5937             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5938 
5939         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5940 
5941         /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
5942         if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
5943                 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5944                     port) != AHCI_SUCCESS)
5945                         return (AHCI_FAILURE);
5946 
5947                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5948                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5949         }
5950 
5951         /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
5952         if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
5953                 int loop_count = 0;
5954 
5955                 /* Clear PxCMD.FRE */
5956                 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
5957                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5958                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5959                     port_cmd_status);
5960 
5961                 /* Wait until PxCMD.FR is cleared */
5962                 for (;;) {
5963                         port_cmd_status =
5964                             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5965                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5966 
5967                         if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
5968                                 break;
5969 
5970                         if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
5971                                 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
5972                                     "ahci_setup_port_base_addresses: cannot "
5973                                     "clear PxCMD.FR for port %d.", port);
5974 
5975                                 /*
5976                                  * We are effectively timing out after 0.5 sec.
5977                                  * This value is specified in AHCI spec.
5978                                  */
5979                                 return (AHCI_FAILURE);
5980                         }
5981 
5982                         /* Wait for 1 millisec */
5983                         drv_usecwait(AHCI_1MS_USECS);
5984                 }
5985         }
5986 
5987         /* Step 3: Config Port Command List Base Address */
5988         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5989             (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
5990             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
5991 
5992         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5993             (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
5994             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
5995 
5996         /* Step 4: Config Port Received FIS Base Address */
5997         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5998             (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
5999             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6000 
6001         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6002             (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6003             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6004 
6005         return (AHCI_SUCCESS);
6006 }
6007 
6008 /*
6009  * Allocate the ahci_port_t including Received FIS and Command List.
6010  * The argument - port is the physical port number, and not logical
6011  * port number seen by the SATA framework.
6012  */
6013 static int
6014 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6015 {
6016         dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6017         ahci_port_t *ahci_portp;
6018         char taskq_name[64] = "event_handle_taskq";
6019 
6020         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6021 
6022         ahci_portp =
6023             (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6024 
6025         ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6026         ahci_portp->ahciport_port_num = port;
6027 
6028         /* Initialize the port condition variable */
6029         cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6030 
6031         /* Initialize the port mutex */
6032         mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6033             (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6034 
6035         mutex_enter(&ahci_portp->ahciport_mutex);
6036 
6037         /*
6038          * Allocate memory for received FIS structure and
6039          * command list for this port
6040          */
6041         if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6042                 goto err_case1;
6043         }
6044 
6045         if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6046                 goto err_case2;
6047         }
6048 
6049         /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6050         if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6051             AHCI_SUCCESS) {
6052                 goto err_case3;
6053         }
6054 
6055         (void) snprintf(taskq_name + strlen(taskq_name),
6056             sizeof (taskq_name) - strlen(taskq_name),
6057             "_port%d", port);
6058 
6059         /* Create the taskq for the port */
6060         if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6061             taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6062                 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6063                     "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6064                 goto err_case3;
6065         }
6066 
6067         /* Allocate the argument for the taskq */
6068         ahci_portp->ahciport_event_args =
6069             kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6070 
6071         ahci_portp->ahciport_event_args->ahciea_addrp =
6072             kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6073 
6074         if (ahci_portp->ahciport_event_args == NULL)
6075                 goto err_case4;
6076 
6077         /* Initialize the done queue */
6078         ahci_portp->ahciport_doneq = NULL;
6079         ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6080         ahci_portp->ahciport_doneq_len = 0;
6081 
6082         mutex_exit(&ahci_portp->ahciport_mutex);
6083 
6084         return (AHCI_SUCCESS);
6085 
6086 err_case4:
6087         ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6088 
6089 err_case3:
6090         ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6091 
6092 err_case2:
6093         ahci_dealloc_rcvd_fis(ahci_portp);
6094 
6095 err_case1:
6096         mutex_exit(&ahci_portp->ahciport_mutex);
6097         mutex_destroy(&ahci_portp->ahciport_mutex);
6098         cv_destroy(&ahci_portp->ahciport_cv);
6099 
6100         kmem_free(ahci_portp, sizeof (ahci_port_t));
6101 
6102         return (AHCI_FAILURE);
6103 }
6104 
6105 /*
6106  * Reverse of ahci_alloc_port_state().
6107  */
6108 static void
6109 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6110 {
6111         ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6112 
6113         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6114         ASSERT(ahci_portp != NULL);
6115 
6116         mutex_enter(&ahci_portp->ahciport_mutex);
6117         kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6118             sizeof (ahci_addr_t));
6119         ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6120         kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6121         ahci_portp->ahciport_event_args = NULL;
6122         ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6123         ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6124         ahci_dealloc_rcvd_fis(ahci_portp);
6125         ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6126         mutex_exit(&ahci_portp->ahciport_mutex);
6127 
6128         mutex_destroy(&ahci_portp->ahciport_mutex);
6129         cv_destroy(&ahci_portp->ahciport_cv);
6130 
6131         kmem_free(ahci_portp, sizeof (ahci_port_t));
6132 
6133         ahci_ctlp->ahcictl_ports[port] = NULL;
6134 }
6135 
6136 /*
6137  * Allocates memory for the Received FIS Structure
6138  */
6139 static int
6140 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6141 {
6142         size_t rcvd_fis_size;
6143         size_t ret_len;
6144         uint_t cookie_count;
6145 
6146         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6147 
6148         rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6149 
6150         /* allocate rcvd FIS dma handle. */
6151         if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6152             &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6153             DDI_DMA_SLEEP,
6154             NULL,
6155             &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6156             DDI_SUCCESS) {
6157                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6158                     "rcvd FIS dma handle alloc failed", NULL);
6159 
6160                 return (AHCI_FAILURE);
6161         }
6162 
6163         if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6164             rcvd_fis_size,
6165             &accattr,
6166             DDI_DMA_CONSISTENT,
6167             DDI_DMA_SLEEP,
6168             NULL,
6169             (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6170             &ret_len,
6171             &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6172 
6173                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6174                     "rcvd FIS dma mem alloc fail", NULL);
6175                 /* error.. free the dma handle. */
6176                 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6177                 return (AHCI_FAILURE);
6178         }
6179 
6180         if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6181             NULL,
6182             (caddr_t)ahci_portp->ahciport_rcvd_fis,
6183             rcvd_fis_size,
6184             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6185             DDI_DMA_SLEEP,
6186             NULL,
6187             &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6188             &cookie_count) !=  DDI_DMA_MAPPED) {
6189 
6190                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6191                     "rcvd FIS dma handle bind fail", NULL);
6192                 /*  error.. free the dma handle & free the memory. */
6193                 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6194                 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6195                 return (AHCI_FAILURE);
6196         }
6197 
6198         bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6199 
6200         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6201             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6202         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6203             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6204 
6205         return (AHCI_SUCCESS);
6206 }
6207 
6208 /*
6209  * Deallocates the Received FIS Structure
6210  */
6211 static void
6212 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6213 {
6214         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6215 
6216         /* Unbind the cmd list dma handle first. */
6217         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6218 
6219         /* Then free the underlying memory. */
6220         ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6221 
6222         /* Now free the handle itself. */
6223         ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6224 }
6225 
6226 /*
6227  * Allocates memory for the Command List, which contains up to 32 entries.
6228  * Each entry contains a command header, which is a 32-byte structure that
6229  * includes the pointer to the command table.
6230  */
6231 static int
6232 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6233 {
6234         size_t cmd_list_size;
6235         size_t ret_len;
6236         uint_t cookie_count;
6237 
6238         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6239 
6240         cmd_list_size =
6241             ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6242 
6243         /* allocate cmd list dma handle. */
6244         if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6245             &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6246             DDI_DMA_SLEEP,
6247             NULL,
6248             &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6249 
6250                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6251                     "cmd list dma handle alloc failed", NULL);
6252                 return (AHCI_FAILURE);
6253         }
6254 
6255         if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6256             cmd_list_size,
6257             &accattr,
6258             DDI_DMA_CONSISTENT,
6259             DDI_DMA_SLEEP,
6260             NULL,
6261             (caddr_t *)&ahci_portp->ahciport_cmd_list,
6262             &ret_len,
6263             &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6264 
6265                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6266                     "cmd list dma mem alloc fail", NULL);
6267                 /* error.. free the dma handle. */
6268                 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6269                 return (AHCI_FAILURE);
6270         }
6271 
6272         if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6273             NULL,
6274             (caddr_t)ahci_portp->ahciport_cmd_list,
6275             cmd_list_size,
6276             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6277             DDI_DMA_SLEEP,
6278             NULL,
6279             &ahci_portp->ahciport_cmd_list_dma_cookie,
6280             &cookie_count) !=  DDI_DMA_MAPPED) {
6281 
6282                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6283                     "cmd list dma handle bind fail", NULL);
6284                 /*  error.. free the dma handle & free the memory. */
6285                 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6286                 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6287                 return (AHCI_FAILURE);
6288         }
6289 
6290         bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6291 
6292         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6293             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6294 
6295         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6296             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6297 
6298         if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6299                 goto err_out;
6300         }
6301 
6302         return (AHCI_SUCCESS);
6303 
6304 err_out:
6305         /* Unbind the cmd list dma handle first. */
6306         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6307 
6308         /* Then free the underlying memory. */
6309         ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6310 
6311         /* Now free the handle itself. */
6312         ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6313 
6314         return (AHCI_FAILURE);
6315 }
6316 
6317 /*
6318  * Deallocates the Command List
6319  */
6320 static void
6321 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6322 {
6323         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6324 
6325         /* First dealloc command table */
6326         ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6327 
6328         /* Unbind the cmd list dma handle first. */
6329         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6330 
6331         /* Then free the underlying memory. */
6332         ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6333 
6334         /* Now free the handle itself. */
6335         ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6336 }
6337 
6338 /*
6339  * Allocates memory for all Command Tables, which contains Command FIS,
6340  * ATAPI Command and Physical Region Descriptor Table.
6341  */
6342 static int
6343 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6344 {
6345         size_t ret_len;
6346         ddi_dma_cookie_t cmd_table_dma_cookie;
6347         uint_t cookie_count;
6348         int slot;
6349 
6350         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6351 
6352         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6353             "ahci_alloc_cmd_tables: port %d enter",
6354             ahci_portp->ahciport_port_num);
6355 
6356         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6357                 /* Allocate cmd table dma handle. */
6358                 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6359                     &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6360                     DDI_DMA_SLEEP,
6361                     NULL,
6362                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6363                     DDI_SUCCESS) {
6364 
6365                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6366                             "cmd table dma handle alloc failed", NULL);
6367 
6368                         goto err_out;
6369                 }
6370 
6371                 if (ddi_dma_mem_alloc(
6372                     ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6373                     ahci_cmd_table_size,
6374                     &accattr,
6375                     DDI_DMA_CONSISTENT,
6376                     DDI_DMA_SLEEP,
6377                     NULL,
6378                     (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6379                     &ret_len,
6380                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6381                     NULL) {
6382 
6383                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6384                             "cmd table dma mem alloc fail", NULL);
6385 
6386                         /* error.. free the dma handle. */
6387                         ddi_dma_free_handle(
6388                             &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6389                         goto err_out;
6390                 }
6391 
6392                 if (ddi_dma_addr_bind_handle(
6393                     ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6394                     NULL,
6395                     (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6396                     ahci_cmd_table_size,
6397                     DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6398                     DDI_DMA_SLEEP,
6399                     NULL,
6400                     &cmd_table_dma_cookie,
6401                     &cookie_count) !=  DDI_DMA_MAPPED) {
6402 
6403                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6404                             "cmd table dma handle bind fail", NULL);
6405                         /*  error.. free the dma handle & free the memory. */
6406                         ddi_dma_mem_free(
6407                             &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6408                         ddi_dma_free_handle(
6409                             &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6410                         goto err_out;
6411                 }
6412 
6413                 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6414                     ahci_cmd_table_size);
6415 
6416                 /* Config Port Command Table Base Address */
6417                 SET_COMMAND_TABLE_BASE_ADDR(
6418                     (&ahci_portp->ahciport_cmd_list[slot]),
6419                     cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6420 
6421 #ifndef __lock_lint
6422                 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6423                     (&ahci_portp->ahciport_cmd_list[slot]),
6424                     cmd_table_dma_cookie.dmac_laddress >> 32);
6425 #endif
6426         }
6427 
6428         return (AHCI_SUCCESS);
6429 err_out:
6430 
6431         for (slot--; slot >= 0; slot--) {
6432                 /* Unbind the cmd table dma handle first */
6433                 (void) ddi_dma_unbind_handle(
6434                     ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6435 
6436                 /* Then free the underlying memory */
6437                 ddi_dma_mem_free(
6438                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6439 
6440                 /* Now free the handle itself */
6441                 ddi_dma_free_handle(
6442                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6443         }
6444 
6445         return (AHCI_FAILURE);
6446 }
6447 
6448 /*
6449  * Deallocates memory for all Command Tables.
6450  */
6451 static void
6452 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6453 {
6454         int slot;
6455 
6456         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6457 
6458         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6459             "ahci_dealloc_cmd_tables: %d enter",
6460             ahci_portp->ahciport_port_num);
6461 
6462         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6463                 /* Unbind the cmd table dma handle first. */
6464                 (void) ddi_dma_unbind_handle(
6465                     ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6466 
6467                 /* Then free the underlying memory. */
6468                 ddi_dma_mem_free(
6469                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6470 
6471                 /* Now free the handle itself. */
6472                 ddi_dma_free_handle(
6473                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6474         }
6475 }
6476 
6477 /*
6478  * Update SATA registers at controller ports
6479  */
6480 static void
6481 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6482     sata_device_t *sd)
6483 {
6484         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6485 
6486         sd->satadev_scr.sstatus =
6487             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6488             (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6489         sd->satadev_scr.serror =
6490             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6491             (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6492         sd->satadev_scr.scontrol =
6493             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6494             (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6495         sd->satadev_scr.sactive =
6496             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6497             (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6498 }
6499 
6500 /*
6501  * For poll mode, ahci_port_intr will be called to emulate the interrupt
6502  */
6503 static void
6504 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6505 {
6506         uint32_t port_intr_status;
6507         uint32_t port_intr_enable;
6508 
6509         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6510             "ahci_port_intr enter: port %d", port);
6511 
6512         mutex_enter(&ahci_portp->ahciport_mutex);
6513         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6514                 /* For SATA_OPMODE_POLLING commands */
6515                 port_intr_enable =
6516                     (AHCI_INTR_STATUS_DHRS |
6517                     AHCI_INTR_STATUS_PSS |
6518                     AHCI_INTR_STATUS_SDBS |
6519                     AHCI_INTR_STATUS_UFS |
6520                     AHCI_INTR_STATUS_PCS |
6521                     AHCI_INTR_STATUS_PRCS |
6522                     AHCI_INTR_STATUS_OFS |
6523                     AHCI_INTR_STATUS_INFS |
6524                     AHCI_INTR_STATUS_IFS |
6525                     AHCI_INTR_STATUS_HBDS |
6526                     AHCI_INTR_STATUS_HBFS |
6527                     AHCI_INTR_STATUS_TFES);
6528         } else {
6529                 /*
6530                  * port_intr_enable indicates that the corresponding interrrupt
6531                  * reporting is enabled.
6532                  */
6533                 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6534                     (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6535         }
6536 
6537         /* IPMS error in port reset should be ignored according AHCI spec. */
6538         if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6539                 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6540         mutex_exit(&ahci_portp->ahciport_mutex);
6541 
6542         /*
6543          * port_intr_stats indicates that the corresponding interrupt
6544          * condition is active.
6545          */
6546         port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6547             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6548 
6549         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6550             "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6551             "port_intr_enable = 0x%x",
6552             port, port_intr_status, port_intr_enable);
6553 
6554         port_intr_status &= port_intr_enable;
6555 
6556         /*
6557          * Pending interrupt events are indicated by the PxIS register.
6558          * Make sure we don't miss any event.
6559          */
6560         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6561                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6562                     DDI_SERVICE_UNAFFECTED);
6563                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6564                     DDI_FME_VERSION);
6565                 return;
6566         }
6567 
6568         /* First clear the port interrupts status */
6569         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6570             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6571             port_intr_status);
6572 
6573         /* Check the completed non-queued commands */
6574         if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6575             AHCI_INTR_STATUS_PSS)) {
6576                 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6577                     ahci_portp, port);
6578         }
6579 
6580         /* Check the completed queued commands */
6581         if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6582                 (void) ahci_intr_set_device_bits(ahci_ctlp,
6583                     ahci_portp, port);
6584         }
6585 
6586         /* Check the port connect change status interrupt bit */
6587         if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6588                 (void) ahci_intr_port_connect_change(ahci_ctlp,
6589                     ahci_portp, port);
6590         }
6591 
6592         /* Check the device mechanical presence status interrupt bit */
6593         if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6594                 (void) ahci_intr_device_mechanical_presence_status(
6595                     ahci_ctlp, ahci_portp, port);
6596         }
6597 
6598         /* Check the PhyRdy change status interrupt bit */
6599         if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6600                 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6601                     port);
6602         }
6603 
6604         /*
6605          * Check the non-fatal error interrupt bits, there are four
6606          * kinds of non-fatal errors at the time being:
6607          *
6608          *    PxIS.UFS - Unknown FIS Error
6609          *    PxIS.OFS - Overflow Error
6610          *    PxIS.INFS - Interface Non-Fatal Error
6611          *    PxIS.IPMS - Incorrect Port Multiplier Status Error
6612          *
6613          * For these non-fatal errors, the HBA can continue to operate,
6614          * so the driver just log the error messages.
6615          */
6616         if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6617             AHCI_INTR_STATUS_OFS |
6618             AHCI_INTR_STATUS_IPMS |
6619             AHCI_INTR_STATUS_INFS)) {
6620                 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6621                     port, port_intr_status);
6622         }
6623 
6624         /*
6625          * Check the fatal error interrupt bits, there are four kinds
6626          * of fatal errors for AHCI controllers:
6627          *
6628          *    PxIS.HBFS - Host Bus Fatal Error
6629          *    PxIS.HBDS - Host Bus Data Error
6630          *    PxIS.IFS - Interface Fatal Error
6631          *    PxIS.TFES - Task File Error
6632          *
6633          * The fatal error means the HBA can not recover from it by
6634          * itself, and it will try to abort the transfer, and the software
6635          * must intervene to restart the port.
6636          */
6637         if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6638             AHCI_INTR_STATUS_HBDS |
6639             AHCI_INTR_STATUS_HBFS |
6640             AHCI_INTR_STATUS_TFES))
6641                 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6642                     port, port_intr_status);
6643 
6644         /* Check the cold port detect interrupt bit */
6645         if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6646                 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6647         }
6648 
6649         /* Second clear the corresponding bit in IS.IPS */
6650         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6651             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6652 
6653         /* Try to recover at the end of the interrupt handler. */
6654         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6655             DDI_FM_OK) {
6656                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6657                     DDI_SERVICE_UNAFFECTED);
6658                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6659                     DDI_FME_VERSION);
6660         }
6661 }
6662 
6663 /*
6664  * Interrupt service handler
6665  */
6666 static uint_t
6667 ahci_intr(caddr_t arg1, caddr_t arg2)
6668 {
6669 #ifndef __lock_lint
6670         _NOTE(ARGUNUSED(arg2))
6671 #endif
6672         /* LINTED */
6673         ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6674         ahci_port_t *ahci_portp;
6675         int32_t global_intr_status;
6676         uint8_t port;
6677 
6678         /*
6679          * global_intr_status indicates that the corresponding port has
6680          * an interrupt pending.
6681          */
6682         global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6683             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6684 
6685         if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6686                 /* The interrupt is not ours */
6687                 return (DDI_INTR_UNCLAIMED);
6688         }
6689 
6690         /*
6691          * Check the handle after reading global_intr_status - we don't want
6692          * to miss any port with pending interrupts.
6693          */
6694         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6695             DDI_FM_OK) {
6696                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6697                     DDI_SERVICE_UNAFFECTED);
6698                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6699                     DDI_FME_VERSION);
6700                 return (DDI_INTR_UNCLAIMED);
6701         }
6702 
6703         /* Loop for all the ports */
6704         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6705                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6706                         continue;
6707                 }
6708                 if (!((0x1 << port) & global_intr_status)) {
6709                         continue;
6710                 }
6711 
6712                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6713 
6714                 /* Call ahci_port_intr */
6715                 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6716         }
6717 
6718         return (DDI_INTR_CLAIMED);
6719 }
6720 
6721 /*
6722  * For non-queued commands, when the corresponding bit in the PxCI register
6723  * is cleared, it means the command is completed successfully. And according
6724  * to the HBA state machine, there are three conditions which possibly will
6725  * try to clear the PxCI register bit.
6726  *      1. Receive one D2H Register FIS which is with 'I' bit set
6727  *      2. Update PIO Setup FIS
6728  *      3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6729  *
6730  * Process completed non-queued commands when the interrupt status bit -
6731  * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6732  *
6733  * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6734  * with the 'I' bit set. And the following commands will send thus
6735  * FIS with 'I' bit set upon the successful completion:
6736  *      1. Non-data commands
6737  *      2. DMA data-in command
6738  *      3. DMA data-out command
6739  *      4. PIO data-out command
6740  *      5. PACKET non-data commands
6741  *      6. PACKET PIO data-in command
6742  *      7. PACKET PIO data-out command
6743  *      8. PACKET DMA data-in command
6744  *      9. PACKET DMA data-out command
6745  *
6746  * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6747  * with the 'I' bit set. And the following commands will send this
6748  * FIS upon the successful completion:
6749  *      1. PIO data-in command
6750  */
6751 static int
6752 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6753     ahci_port_t *ahci_portp, uint8_t port)
6754 {
6755         uint32_t port_cmd_issue = 0;
6756         uint32_t finished_tags;
6757         int finished_slot;
6758         sata_pkt_t *satapkt;
6759         ahci_fis_d2h_register_t *rcvd_fisp;
6760 #if AHCI_DEBUG
6761         ahci_cmd_header_t *cmd_header;
6762         uint32_t cmd_dmacount;
6763 #endif
6764 
6765         mutex_enter(&ahci_portp->ahciport_mutex);
6766 
6767         if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6768             !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6769             !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6770                 /*
6771                  * Spurious interrupt. Nothing to be done.
6772                  */
6773                 mutex_exit(&ahci_portp->ahciport_mutex);
6774                 return (AHCI_SUCCESS);
6775         }
6776 
6777         port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6778             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6779 
6780         /* If the PxCI corrupts, don't complete the commmands. */
6781         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6782             != DDI_FM_OK) {
6783                 mutex_exit(&ahci_portp->ahciport_mutex);
6784                 return (AHCI_FAILURE);
6785         }
6786 
6787         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6788                 /* Slot 0 is always used during error recovery */
6789                 finished_tags = 0x1 & ~port_cmd_issue;
6790                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6791                     "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6792                     "retrieval is finished, and finished_tags = 0x%x",
6793                     port, finished_tags);
6794         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6795                 finished_tags = 0x1 & ~port_cmd_issue;
6796                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6797                     "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6798                     "port multiplier is finished, and finished_tags = 0x%x",
6799                     port, finished_tags);
6800 
6801         } else {
6802 
6803                 finished_tags = ahci_portp->ahciport_pending_tags &
6804                     ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6805         }
6806 
6807         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6808             "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6809             "port_cmd_issue = 0x%x finished_tags = 0x%x",
6810             ahci_portp->ahciport_pending_tags, port_cmd_issue,
6811             finished_tags);
6812 
6813         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6814             (finished_tags == 0x1)) {
6815                 satapkt = ahci_portp->ahciport_err_retri_pkt;
6816                 ASSERT(satapkt != NULL);
6817 
6818                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6819                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6820                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6821 
6822                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6823                 goto out;
6824         }
6825 
6826         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6827             (finished_tags == 0x1)) {
6828                 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6829                 ASSERT(satapkt != NULL);
6830 
6831                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6832                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6833                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6834 
6835                 /* READ PORTMULT need copy out FIS content. */
6836                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6837                         rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6838                             ahcirf_d2h_register_fis);
6839                         satapkt->satapkt_cmd.satacmd_status_reg =
6840                             GET_RFIS_STATUS(rcvd_fisp);
6841                         ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6842                 }
6843 
6844                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6845                 goto out;
6846         }
6847 
6848         while (finished_tags) {
6849                 finished_slot = ddi_ffs(finished_tags) - 1;
6850                 if (finished_slot == -1) {
6851                         goto out;
6852                 }
6853 
6854                 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6855                 ASSERT(satapkt != NULL);
6856 #if AHCI_DEBUG
6857                 /*
6858                  * For non-native queued commands, the PRD byte count field
6859                  * shall contain an accurate count of the number of bytes
6860                  * transferred for the command before the PxCI bit is cleared
6861                  * to '0' for the command.
6862                  *
6863                  * The purpose of this field is to let software know how many
6864                  * bytes transferred for a given operation in order to
6865                  * determine if underflow occurred. When issuing native command
6866                  * queuing commands, this field should not be used and is not
6867                  * required to be valid since in this case underflow is always
6868                  * illegal.
6869                  *
6870                  * For data reads, the HBA will update its PRD byte count with
6871                  * the total number of bytes received from the last FIS, and
6872                  * may be able to continue normally. For data writes, the
6873                  * device will detect an error, and HBA most likely will get
6874                  * a fatal error.
6875                  *
6876                  * Therefore, here just put code to debug part. And please
6877                  * refer to the comment above ahci_intr_fatal_error for the
6878                  * definition of underflow error.
6879                  */
6880                 cmd_dmacount =
6881                     ahci_portp->ahciport_prd_bytecounts[finished_slot];
6882                 if (cmd_dmacount) {
6883                         cmd_header =
6884                             &ahci_portp->ahciport_cmd_list[finished_slot];
6885                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6886                             "ahci_intr_cmd_cmplt: port %d, "
6887                             "PRD Byte Count = 0x%x, "
6888                             "ahciport_prd_bytecounts = 0x%x", port,
6889                             cmd_header->ahcich_prd_byte_count,
6890                             cmd_dmacount);
6891 
6892                         if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6893                                 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6894                                     "ahci_intr_cmd_cmplt: port %d, "
6895                                     "an underflow occurred", port);
6896                         }
6897                 }
6898 #endif
6899 
6900                 /*
6901                  * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6902                  * feature, sata_special_regs flag will be set, and the
6903                  * driver should copy the status and the other corresponding
6904                  * register values in the D2H Register FIS received (It's
6905                  * working on Non-data protocol) from the device back to
6906                  * the sata_cmd.
6907                  *
6908                  * For every AHCI port, there is only one Received FIS
6909                  * structure, which contains the FISes received from the
6910                  * device, So we're trying to copy the content of D2H
6911                  * Register FIS in the Received FIS structure back to
6912                  * the sata_cmd.
6913                  */
6914                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6915                         rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6916                             ahcirf_d2h_register_fis);
6917                         satapkt->satapkt_cmd.satacmd_status_reg =
6918                             GET_RFIS_STATUS(rcvd_fisp);
6919                         ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6920                 }
6921 
6922                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6923                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6924                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6925 
6926                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
6927                 CLEAR_BIT(finished_tags, finished_slot);
6928                 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
6929 
6930                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6931         }
6932 out:
6933         AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
6934             "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
6935             ahci_portp->ahciport_pending_tags);
6936 
6937         ahci_flush_doneq(ahci_portp);
6938 
6939         mutex_exit(&ahci_portp->ahciport_mutex);
6940 
6941         return (AHCI_SUCCESS);
6942 }
6943 
6944 /*
6945  * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
6946  * with the 'I' bit set and has been copied into system memory. It will
6947  * be sent under the following situations:
6948  *
6949  * 1. NCQ command is completed
6950  *
6951  * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
6952  * via the Set Device Bits FIS. When such event is generated, the software
6953  * needs to read PxSACT register and compares the current value to the
6954  * list of commands previously issue by software. ahciport_pending_ncq_tags
6955  * keeps the tags of previously issued commands.
6956  *
6957  * 2. Asynchronous Notification
6958  *
6959  * Asynchronous Notification is a feature in SATA spec 2.6.
6960  *
6961  * 1) ATAPI device will send a signal to the host when media is inserted or
6962  * removed and avoids polling the device for media changes. The signal
6963  * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
6964  * set to '1'. At the moment, it's not supported yet.
6965  *
6966  * 2) Port multiplier will send a signal to the host when a hot plug event
6967  * has occured on a port multiplier port. It is used when command based
6968  * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
6969  */
6970 static int
6971 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
6972     ahci_port_t *ahci_portp, uint8_t port)
6973 {
6974         ahci_addr_t addr;
6975 
6976         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
6977             "ahci_intr_set_device_bits enter: port %d", port);
6978 
6979         /* Initialize HBA port address */
6980         AHCI_ADDR_SET_PORT(&addr, port);
6981 
6982         /* NCQ plug handler */
6983         (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
6984 
6985         /* Check port multiplier's asynchronous notification events */
6986         if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
6987                 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
6988                     ahci_portp, port);
6989         }
6990 
6991         /* ATAPI events is not supported yet */
6992 
6993         return (AHCI_SUCCESS);
6994 }
6995 /*
6996  * NCQ interrupt handler. Called upon a NCQ command is completed.
6997  * Only be called from ahci_intr_set_device_bits().
6998  */
6999 static int
7000 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7001     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7002 {
7003         uint32_t port_sactive;
7004         uint32_t port_cmd_issue;
7005         uint32_t issued_tags;
7006         int issued_slot;
7007         uint32_t finished_tags;
7008         int finished_slot;
7009         uint8_t port = addrp->aa_port;
7010         sata_pkt_t *satapkt;
7011 
7012         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7013             "ahci_intr_set_device_bits enter: port %d", port);
7014 
7015         mutex_enter(&ahci_portp->ahciport_mutex);
7016         if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7017                 mutex_exit(&ahci_portp->ahciport_mutex);
7018                 return (AHCI_SUCCESS);
7019         }
7020 
7021         /*
7022          * First the handler got which commands are finished by checking
7023          * PxSACT register
7024          */
7025         port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7026             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7027 
7028         finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7029             ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7030 
7031         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7032             "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7033             "port_sactive = 0x%x", port,
7034             ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7035 
7036         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7037             "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7038 
7039         /*
7040          * For NCQ commands, the software can determine which command has
7041          * already been transmitted to the device by checking PxCI register.
7042          */
7043         port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7044             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7045 
7046         issued_tags = ahci_portp->ahciport_pending_tags &
7047             ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7048 
7049         /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7050         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7051             != DDI_FM_OK) {
7052                 mutex_exit(&ahci_portp->ahciport_mutex);
7053                 return (AHCI_FAILURE);
7054         }
7055 
7056         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7057             "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7058             "port_cmd_issue = 0x%x", port,
7059             ahci_portp->ahciport_pending_tags, port_cmd_issue);
7060 
7061         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7062             "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7063 
7064         /*
7065          * Clear ahciport_pending_tags bit when the corresponding command
7066          * is already sent down to the device.
7067          */
7068         while (issued_tags) {
7069                 issued_slot = ddi_ffs(issued_tags) - 1;
7070                 if (issued_slot == -1) {
7071                         goto next;
7072                 }
7073                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7074                 CLEAR_BIT(issued_tags, issued_slot);
7075         }
7076 
7077 next:
7078         while (finished_tags) {
7079                 finished_slot = ddi_ffs(finished_tags) - 1;
7080                 if (finished_slot == -1) {
7081                         goto out;
7082                 }
7083 
7084                 /* The command is certainly transmitted to the device */
7085                 ASSERT(!(ahci_portp->ahciport_pending_tags &
7086                     (0x1 << finished_slot)));
7087 
7088                 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7089                 ASSERT(satapkt != NULL);
7090 
7091                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7092                     "ahci_intr_set_device_bits: sending up pkt 0x%p "
7093                     "with SATA_PKT_COMPLETED", (void *)satapkt);
7094 
7095                 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7096                 CLEAR_BIT(finished_tags, finished_slot);
7097                 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7098 
7099                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7100         }
7101 out:
7102         AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7103             "ahci_intr_set_device_bits: port %d "
7104             "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7105             port, ahci_portp->ahciport_pending_ncq_tags,
7106             ahci_portp->ahciport_pending_tags);
7107 
7108         ahci_flush_doneq(ahci_portp);
7109 
7110         mutex_exit(&ahci_portp->ahciport_mutex);
7111 
7112         return (AHCI_SUCCESS);
7113 }
7114 
7115 /*
7116  * Port multiplier asynchronous notification event handler. Called upon a
7117  * device is hot plugged/pulled.
7118  *
7119  * The async-notification event will only be recorded by ahcipmi_snotif_tags
7120  * here and will be handled by ahci_probe_pmult().
7121  *
7122  * NOTE: called only from ahci_port_intr().
7123  */
7124 static int
7125 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7126     uint8_t port)
7127 {
7128         sata_device_t sdevice;
7129 
7130         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7131             "ahci_intr_pmult_sntf_events enter: port %d ", port);
7132 
7133         /* no hot-plug while attaching process */
7134         mutex_enter(&ahci_ctlp->ahcictl_mutex);
7135         if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7136                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7137                 return (AHCI_SUCCESS);
7138         }
7139         mutex_exit(&ahci_ctlp->ahcictl_mutex);
7140 
7141         mutex_enter(&ahci_portp->ahciport_mutex);
7142         if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7143                 mutex_exit(&ahci_portp->ahciport_mutex);
7144                 return (AHCI_SUCCESS);
7145         }
7146 
7147         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7148 
7149         ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7150             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7151             (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7152         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7153             (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7154             AHCI_SNOTIF_CLEAR_ALL);
7155 
7156         if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7157                 mutex_exit(&ahci_portp->ahciport_mutex);
7158                 return (AHCI_SUCCESS);
7159         }
7160 
7161         /* Port Multiplier sub-device hot-plug handler */
7162         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7163                 mutex_exit(&ahci_portp->ahciport_mutex);
7164                 return (AHCI_SUCCESS);
7165         }
7166 
7167         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7168                 /* Not allowed to re-enter. */
7169                 mutex_exit(&ahci_portp->ahciport_mutex);
7170                 return (AHCI_SUCCESS);
7171         }
7172 
7173         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7174 
7175         /*
7176          * NOTE:
7177          * Even if Asynchronous Notification is supported (and enabled) by
7178          * both controller and the port multiplier, the content of PxSNTF
7179          * register is always set to 0x8000 by async notification event. We
7180          * need to check GSCR[32] on the port multiplier to find out the
7181          * owner of this event.
7182          * This is not accord with SATA spec 2.6 and needs further
7183          * clarification.
7184          */
7185         /* hot-plug will not reported while reseting. */
7186         if (ahci_portp->ahciport_reset_in_progress == 1) {
7187                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7188                     "port %d snotif event ignored", port);
7189                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7190                 mutex_exit(&ahci_portp->ahciport_mutex);
7191                 return (AHCI_SUCCESS);
7192         }
7193 
7194         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7195             "PxSNTF is set to 0x%x by port multiplier",
7196             ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7197 
7198         /*
7199          * Now we need do some necessary operation and inform SATA framework
7200          * that link/device events has happened.
7201          */
7202         bzero((void *)&sdevice, sizeof (sata_device_t));
7203         sdevice.satadev_addr.cport = ahci_ctlp->
7204             ahcictl_port_to_cport[port];
7205         sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7206         sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7207         sdevice.satadev_state = SATA_PSTATE_PWRON;
7208 
7209         /* Just reject packets, do not stop that port. */
7210         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7211 
7212         mutex_exit(&ahci_portp->ahciport_mutex);
7213         sata_hba_event_notify(
7214             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7215             &sdevice,
7216             SATA_EVNT_PMULT_LINK_CHANGED);
7217         mutex_enter(&ahci_portp->ahciport_mutex);
7218 
7219         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7220         mutex_exit(&ahci_portp->ahciport_mutex);
7221 
7222         return (AHCI_SUCCESS);
7223 }
7224 
7225 /*
7226  * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7227  * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7228  * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7229  * indicates a COMINIT signal was received.
7230  *
7231  * Hot plug insertion is detected by reception of a COMINIT signal from the
7232  * device. On reception of unsolicited COMINIT, the HBA shall generate a
7233  * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7234  * begin the normal communication negotiation sequence as outlined in the
7235  * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7236  * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7237  * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7238  * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7239  * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7240  * to handle hot plug insertion. In this interrupt handler, just do nothing
7241  * but print some log message and clear the bit.
7242  */
7243 static int
7244 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7245     ahci_port_t *ahci_portp, uint8_t port)
7246 {
7247 #if AHCI_DEBUG
7248         uint32_t port_serror;
7249 #endif
7250 
7251         mutex_enter(&ahci_portp->ahciport_mutex);
7252 
7253 #if AHCI_DEBUG
7254         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7255             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7256 
7257         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7258             "ahci_intr_port_connect_change: port %d, "
7259             "port_serror = 0x%x", port, port_serror);
7260 #endif
7261 
7262         /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7263         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7264             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7265             SERROR_EXCHANGED_ERR);
7266 
7267         mutex_exit(&ahci_portp->ahciport_mutex);
7268 
7269         return (AHCI_SUCCESS);
7270 }
7271 
7272 /*
7273  * Hot Plug Operation for platforms that support Mechanical Presence
7274  * Switches.
7275  *
7276  * When set, it indicates that a mechanical presence switch attached to this
7277  * port has been opened or closed, which may lead to a change in the connection
7278  * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7279  * are set to '1'.
7280  *
7281  * At the moment, this interrupt is not needed and disabled and we just log
7282  * the debug message.
7283  */
7284 static int
7285 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7286     ahci_port_t *ahci_portp, uint8_t port)
7287 {
7288         uint32_t cap_status, port_cmd_status;
7289 
7290         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7291             "ahci_intr_device_mechanical_presence_status enter, "
7292             "port %d", port);
7293 
7294         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7295             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7296 
7297         mutex_enter(&ahci_portp->ahciport_mutex);
7298         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7299             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7300 
7301         if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7302             !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7303                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7304                     "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7305                     "the interrupt: cap_status = 0x%x, "
7306                     "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7307                 mutex_exit(&ahci_portp->ahciport_mutex);
7308 
7309                 return (AHCI_SUCCESS);
7310         }
7311 
7312 #if AHCI_DEBUG
7313         if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7314                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7315                     "The mechanical presence switch is open: "
7316                     "port %d, port_cmd_status = 0x%x",
7317                     port, port_cmd_status);
7318         } else {
7319                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7320                     "The mechanical presence switch is close: "
7321                     "port %d, port_cmd_status = 0x%x",
7322                     port, port_cmd_status);
7323         }
7324 #endif
7325 
7326         mutex_exit(&ahci_portp->ahciport_mutex);
7327 
7328         return (AHCI_SUCCESS);
7329 }
7330 
7331 /*
7332  * Native Hot Plug Support.
7333  *
7334  * When set, it indicates that the internal PHYRDY signal changed state.
7335  * This bit reflects the state of PxSERR.DIAG.N.
7336  *
7337  * There are three kinds of conditions to generate this interrupt event:
7338  * 1. a device is inserted
7339  * 2. a device is disconnected
7340  * 3. when the link enters/exits a Partial or Slumber interface power
7341  *    management state
7342  *
7343  * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7344  * bit may be set due to the link entering the Partial or Slumber power
7345  * management state, rather than due to a hot plug insertion or removal
7346  * event. So far, the interface power management is disabled, so the
7347  * driver can reliably get removal detection notification via the
7348  * PxSERR.DIAG.N bit.
7349  */
7350 static int
7351 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7352     ahci_port_t *ahci_portp, uint8_t port)
7353 {
7354         uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7355         sata_device_t sdevice;
7356         int dev_exists_now = 0;
7357         int dev_existed_previously = 0;
7358         ahci_addr_t port_addr;
7359 
7360         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7361             "ahci_intr_phyrdy_change enter, port %d", port);
7362 
7363         /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7364         mutex_enter(&ahci_portp->ahciport_mutex);
7365         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7366             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7367             SERROR_PHY_RDY_CHG);
7368         mutex_exit(&ahci_portp->ahciport_mutex);
7369 
7370         mutex_enter(&ahci_ctlp->ahcictl_mutex);
7371         if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7372             (ahci_portp == NULL)) {
7373                 /* The whole controller setup is not yet done. */
7374                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7375                 return (AHCI_SUCCESS);
7376         }
7377         mutex_exit(&ahci_ctlp->ahcictl_mutex);
7378 
7379         mutex_enter(&ahci_portp->ahciport_mutex);
7380 
7381         /* SStatus tells the presence of device. */
7382         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7383             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7384 
7385         if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7386                 dev_exists_now = 1;
7387         }
7388 
7389         if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7390                 dev_existed_previously = 1;
7391         }
7392 
7393         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7394                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7395                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7396                     "ahci_intr_phyrdy_change: port %d "
7397                     "AHCI_PORT_FLAG_NODEV is cleared", port);
7398                 if (dev_exists_now == 0)
7399                         dev_existed_previously = 1;
7400         }
7401 
7402         bzero((void *)&sdevice, sizeof (sata_device_t));
7403         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7404         sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7405         sdevice.satadev_addr.pmport = 0;
7406         sdevice.satadev_state = SATA_PSTATE_PWRON;
7407         ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7408 
7409         AHCI_ADDR_SET_PORT(&port_addr, port);
7410 
7411         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7412         if (dev_exists_now) {
7413                 if (dev_existed_previously) { /* 1 -> 1 */
7414                         /* Things are fine now. The loss was temporary. */
7415                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7416                             "ahci_intr_phyrdy_change  port %d "
7417                             "device link lost/established", port);
7418 
7419                         mutex_exit(&ahci_portp->ahciport_mutex);
7420                         sata_hba_event_notify(
7421                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7422                             &sdevice,
7423                             SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7424                         mutex_enter(&ahci_portp->ahciport_mutex);
7425 
7426                 } else { /* 0 -> 1 */
7427                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7428                             "ahci_intr_phyrdy_change: port %d "
7429                             "device link established", port);
7430 
7431                         /*
7432                          * A new device has been detected. The new device
7433                          * might be a port multiplier instead of a drive, so
7434                          * we cannot update the signature directly.
7435                          */
7436                         (void) ahci_initialize_port(ahci_ctlp,
7437                             ahci_portp, &port_addr);
7438 
7439                         /* Try to start the port */
7440                         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7441                             != AHCI_SUCCESS) {
7442                                 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7443                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7444                                     "ahci_intr_phyrdy_change: port %d failed "
7445                                     "at start port", port);
7446                         }
7447 
7448                         /* Clear the max queue depth for inserted device */
7449                         ahci_portp->ahciport_max_ncq_tags = 0;
7450 
7451                         mutex_exit(&ahci_portp->ahciport_mutex);
7452                         sata_hba_event_notify(
7453                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7454                             &sdevice,
7455                             SATA_EVNT_LINK_ESTABLISHED);
7456                         mutex_enter(&ahci_portp->ahciport_mutex);
7457 
7458                 }
7459         } else { /* No device exists now */
7460 
7461                 if (dev_existed_previously) { /* 1 -> 0 */
7462                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7463                             "ahci_intr_phyrdy_change: port %d "
7464                             "device link lost", port);
7465 
7466                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7467                         (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7468                             ahci_portp, port);
7469 
7470                         if (ahci_portp->ahciport_device_type ==
7471                             SATA_DTYPE_PMULT) {
7472                                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7473                         }
7474 
7475                         /* An existing device is lost. */
7476                         ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7477                         ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7478 
7479                         mutex_exit(&ahci_portp->ahciport_mutex);
7480                         sata_hba_event_notify(
7481                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7482                             &sdevice,
7483                             SATA_EVNT_LINK_LOST);
7484                         mutex_enter(&ahci_portp->ahciport_mutex);
7485                 }
7486         }
7487         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7488 
7489         mutex_exit(&ahci_portp->ahciport_mutex);
7490 
7491         return (AHCI_SUCCESS);
7492 }
7493 
7494 /*
7495  * PxIS.UFS - Unknown FIS Error
7496  *
7497  * This interrupt event means an unknown FIS was received and has been
7498  * copied into system memory. An unknown FIS is not considered an illegal
7499  * FIS, unless the length received is more than 64 bytes. If an unknown
7500  * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7501  * normal operation. If the unknown FIS is more than 64 bytes, then it
7502  * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7503  * a fatal error.
7504  *
7505  * PxIS.IPMS - Incorrect Port Multiplier Status
7506  *
7507  * IPMS Indicates that the HBA received a FIS from a device that did not
7508  * have a command outstanding. The IPMS bit may be set during enumeration
7509  * of devices on a Port Multiplier due to the normal Port Multiplier
7510  * enumeration process. It is recommended that IPMS only be used after
7511  * enumeration is complete on the Port Multiplier (copied from spec).
7512  *
7513  * PxIS.OFS - Overflow Error
7514  *
7515  * Command list overflow is defined as software building a command table
7516  * that has fewer total bytes than the transaction given to the device.
7517  * On device writes, the HBA will run out of data, and on reads, there
7518  * will be no room to put the data.
7519  *
7520  * For an overflow on data read, either PIO or DMA, the HBA will set
7521  * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7522  * non-fatal error when the HBA can continues. Sometimes, it will cause
7523  * a fatal error and need the software to do something.
7524  *
7525  * For an overflow on data write, setting PxIS.OFS is optional for both
7526  * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7527  * software to clean up from this serious error.
7528  *
7529  * PxIS.INFS - Interface Non-Fatal Error
7530  *
7531  * This interrupt event indicates that the HBA encountered an error on
7532  * the Serial ATA interface but was able to continue operation. The kind
7533  * of error usually occurred during a non-Data FIS, and under this condition
7534  * the FIS will be re-transmitted by HBA automatically.
7535  *
7536  * When the FMA is implemented, there should be a stat structure to
7537  * record how many every kind of error happens.
7538  */
7539 static int
7540 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7541     uint8_t port, uint32_t intr_status)
7542 {
7543         uint32_t port_serror;
7544 #if AHCI_DEBUG
7545         uint32_t port_cmd_status;
7546         uint32_t port_cmd_issue;
7547         uint32_t port_sactive;
7548         int current_slot;
7549         uint32_t current_tags;
7550         sata_pkt_t *satapkt;
7551         ahci_cmd_header_t *cmd_header;
7552         uint32_t cmd_dmacount;
7553 #endif
7554 
7555         mutex_enter(&ahci_portp->ahciport_mutex);
7556 
7557         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7558             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7559 
7560         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7561             "ahci_intr_non_fatal_error: port %d, "
7562             "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7563 
7564         ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7565 
7566         if (intr_status & AHCI_INTR_STATUS_UFS) {
7567                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7568                     "ahci port %d has unknown FIS error", port);
7569 
7570                 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7571                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7572                     (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7573                     SERROR_FIS_TYPE);
7574         }
7575 
7576 #if AHCI_DEBUG
7577         if (intr_status & AHCI_INTR_STATUS_IPMS) {
7578                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7579                     "has Incorrect Port Multiplier Status error", port);
7580         }
7581 
7582         if (intr_status & AHCI_INTR_STATUS_OFS) {
7583                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7584                     "ahci port %d has overflow error", port);
7585         }
7586 
7587         if (intr_status & AHCI_INTR_STATUS_INFS) {
7588                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7589                     "ahci port %d has interface non fatal error", port);
7590         }
7591 
7592         /*
7593          * Record the error occurred command's slot.
7594          */
7595         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7596             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7597                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7598                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7599 
7600                 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7601                     AHCI_CMD_STATUS_CCS_SHIFT;
7602 
7603                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7604                         satapkt = ahci_portp->ahciport_err_retri_pkt;
7605                         ASSERT(satapkt != NULL);
7606                         ASSERT(current_slot == 0);
7607                 } else {
7608                         satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7609                 }
7610 
7611                 if (satapkt != NULL) {
7612                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7613                             "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7614                             "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7615                             satapkt->satapkt_cmd.satacmd_cmd_reg);
7616 
7617                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7618                             "ahci_intr_non_fatal_error: port %d, "
7619                             "satapkt 0x%p is being processed when error occurs",
7620                             port, (void *)satapkt);
7621 
7622                         /*
7623                          * PRD Byte Count field of command header is not
7624                          * required to reflect the total number of bytes
7625                          * transferred when an overflow occurs, so here
7626                          * just log the value.
7627                          */
7628                         cmd_dmacount =
7629                             ahci_portp->ahciport_prd_bytecounts[current_slot];
7630                         if (cmd_dmacount) {
7631                                 cmd_header = &ahci_portp->
7632                                     ahciport_cmd_list[current_slot];
7633                                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7634                                     "ahci_intr_non_fatal_error: port %d, "
7635                                     "PRD Byte Count = 0x%x, "
7636                                     "ahciport_prd_bytecounts = 0x%x", port,
7637                                     cmd_header->ahcich_prd_byte_count,
7638                                     cmd_dmacount);
7639                         }
7640                 }
7641         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7642                 /*
7643                  * For queued command, list those command which have already
7644                  * been transmitted to the device and still not completed.
7645                  */
7646                 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7647                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7648 
7649                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7650                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7651 
7652                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7653                     "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7654                     "port_sactive = 0x%x port_cmd_issue = 0x%x",
7655                     ahci_portp->ahciport_pending_ncq_tags,
7656                     port_sactive, port_cmd_issue);
7657 
7658                 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7659                     port_sactive & ~port_cmd_issue &
7660                     AHCI_NCQ_SLOT_MASK(ahci_portp);
7661 
7662                 while (current_tags) {
7663                         current_slot = ddi_ffs(current_tags) - 1;
7664                         if (current_slot == -1) {
7665                                 goto out;
7666                         }
7667 
7668                         satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7669                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7670                             ahci_ctlp, "ahci_intr_non_fatal_error: "
7671                             "port %d, satapkt 0x%p is outstanding when "
7672                             "error occurs", port, (void *)satapkt);
7673 
7674                         CLEAR_BIT(current_tags, current_slot);
7675                 }
7676         }
7677 out:
7678 #endif
7679         mutex_exit(&ahci_portp->ahciport_mutex);
7680 
7681         return (AHCI_SUCCESS);
7682 }
7683 
7684 /*
7685  * According to the AHCI spec, the error types include system memory
7686  * errors, interface errors, port multiplier errors, device errors,
7687  * command list overflow, command list underflow, native command
7688  * queuing tag errors and pio data transfer errors.
7689  *
7690  * System memory errors such as target abort, master abort, and parity
7691  * may cause the host to stop, and they are serious errors and needed
7692  * to be recovered with software intervention. When system software
7693  * has given a pointer to the HBA that doesn't exist in physical memory,
7694  * a master/target abort error occurs, and PxIS.HBFS will be set. A
7695  * data error such as CRC or parity occurs, the HBA aborts the transfer
7696  * (if necessary) and PxIS.HBDS will be set.
7697  *
7698  * Interface errors are errors that occur due to electrical issues on
7699  * the interface, or protocol miscommunication between the device and
7700  * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7701  * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7702  * causes PxIS.IFS/PxIS.INFS to be set are
7703  *      1. in PxSERR.ERR, P bit is set to '1'
7704  *      2. in PxSERR.DIAG, C or H bit is set to '1'
7705  *      3. PhyRdy drop unexpectly, N bit is set to '1'
7706  * If the error occurred during a non-data FIS, the FIS must be
7707  * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7708  * the error occurred during a data FIS, the transfer will stop, so
7709  * the error is fatal and PxIS.IFS is set.
7710  *
7711  * When a FIS arrives that updates the taskfile, the HBA checks to see
7712  * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7713  * stops processing any more commands.
7714  *
7715  * Command list overflow is defined as software building a command table
7716  * that has fewer total bytes than the transaction given to the device.
7717  * On device writes, the HBA will run out of data, and on reads, there
7718  * will be no room to put the data. For an overflow on data read, either
7719  * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7720  * For an overflow on data write, setting PxIS.OFS is optional for both
7721  * DMA and PIO, and a COMRESET is required by software to clean up from
7722  * this serious error.
7723  *
7724  * Command list underflow is defined as software building a command
7725  * table that has more total bytes than the transaction given to the
7726  * device. For data writes, both PIO and DMA, the device will detect
7727  * an error and end the transfer. And these errors are most likely going
7728  * to be fatal errors that will cause the port to be restarted. For
7729  * data reads, the HBA updates its PRD byte count, and may be
7730  * able to continue normally, but is not required to. And The HBA is
7731  * not required to detect underflow conditions for native command
7732  * queuing command.
7733  *
7734  * The HBA does not actively check incoming DMA Setup FISes to ensure
7735  * that the PxSACT register bit for that slot is set. Existing error
7736  * mechanisms, such as host bus failure, or bad protocol, are used to
7737  * recover from this case.
7738  *
7739  * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7740  * DATA FIS must be an integral number of Dwords. If the HBA receives
7741  * a request which is not an integral number of Dwords, the HBA
7742  * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7743  * software restarts the port. And the HBA ensures that the size
7744  * of the DATA FIS received during a PIO command matches the size in
7745  * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7746  * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7747  * stop running until software restarts the port.
7748  */
7749 /*
7750  * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7751  *
7752  * PxIS.IFS indicates that the hba encountered an error on the serial ata
7753  * interface which caused the transfer to stop.
7754  *
7755  * PxIS.HBDS indicates that the hba encountered a data error
7756  * (uncorrectable ecc/parity) when reading from or writing to system memory.
7757  *
7758  * PxIS.HBFS indicates that the hba encountered a host bus error that it
7759  * cannot recover from, such as a bad software pointer.
7760  *
7761  * PxIS.TFES is set whenever the status register is updated by the device
7762  * and the error bit (bit 0) is set.
7763  */
7764 static int
7765 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7766     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7767 {
7768         uint32_t port_cmd_status;
7769         uint32_t port_serror;
7770         uint32_t task_file_status;
7771         int failed_slot;
7772         sata_pkt_t *spkt = NULL;
7773         uint8_t err_byte;
7774         ahci_event_arg_t *args;
7775         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7776         uint32_t failed_tags = 0;
7777         int task_fail_flag = 0, task_abort_flag = 0;
7778         uint32_t slot_status;
7779 
7780         mutex_enter(&ahci_portp->ahciport_mutex);
7781 
7782         /*
7783          * ahci_intr_phyrdy_change() may have rendered it to
7784          * SATA_DTYPE_NONE.
7785          */
7786         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7787                 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7788                     "ahci_intr_fatal_error: port %d no device attached, "
7789                     "and just return without doing anything", port);
7790                 goto out0;
7791         }
7792 
7793         if (intr_status & AHCI_INTR_STATUS_TFES) {
7794                 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7795                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7796                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7797                     "ahci_intr_fatal_error: port %d "
7798                     "task_file_status = 0x%x", port, task_file_status);
7799                 task_fail_flag = 1;
7800 
7801                 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7802                     >> AHCI_TFD_ERR_SHIFT;
7803                 if (err_byte == SATA_ERROR_ABORT)
7804                         task_abort_flag = 1;
7805         }
7806 
7807         /*
7808          * Here we just log the fatal error info in interrupt context.
7809          * Misc recovery processing will be handled in task queue.
7810          */
7811         if (task_fail_flag  == 1) {
7812                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7813                         /*
7814                          * Read PxCMD.CCS to determine the slot that the HBA
7815                          * was processing when the error occurred.
7816                          */
7817                         port_cmd_status = ddi_get32(
7818                             ahci_ctlp->ahcictl_ahci_acc_handle,
7819                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7820                         failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7821                             AHCI_CMD_STATUS_CCS_SHIFT;
7822                         failed_tags = 0x1 << failed_slot;
7823 
7824                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7825                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7826                             "ahci_intr_fatal_error: spkt 0x%p is being "
7827                             "processed when fatal error occurred for port %d",
7828                             spkt, port);
7829 
7830                         /*
7831                          * Won't emit the error message if it is an IDENTIFY
7832                          * DEVICE command sent to an ATAPI device.
7833                          */
7834                         if ((spkt != NULL) &&
7835                             (spkt->satapkt_cmd.satacmd_cmd_reg ==
7836                             SATAC_ID_DEVICE) &&
7837                             (task_abort_flag == 1))
7838                         goto out1;
7839 
7840                         /*
7841                          * Won't emit the error message if it is an ATAPI PACKET
7842                          * command
7843                          */
7844                         if ((spkt != NULL) &&
7845                             (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7846                                 goto out1;
7847 
7848                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7849                         slot_status = ddi_get32(
7850                             ahci_ctlp->ahcictl_ahci_acc_handle,
7851                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7852                         failed_tags = slot_status &
7853                             AHCI_NCQ_SLOT_MASK(ahci_portp);
7854                 }
7855         }
7856 
7857         /* print the fatal error type */
7858         ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7859         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7860 
7861         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7862             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7863 
7864         /* print PxSERR related error message */
7865         ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7866 
7867         /* print task file register value */
7868         if (task_fail_flag == 1) {
7869                 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7870                     "= 0x%x", instance, port, task_file_status);
7871                 if (task_abort_flag == 1) {
7872                         cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7873                             "port %d are aborted", instance, port);
7874                         ahci_dump_commands(ahci_ctlp, port, failed_tags);
7875                 }
7876         }
7877 
7878 out1:
7879         /* Prepare the argument for the taskq */
7880         args = ahci_portp->ahciport_event_args;
7881         args->ahciea_ctlp = (void *)ahci_ctlp;
7882         args->ahciea_portp = (void *)ahci_portp;
7883         args->ahciea_event = intr_status;
7884         AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7885 
7886         /* Start the taskq to handle error recovery */
7887         if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7888             ahci_events_handler,
7889             (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7890                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7891                 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7892                     "port %d failed", instance, port);
7893         }
7894 out0:
7895         mutex_exit(&ahci_portp->ahciport_mutex);
7896 
7897         return (AHCI_SUCCESS);
7898 }
7899 
7900 /*
7901  * Hot Plug Operation for platforms that support Cold Presence Detect.
7902  *
7903  * When set, a device status has changed as detected by the cold presence
7904  * detect logic. This bit can either be set due to a non-connected port
7905  * receiving a device, or a connected port having its device removed.
7906  * This bit is only valid if the port supports cold presence detect as
7907  * indicated by PxCMD.CPD set to '1'.
7908  *
7909  * At the moment, this interrupt is not needed and disabled and we just
7910  * log the debug message.
7911  */
7912 static int
7913 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
7914     ahci_port_t *ahci_portp, uint8_t port)
7915 {
7916         uint32_t port_cmd_status;
7917         sata_device_t sdevice;
7918 
7919         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7920             "ahci_intr_cold_port_detect enter, port %d", port);
7921 
7922         mutex_enter(&ahci_portp->ahciport_mutex);
7923 
7924         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7925             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7926         if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
7927                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7928                     "port %d does not support cold presence detect, so "
7929                     "we just ignore this interrupt", port);
7930                 mutex_exit(&ahci_portp->ahciport_mutex);
7931                 return (AHCI_SUCCESS);
7932         }
7933 
7934         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7935             "port %d device status has changed", port);
7936 
7937         bzero((void *)&sdevice, sizeof (sata_device_t));
7938         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7939         sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7940         sdevice.satadev_addr.pmport = 0;
7941         sdevice.satadev_state = SATA_PSTATE_PWRON;
7942 
7943         if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
7944                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7945                     "port %d: a device is hot plugged", port);
7946                 mutex_exit(&ahci_portp->ahciport_mutex);
7947                 sata_hba_event_notify(
7948                     ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7949                     &sdevice,
7950                     SATA_EVNT_DEVICE_ATTACHED);
7951                 mutex_enter(&ahci_portp->ahciport_mutex);
7952 
7953         } else {
7954                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7955                     "port %d: a device is hot unplugged", port);
7956                 mutex_exit(&ahci_portp->ahciport_mutex);
7957                 sata_hba_event_notify(
7958                     ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7959                     &sdevice,
7960                     SATA_EVNT_DEVICE_DETACHED);
7961                 mutex_enter(&ahci_portp->ahciport_mutex);
7962         }
7963 
7964         mutex_exit(&ahci_portp->ahciport_mutex);
7965 
7966         return (AHCI_SUCCESS);
7967 }
7968 
7969 /*
7970  * Enable the interrupts for a particular port.
7971  */
7972 static void
7973 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
7974 {
7975         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
7976 
7977         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
7978             "ahci_enable_port_intrs enter, port %d", port);
7979 
7980         /*
7981          * Clear port interrupt status before enabling interrupt
7982          */
7983         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7984             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
7985             AHCI_PORT_INTR_MASK);
7986 
7987         /*
7988          * Clear the pending bit from IS.IPS
7989          */
7990         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7991             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
7992 
7993         /*
7994          * Enable the following interrupts:
7995          *      Device to Host Register FIS Interrupt (DHRS)
7996          *      PIO Setup FIS Interrupt (PSS)
7997          *      Set Device Bits Interrupt (SDBS)
7998          *      Unknown FIS Interrupt (UFS)
7999          *      Port Connect Change Status (PCS)
8000          *      PhyRdy Change Status (PRCS)
8001          *      Overflow Status (OFS)
8002          *      Interface Non-fatal Error Status (INFS)
8003          *      Interface Fatal Error Status (IFS)
8004          *      Host Bus Data Error Status (HBDS)
8005          *      Host Bus Fatal Error Status (HBFS)
8006          *      Task File Error Status (TFES)
8007          */
8008         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8009             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8010             (AHCI_INTR_STATUS_DHRS |
8011             AHCI_INTR_STATUS_PSS |
8012             AHCI_INTR_STATUS_SDBS |
8013             AHCI_INTR_STATUS_UFS |
8014             AHCI_INTR_STATUS_DPS |
8015             AHCI_INTR_STATUS_PCS |
8016             AHCI_INTR_STATUS_PRCS |
8017             AHCI_INTR_STATUS_OFS |
8018             AHCI_INTR_STATUS_INFS |
8019             AHCI_INTR_STATUS_IFS |
8020             AHCI_INTR_STATUS_HBDS |
8021             AHCI_INTR_STATUS_HBFS |
8022             AHCI_INTR_STATUS_TFES));
8023 }
8024 
8025 /*
8026  * Enable interrupts for all the ports.
8027  */
8028 static void
8029 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8030 {
8031         uint32_t ghc_control;
8032 
8033         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8034 
8035         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8036 
8037         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8038             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8039 
8040         ghc_control |= AHCI_HBA_GHC_IE;
8041 
8042         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8043             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8044 }
8045 
8046 /*
8047  * Disable interrupts for a particular port.
8048  */
8049 static void
8050 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8051 {
8052         ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8053             MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8054 
8055         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8056             "ahci_disable_port_intrs enter, port %d", port);
8057 
8058         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8059             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8060 }
8061 
8062 /*
8063  * Disable interrupts for the whole HBA.
8064  *
8065  * The global bit is cleared, then all interrupt sources from all
8066  * ports are disabled.
8067  */
8068 static void
8069 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8070 {
8071         uint32_t ghc_control;
8072 
8073         ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8074             MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8075 
8076         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8077             NULL);
8078 
8079         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8080             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8081 
8082         ghc_control &= ~AHCI_HBA_GHC_IE;
8083 
8084         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8085             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8086 }
8087 
8088 /*
8089  * Handle FIXED or MSI interrupts.
8090  */
8091 /*
8092  * According to AHCI spec, the HBA may support several interrupt modes:
8093  *      * pin based interrupts (FIXED)
8094  *      * single MSI message interrupts
8095  *      * multiple MSI based message interrupts
8096  *
8097  * For pin based interrupts, the software interrupt handler need to check IS
8098  * register to find out which port has pending interrupts. And then check
8099  * PxIS register to find out which interrupt events happened on that port.
8100  *
8101  * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8102  * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8103  * in that software interrupt handler need to check IS register to determine
8104  * which port triggered the interrupts since it uses a single message for all
8105  * port interrupts.
8106  *
8107  * HBA may optionally support multiple MSI message for better performance. In
8108  * this mode, each port may have its own interrupt message, and thus generation
8109  * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8110  * represents a power-of-2 wrapper on the number of implemented ports, and
8111  * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8112  * maximum number of assigned interrupts. When the number of MSI messages
8113  * allocated is less than the number requested, then hardware may have two
8114  * implementation behaviors:
8115  *      * assign each ports its own interrupt and then force all additional
8116  *        ports to share the last interrupt message, and this condition is
8117  *        indicated by clearing GHC.MRSM to '0'
8118  *      * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8119  * When multiple-message MSI is enabled, hardware will still set IS register
8120  * as single message case. And this IS register may be used by software when
8121  * fewer than the requested number of messages is granted in order to determine
8122  * which port had the interrupt.
8123  *
8124  * Note: The current ahci driver only supports the first two interrupt modes:
8125  * pin based interrupts and single MSI message interrupts, and the reason
8126  * is indicated in below code.
8127  */
8128 static int
8129 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8130 {
8131         dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8132         int             count, avail, actual;
8133         int             i, rc;
8134 
8135         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8136             "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8137 
8138         /* get number of interrupts. */
8139         rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8140         if ((rc != DDI_SUCCESS) || (count == 0)) {
8141                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8142                     "ddi_intr_get_nintrs() failed, "
8143                     "rc %d count %d\n", rc, count);
8144                 return (DDI_FAILURE);
8145         }
8146 
8147         /* get number of available interrupts. */
8148         rc = ddi_intr_get_navail(dip, intr_type, &avail);
8149         if ((rc != DDI_SUCCESS) || (avail == 0)) {
8150                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8151                     "ddi_intr_get_navail() failed, "
8152                     "rc %d avail %d\n", rc, avail);
8153                 return (DDI_FAILURE);
8154         }
8155 
8156 #if AHCI_DEBUG
8157         if (avail < count) {
8158                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8159                     "ddi_intr_get_nintrs returned %d, navail() returned %d",
8160                     count, avail);
8161         }
8162 #endif
8163 
8164         /*
8165          * Note: So far Solaris restricts the maximum number of messages for
8166          * x86 to 2, that is avail is 2, so here we set the count with 1 to
8167          * force the driver to use single MSI message interrupt. In future if
8168          * Solaris remove the restriction, then we need to delete the below
8169          * code and try to use multiple interrupt routine to gain better
8170          * performance.
8171          */
8172         if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8173                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8174                     "force to use one interrupt routine though the "
8175                     "HBA supports %d interrupt", count);
8176                 count = 1;
8177         }
8178 
8179         /* Allocate an array of interrupt handles. */
8180         ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8181         ahci_ctlp->ahcictl_intr_htable =
8182             kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8183 
8184         /* call ddi_intr_alloc(). */
8185         rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8186             intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8187 
8188         if ((rc != DDI_SUCCESS) || (actual == 0)) {
8189                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8190                     "ddi_intr_alloc() failed, rc %d count %d actual %d "
8191                     "avail %d\n", rc, count, actual, avail);
8192                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8193                     ahci_ctlp->ahcictl_intr_size);
8194                 return (DDI_FAILURE);
8195         }
8196 
8197         /* use interrupt count returned */
8198 #if AHCI_DEBUG
8199         if (actual < count) {
8200                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8201                     "Requested: %d, Received: %d", count, actual);
8202         }
8203 #endif
8204 
8205         ahci_ctlp->ahcictl_intr_cnt = actual;
8206 
8207         /*
8208          * Get priority for first, assume remaining are all the same.
8209          */
8210         if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8211             &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8212                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8213                     "ddi_intr_get_pri() failed", NULL);
8214 
8215                 /* Free already allocated intr. */
8216                 for (i = 0; i < actual; i++) {
8217                         (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8218                 }
8219 
8220                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8221                     ahci_ctlp->ahcictl_intr_size);
8222                 return (DDI_FAILURE);
8223         }
8224 
8225         /* Test for high level interrupt. */
8226         if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8227                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8228                     "ahci_add_intrs: Hi level intr not supported", NULL);
8229 
8230                 /* Free already allocated intr. */
8231                 for (i = 0; i < actual; i++) {
8232                         (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8233                 }
8234 
8235                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8236                     sizeof (ddi_intr_handle_t));
8237 
8238                 return (DDI_FAILURE);
8239         }
8240 
8241         /* Call ddi_intr_add_handler(). */
8242         for (i = 0; i < actual; i++) {
8243                 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8244                     ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8245                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8246                             "ddi_intr_add_handler() failed", NULL);
8247 
8248                         /* Free already allocated intr. */
8249                         for (i = 0; i < actual; i++) {
8250                                 (void) ddi_intr_free(
8251                                     ahci_ctlp->ahcictl_intr_htable[i]);
8252                         }
8253 
8254                         kmem_free(ahci_ctlp->ahcictl_intr_htable,
8255                             ahci_ctlp->ahcictl_intr_size);
8256                         return (DDI_FAILURE);
8257                 }
8258         }
8259 
8260         if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8261             &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8262                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8263                     "ddi_intr_get_cap() failed", NULL);
8264 
8265                 /* Free already allocated intr. */
8266                 for (i = 0; i < actual; i++) {
8267                         (void) ddi_intr_free(
8268                             ahci_ctlp->ahcictl_intr_htable[i]);
8269                 }
8270 
8271                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8272                     ahci_ctlp->ahcictl_intr_size);
8273                 return (DDI_FAILURE);
8274         }
8275 
8276         if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8277                 /* Call ddi_intr_block_enable() for MSI. */
8278                 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8279                     ahci_ctlp->ahcictl_intr_cnt);
8280         } else {
8281                 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8282                 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8283                         (void) ddi_intr_enable(
8284                             ahci_ctlp->ahcictl_intr_htable[i]);
8285                 }
8286         }
8287 
8288         return (DDI_SUCCESS);
8289 }
8290 
8291 /*
8292  * Removes the registered interrupts irrespective of whether they
8293  * were legacy or MSI.
8294  *
8295  * NOTE: The controller interrupts must be disabled before calling
8296  * this routine.
8297  */
8298 static void
8299 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8300 {
8301         int x;
8302 
8303         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8304 
8305         /* Disable all interrupts. */
8306         if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8307             (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8308                 /* Call ddi_intr_block_disable(). */
8309                 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8310                     ahci_ctlp->ahcictl_intr_cnt);
8311         } else {
8312                 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8313                         (void) ddi_intr_disable(
8314                             ahci_ctlp->ahcictl_intr_htable[x]);
8315                 }
8316         }
8317 
8318         /* Call ddi_intr_remove_handler(). */
8319         for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8320                 (void) ddi_intr_remove_handler(
8321                     ahci_ctlp->ahcictl_intr_htable[x]);
8322                 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8323         }
8324 
8325         kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8326 }
8327 
8328 /*
8329  * This routine tries to put port into P:NotRunning state by clearing
8330  * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8331  * and PxCMD.CR to '0'.
8332  */
8333 static int
8334 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8335     ahci_port_t *ahci_portp, uint8_t port)
8336 {
8337         uint32_t port_cmd_status;
8338         int loop_count;
8339 
8340         ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8341             MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8342 
8343         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8344             "ahci_put_port_into_notrunning_state enter: port %d", port);
8345 
8346         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8347             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8348 
8349         port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8350         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8351             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8352 
8353         /* Wait until PxCMD.CR is cleared */
8354         loop_count = 0;
8355         do {
8356                 port_cmd_status =
8357                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8358                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8359 
8360                 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8361                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8362                             "clearing port %d CMD.CR timeout, "
8363                             "port_cmd_status = 0x%x", port,
8364                             port_cmd_status);
8365                         /*
8366                          * We are effectively timing out after 0.5 sec.
8367                          * This value is specified in AHCI spec.
8368                          */
8369                         break;
8370                 }
8371 
8372                 /* Wait for 10 millisec */
8373                 drv_usecwait(AHCI_10MS_USECS);
8374         } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8375 
8376         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8377 
8378         if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8379                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8380                     "ahci_put_port_into_notrunning_state: failed to clear "
8381                     "PxCMD.CR to '0' after loop count: %d, and "
8382                     "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8383                 return (AHCI_FAILURE);
8384         } else {
8385                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8386                     "ahci_put_port_into_notrunning_state: succeeded to clear "
8387                     "PxCMD.CR to '0' after loop count: %d, and "
8388                     "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8389                 return (AHCI_SUCCESS);
8390         }
8391 }
8392 
8393 /*
8394  * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8395  * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8396  * stable state, then set PxCMD.ST to '1' to start the port directly.
8397  * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8398  * COMRESET to the device to put it in an idle state.
8399  *
8400  * The fifth argument returns whether the port reset is involved during
8401  * the process.
8402  *
8403  * The routine will be called under following scenarios:
8404  *      + To reset the HBA
8405  *      + To abort the packet(s)
8406  *      + To reset the port
8407  *      + To activate the port
8408  *      + Fatal error recovery
8409  *      + To abort the timeout packet(s)
8410  *
8411  * NOTES!!! During this procedure, PxSERR register will be cleared, and
8412  * according to the spec, the clearance of three bits will also clear
8413  * three interrupt status bits.
8414  *      1. PxSERR.DIAG.F will clear PxIS.UFS
8415  *      2. PxSERR.DIAG.X will clear PxIS.PCS
8416  *      3. PxSERR.DIAG.N will clear PxIS.PRCS
8417  *
8418  * Among these three interrupt events, the driver needs to take care of
8419  * PxIS.PRCS, which is the hot plug event. When the driver found out
8420  * a device was unplugged, it will call the interrupt handler.
8421  */
8422 static int
8423 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8424     ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8425 {
8426         uint32_t port_sstatus;
8427         uint32_t task_file_status;
8428         sata_device_t sdevice;
8429         int rval;
8430         ahci_addr_t addr_port;
8431         ahci_pmult_info_t *pminfo = NULL;
8432         int dev_exists_begin = 0;
8433         int dev_exists_end = 0;
8434         uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8435         int npmport = 0;
8436         uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8437 
8438         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8439 
8440         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8441             "ahci_restart_port_wait_till_ready: port %d enter", port);
8442 
8443         AHCI_ADDR_SET_PORT(&addr_port, port);
8444 
8445         if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8446                 dev_exists_begin = 1;
8447 
8448         /* First clear PxCMD.ST */
8449         rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8450             port);
8451         if (rval != AHCI_SUCCESS)
8452                 /*
8453                  * If PxCMD.CR does not clear within a reasonable time, it
8454                  * may assume the interface is in a hung condition and may
8455                  * continue with issuing the port reset.
8456                  */
8457                 goto reset;
8458 
8459         /* Then clear PxSERR */
8460         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8461             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8462             AHCI_SERROR_CLEAR_ALL);
8463 
8464         /* Then get PxTFD */
8465         task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8466             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8467 
8468         /*
8469          * Check whether the device is in a stable status, if yes,
8470          * then start the port directly. However for ahci_tran_reset_dport,
8471          * we may have to perform a port reset.
8472          */
8473         if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8474             !(flag & AHCI_PORT_RESET))
8475                 goto out;
8476 
8477 reset:
8478         /*
8479          * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8480          * a COMRESET to the device
8481          */
8482         ahci_disable_port_intrs(ahci_ctlp, port);
8483         rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8484         ahci_enable_port_intrs(ahci_ctlp, port);
8485 
8486 #ifdef AHCI_DEBUG
8487         if (rval != AHCI_SUCCESS)
8488                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8489                     "ahci_restart_port_wait_till_ready: port %d failed",
8490                     port);
8491 #endif
8492 
8493         if (reset_flag != NULL)
8494                 *reset_flag = 1;
8495 
8496         /* Indicate to the framework that a reset has happened. */
8497         if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8498             (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8499             !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8500                 /* Set the reset in progress flag */
8501                 ahci_portp->ahciport_reset_in_progress = 1;
8502 
8503                 bzero((void *)&sdevice, sizeof (sata_device_t));
8504                 sdevice.satadev_addr.cport =
8505                     ahci_ctlp->ahcictl_port_to_cport[port];
8506                 sdevice.satadev_addr.pmport = 0;
8507                 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8508 
8509                 sdevice.satadev_state = SATA_DSTATE_RESET |
8510                     SATA_DSTATE_PWR_ACTIVE;
8511                 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8512                         mutex_exit(&ahci_portp->ahciport_mutex);
8513                         sata_hba_event_notify(
8514                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8515                             &sdevice,
8516                             SATA_EVNT_DEVICE_RESET);
8517                         mutex_enter(&ahci_portp->ahciport_mutex);
8518                 }
8519 
8520                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8521                     "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8522         } else {
8523                 ahci_portp->ahciport_reset_in_progress = 0;
8524         }
8525 
8526 out:
8527         (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8528 
8529         /* SStatus tells the presence of device. */
8530         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8531             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8532 
8533         if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8534                 dev_exists_end = 1;
8535         }
8536 
8537         if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8538                 return (rval);
8539 
8540         /* Check whether a hot plug event happened */
8541         if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8542                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8543                     "ahci_restart_port_wait_till_ready: port %d "
8544                     "device is removed", port);
8545                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8546                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8547                     "ahci_restart_port_wait_till_ready: port %d "
8548                     "AHCI_PORT_FLAG_NODEV flag is set", port);
8549                 mutex_exit(&ahci_portp->ahciport_mutex);
8550                 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8551                 mutex_enter(&ahci_portp->ahciport_mutex);
8552 
8553                 return (rval);
8554         }
8555 
8556 
8557         /* 0/1 -> 1 : device may change */
8558         /*
8559          * May be called by ahci_fatal_error_recovery_handler, so
8560          * don't issue software if the previous device is ATAPI.
8561          */
8562         if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8563                 return (rval);
8564 
8565         /*
8566          * The COMRESET will make port multiplier enter legacy mode.
8567          * Issue a software reset to make it work again.
8568          */
8569         ahci_disable_port_intrs(ahci_ctlp, port);
8570         ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8571         ahci_enable_port_intrs(ahci_ctlp, port);
8572 
8573         /*
8574          * Following codes are specific for the port multiplier
8575          */
8576         if (previous_dev_type != SATA_DTYPE_PMULT &&
8577             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8578                 /* in case previous_dev_type is corrupt */
8579                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8580                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8581                 return (rval);
8582         }
8583 
8584         /* Device change: PMult -> Non-PMult */
8585         if (previous_dev_type == SATA_DTYPE_PMULT &&
8586             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8587                 /*
8588                  * This might happen because
8589                  * 1. Software reset failed. Port multiplier is not correctly
8590                  *    enumerated.
8591                  * 2. Another non-port-multiplier device is attached. Perhaps
8592                  *    the port multiplier was replaced by another device by
8593                  *    whatever reason, but AHCI driver missed hot-plug event.
8594                  *
8595                  * Now that the port has been initialized, we just need to
8596                  * update the port structure according new device, then report
8597                  * and wait SATA framework to probe new device.
8598                  */
8599 
8600                 /* Force to release pmult resource */
8601                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8602                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8603 
8604                 bzero((void *)&sdevice, sizeof (sata_device_t));
8605                 sdevice.satadev_addr.cport =
8606                     ahci_ctlp->ahcictl_port_to_cport[port];
8607                 sdevice.satadev_addr.pmport = 0;
8608                 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8609 
8610                 sdevice.satadev_state = SATA_DSTATE_RESET |
8611                     SATA_DSTATE_PWR_ACTIVE;
8612 
8613                 mutex_exit(&ahci_portp->ahciport_mutex);
8614                 sata_hba_event_notify(
8615                     ahci_ctlp->ahcictl_dip,
8616                     &sdevice,
8617                     SATA_EVNT_DEVICE_RESET);
8618                 mutex_enter(&ahci_portp->ahciport_mutex);
8619 
8620                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8621                     "Port multiplier is [Gone] at port %d ", port);
8622                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8623                     "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8624 
8625                 return (AHCI_SUCCESS);
8626         }
8627 
8628         /* Device change: Non-PMult -> PMult */
8629         if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8630 
8631                 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8632                 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8633 
8634                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8635         }
8636         pminfo = ahci_portp->ahciport_pmult_info;
8637         ASSERT(pminfo != NULL);
8638 
8639         /* Device (may) change: PMult -> PMult */
8640         /*
8641          * First initialize port multiplier. Set state to READY and wait for
8642          * probe entry point to initialize it
8643          */
8644         ahci_portp->ahciport_port_state = SATA_STATE_READY;
8645 
8646         /*
8647          * It's a little complicated while target is a port multiplier. we
8648          * need to COMRESET all pmports behind that PMult otherwise those
8649          * sub-links between the PMult and the sub-devices will be in an
8650          * inactive state (indicated by PSCR0/PxSSTS) and the following access
8651          * to those sub-devices will be rejected by Link-Fatal-Error.
8652          */
8653         /*
8654          * The PxSNTF will be set soon after the pmult is plugged. While the
8655          * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8656          * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8657          * will initialized it.
8658          */
8659         for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8660                 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8661 
8662         /* Report reset event. */
8663         ahci_portp->ahciport_reset_in_progress = 1;
8664 
8665         bzero((void *)&sdevice, sizeof (sata_device_t));
8666         sdevice.satadev_addr.cport = cport;
8667         sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8668         sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8669         sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8670         sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8671             SATA_EVNT_DEVICE_RESET);
8672 
8673         return (rval);
8674 }
8675 
8676 /*
8677  * This routine may be called under four scenarios:
8678  *      a) do the recovery from fatal error
8679  *      b) or we need to timeout some commands
8680  *      c) or we need to abort some commands
8681  *      d) or we need reset device/port/controller
8682  *
8683  * In all these scenarios, we need to send any pending unfinished
8684  * commands up to sata framework.
8685  */
8686 static void
8687 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8688     ahci_port_t *ahci_portp,
8689     uint32_t slot_status,
8690     uint32_t failed_tags,
8691     uint32_t timeout_tags,
8692     uint32_t aborted_tags,
8693     uint32_t reset_tags)
8694 {
8695         uint32_t finished_tags = 0;
8696         uint32_t unfinished_tags = 0;
8697         int tmp_slot;
8698         sata_pkt_t *satapkt;
8699         int ncq_cmd_in_progress = 0;
8700         int err_retri_cmd_in_progress = 0;
8701         int rdwr_pmult_cmd_in_progress = 0;
8702 
8703         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8704 
8705         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8706             "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8707             ahci_portp->ahciport_port_num, slot_status);
8708 
8709         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8710             "ahci_mop_commands: failed_tags: 0x%x, "
8711             "timeout_tags: 0x%x aborted_tags: 0x%x, "
8712             "reset_tags: 0x%x", failed_tags,
8713             timeout_tags, aborted_tags, reset_tags);
8714 
8715 #ifdef AHCI_DEBUG
8716         if (ahci_debug_flags & AHCIDBG_ERRS) {
8717                 int i;
8718                 char msg_buf[200] = {0, };
8719                 for (i = 0x1f; i >= 0; i--) {
8720                         if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8721                                 msg_buf[i] = 'X';
8722                         else
8723                                 msg_buf[i] = '.';
8724                 }
8725                 msg_buf[0x20] = '\0';
8726                 cmn_err(CE_NOTE, "port[%d] slots: %s",
8727                     ahci_portp->ahciport_port_num, msg_buf);
8728                 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8729                     (void *)ahci_portp->ahciport_err_retri_pkt,
8730                     (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8731         }
8732 #endif
8733 
8734         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8735                 finished_tags = ahci_portp->ahciport_pending_tags &
8736                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8737 
8738                 unfinished_tags = slot_status &
8739                     AHCI_SLOT_MASK(ahci_ctlp) &
8740                     ~failed_tags &
8741                     ~aborted_tags &
8742                     ~reset_tags &
8743                     ~timeout_tags;
8744         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8745                 ncq_cmd_in_progress = 1;
8746                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8747                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8748 
8749                 unfinished_tags = slot_status &
8750                     AHCI_NCQ_SLOT_MASK(ahci_portp) &
8751                     ~failed_tags &
8752                     ~aborted_tags &
8753                     ~reset_tags &
8754                     ~timeout_tags;
8755         } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8756 
8757                 /*
8758                  * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8759                  * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8760                  * complete successfully due to one of the following three
8761                  * conditions:
8762                  *
8763                  *      1. Fatal error - failed_tags includes its slot
8764                  *      2. Timed out - timeout_tags includes its slot
8765                  *      3. Aborted when hot unplug - aborted_tags includes its
8766                  *         slot
8767                  *
8768                  * Please note that the command is always sent down in Slot 0
8769                  */
8770                 err_retri_cmd_in_progress = 1;
8771                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8772                     "ahci_mop_commands is called for port %d while "
8773                     "REQUEST SENSE or READ LOG EXT for error retrieval "
8774                     "is being executed slot_status = 0x%x",
8775                     ahci_portp->ahciport_port_num, slot_status);
8776                 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8777                 ASSERT(slot_status == 0x1);
8778         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8779                 rdwr_pmult_cmd_in_progress = 1;
8780                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8781                     "ahci_mop_commands is called for port %d while "
8782                     "READ/WRITE PORTMULT command is being executed",
8783                     ahci_portp->ahciport_port_num);
8784 
8785                 ASSERT(slot_status == 0x1);
8786         }
8787 
8788 #ifdef AHCI_DEBUG
8789         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8790             "ahci_mop_commands: finished_tags: 0x%x, "
8791             "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8792 #endif
8793 
8794         /* Send up finished packets with SATA_PKT_COMPLETED */
8795         while (finished_tags) {
8796                 tmp_slot = ddi_ffs(finished_tags) - 1;
8797                 if (tmp_slot == -1) {
8798                         break;
8799                 }
8800 
8801                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8802                 ASSERT(satapkt != NULL);
8803 
8804                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8805                     "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8806                     (void *)satapkt);
8807 
8808                 /*
8809                  * Cannot fetch the return register content since the port
8810                  * was restarted, so the corresponding tag will be set to
8811                  * aborted tags.
8812                  */
8813                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8814                         CLEAR_BIT(finished_tags, tmp_slot);
8815                         aborted_tags |= tmp_slot;
8816                         continue;
8817                 }
8818 
8819                 if (ncq_cmd_in_progress)
8820                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8821                             tmp_slot);
8822                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8823                 CLEAR_BIT(finished_tags, tmp_slot);
8824                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8825 
8826                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8827         }
8828 
8829         /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8830         while (failed_tags) {
8831                 if (err_retri_cmd_in_progress) {
8832                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8833                         ASSERT(satapkt != NULL);
8834                         ASSERT(failed_tags == 0x1);
8835 
8836                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8837                             "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8838                             (void *)satapkt);
8839                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8840                         break;
8841                 }
8842                 if (rdwr_pmult_cmd_in_progress) {
8843                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8844                         ASSERT(satapkt != NULL);
8845                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8846                             "ahci_mop_commands: sending up "
8847                             "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8848                             (void *)satapkt);
8849                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8850                         break;
8851                 }
8852 
8853                 tmp_slot = ddi_ffs(failed_tags) - 1;
8854                 if (tmp_slot == -1) {
8855                         break;
8856                 }
8857 
8858                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8859                 ASSERT(satapkt != NULL);
8860 
8861                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8862                     "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8863                     (void *)satapkt);
8864 
8865                 if (ncq_cmd_in_progress)
8866                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8867                             tmp_slot);
8868                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8869                 CLEAR_BIT(failed_tags, tmp_slot);
8870                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8871 
8872                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8873         }
8874 
8875         /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8876         while (timeout_tags) {
8877                 if (err_retri_cmd_in_progress) {
8878                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8879                         ASSERT(satapkt != NULL);
8880                         ASSERT(timeout_tags == 0x1);
8881 
8882                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8883                             "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8884                             (void *)satapkt);
8885                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8886                         break;
8887                 }
8888                 if (rdwr_pmult_cmd_in_progress) {
8889                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8890                         ASSERT(satapkt != NULL);
8891                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8892                             "ahci_mop_commands: sending up "
8893                             "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8894                             (void *)satapkt);
8895                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8896                         break;
8897                 }
8898 
8899                 tmp_slot = ddi_ffs(timeout_tags) - 1;
8900                 if (tmp_slot == -1) {
8901                         break;
8902                 }
8903 
8904                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8905                 ASSERT(satapkt != NULL);
8906 
8907                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8908                     "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8909                     (void *)satapkt);
8910 
8911                 if (ncq_cmd_in_progress)
8912                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8913                             tmp_slot);
8914                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8915                 CLEAR_BIT(timeout_tags, tmp_slot);
8916                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8917 
8918                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8919         }
8920 
8921         /* Send up aborted packets with SATA_PKT_ABORTED */
8922         while (aborted_tags) {
8923                 if (err_retri_cmd_in_progress) {
8924                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8925                         ASSERT(satapkt != NULL);
8926                         ASSERT(aborted_tags == 0x1);
8927 
8928                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8929                             "sending up pkt 0x%p with SATA_PKT_ABORTED",
8930                             (void *)satapkt);
8931                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8932                         break;
8933                 }
8934                 if (rdwr_pmult_cmd_in_progress) {
8935                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8936                         ASSERT(satapkt != NULL);
8937                         ASSERT(aborted_tags == 0x1);
8938                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8939                             "ahci_mop_commands: sending up "
8940                             "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
8941                             (void *)satapkt);
8942                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8943                         break;
8944                 }
8945 
8946                 tmp_slot = ddi_ffs(aborted_tags) - 1;
8947                 if (tmp_slot == -1) {
8948                         break;
8949                 }
8950 
8951                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8952                 ASSERT(satapkt != NULL);
8953 
8954                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8955                     "sending up pkt 0x%p with SATA_PKT_ABORTED",
8956                     (void *)satapkt);
8957 
8958                 if (ncq_cmd_in_progress)
8959                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8960                             tmp_slot);
8961                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8962                 CLEAR_BIT(aborted_tags, tmp_slot);
8963                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8964 
8965                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8966         }
8967 
8968         /* Send up reset packets with SATA_PKT_RESET. */
8969         while (reset_tags) {
8970                 if (rdwr_pmult_cmd_in_progress) {
8971                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8972                         ASSERT(satapkt != NULL);
8973                         ASSERT(aborted_tags == 0x1);
8974                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8975                             "ahci_mop_commands: sending up "
8976                             "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
8977                             (void *)satapkt);
8978                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
8979                         break;
8980                 }
8981 
8982                 tmp_slot = ddi_ffs(reset_tags) - 1;
8983                 if (tmp_slot == -1) {
8984                         break;
8985                 }
8986 
8987                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8988                 ASSERT(satapkt != NULL);
8989 
8990                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8991                     "sending up pkt 0x%p with SATA_PKT_RESET",
8992                     (void *)satapkt);
8993 
8994                 if (ncq_cmd_in_progress)
8995                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8996                             tmp_slot);
8997                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8998                 CLEAR_BIT(reset_tags, tmp_slot);
8999                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9000 
9001                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9002         }
9003 
9004         /* Send up unfinished packets with SATA_PKT_RESET */
9005         while (unfinished_tags) {
9006                 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9007                 if (tmp_slot == -1) {
9008                         break;
9009                 }
9010 
9011                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9012                 ASSERT(satapkt != NULL);
9013 
9014                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9015                     "sending up pkt 0x%p with SATA_PKT_RESET",
9016                     (void *)satapkt);
9017 
9018                 if (ncq_cmd_in_progress)
9019                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9020                             tmp_slot);
9021                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9022                 CLEAR_BIT(unfinished_tags, tmp_slot);
9023                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9024 
9025                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9026         }
9027 
9028         ahci_portp->ahciport_mop_in_progress--;
9029         ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9030 
9031         if (ahci_portp->ahciport_mop_in_progress == 0)
9032                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9033 
9034         ahci_flush_doneq(ahci_portp);
9035 }
9036 
9037 /*
9038  * This routine is going to first request a READ LOG EXT sata pkt from sata
9039  * module, and then deliver it to the HBA to get the ncq failure context.
9040  * The return value is the exactly failed tags.
9041  */
9042 static uint32_t
9043 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9044     uint8_t port)
9045 {
9046         sata_device_t   sdevice;
9047         sata_pkt_t      *rdlog_spkt, *spkt;
9048         ddi_dma_handle_t buf_dma_handle;
9049         ahci_addr_t     addr;
9050         int             loop_count;
9051         int             rval;
9052         int             failed_slot;
9053         uint32_t        failed_tags = 0;
9054         struct sata_ncq_error_recovery_page *ncq_err_page;
9055 
9056         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9057             "ahci_get_rdlogext_data enter: port %d", port);
9058 
9059         /* Prepare the sdevice data */
9060         bzero((void *)&sdevice, sizeof (sata_device_t));
9061         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9062 
9063         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9064         sdevice.satadev_addr.pmport = 0;
9065 
9066         /* Translate sata_device.satadev_addr -> ahci_addr */
9067         ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9068 
9069         /*
9070          * Call the sata hba interface to get a rdlog spkt
9071          */
9072         loop_count = 0;
9073 loop:
9074         rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9075             &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9076         if (rdlog_spkt == NULL) {
9077                 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9078                         /* Sleep for a while */
9079                         drv_usecwait(AHCI_10MS_USECS);
9080                         goto loop;
9081                 }
9082                 /* Timed out after 1s */
9083                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9084                     "failed to get rdlog spkt for port %d", port);
9085                 return (failed_tags);
9086         }
9087 
9088         ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9089 
9090         /*
9091          * This flag is used to handle the specific error recovery when the
9092          * READ LOG EXT command gets a failure (fatal error or time-out).
9093          */
9094         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9095 
9096         /*
9097          * This start is not supposed to fail because after port is restarted,
9098          * the whole command list is empty.
9099          */
9100         ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9101         (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9102         ahci_portp->ahciport_err_retri_pkt = NULL;
9103 
9104         /* Remove the flag after READ LOG EXT command is completed */
9105         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9106 
9107         if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9108                 /* Update the request log data */
9109                 buf_dma_handle = *(ddi_dma_handle_t *)
9110                     (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9111                 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9112                     DDI_DMA_SYNC_FORKERNEL);
9113                 if (rval == DDI_SUCCESS) {
9114                         ncq_err_page =
9115                             (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9116                             satapkt_cmd.satacmd_bp->b_un.b_addr;
9117 
9118                         /* Get the failed tag */
9119                         failed_slot = ncq_err_page->ncq_tag;
9120                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9121                             "ahci_get_rdlogext_data: port %d "
9122                             "failed slot %d", port, failed_slot);
9123                         if (failed_slot & NQ) {
9124                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9125                                     "the failed slot is not a valid tag", NULL);
9126                                 goto out;
9127                         }
9128 
9129                         failed_slot &= NCQ_TAG_MASK;
9130                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9131                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9132                             "ahci_get_rdlogext_data: failed spkt 0x%p",
9133                             (void *)spkt);
9134                         if (spkt == NULL) {
9135                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9136                                     "the failed slot spkt is NULL", NULL);
9137                                 goto out;
9138                         }
9139 
9140                         failed_tags = 0x1 << failed_slot;
9141 
9142                         /* Fill out the error context */
9143                         ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9144                             ncq_err_page);
9145                         ahci_update_sata_registers(ahci_ctlp, port,
9146                             &spkt->satapkt_device);
9147                 }
9148         }
9149 out:
9150         sata_free_error_retrieval_pkt(rdlog_spkt);
9151 
9152         return (failed_tags);
9153 }
9154 
9155 /*
9156  * This routine is going to first request a REQUEST SENSE sata pkt from sata
9157  * module, and then deliver it to the HBA to get the sense data and copy
9158  * the sense data back to the orignal failed sata pkt, and free the REQUEST
9159  * SENSE sata pkt later.
9160  */
9161 static void
9162 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9163     uint8_t port, sata_pkt_t *spkt)
9164 {
9165         sata_device_t   sdevice;
9166         sata_pkt_t      *rs_spkt;
9167         sata_cmd_t      *sata_cmd;
9168         ddi_dma_handle_t buf_dma_handle;
9169         ahci_addr_t     addr;
9170         int             loop_count;
9171 #if AHCI_DEBUG
9172         struct scsi_extended_sense *rqsense;
9173 #endif
9174 
9175         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9176             "ahci_get_rqsense_data enter: port %d", port);
9177 
9178         /* Prepare the sdevice data */
9179         bzero((void *)&sdevice, sizeof (sata_device_t));
9180         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9181 
9182         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9183         sdevice.satadev_addr.pmport = 0;
9184 
9185         /* Translate sata_device.satadev_addr -> ahci_addr */
9186         ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9187 
9188         sata_cmd = &spkt->satapkt_cmd;
9189 
9190         /*
9191          * Call the sata hba interface to get a rs spkt
9192          */
9193         loop_count = 0;
9194 loop:
9195         rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9196             &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9197         if (rs_spkt == NULL) {
9198                 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9199                         /* Sleep for a while */
9200                         drv_usecwait(AHCI_10MS_USECS);
9201                         goto loop;
9202 
9203                 }
9204                 /* Timed out after 1s */
9205                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9206                     "failed to get rs spkt for port %d", port);
9207                 return;
9208         }
9209 
9210         ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9211 
9212         /*
9213          * This flag is used to handle the specific error recovery when the
9214          * REQUEST SENSE command gets a faiure (fatal error or time-out).
9215          */
9216         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9217 
9218         /*
9219          * This start is not supposed to fail because after port is restarted,
9220          * the whole command list is empty.
9221          */
9222         ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9223         (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9224         ahci_portp->ahciport_err_retri_pkt = NULL;
9225 
9226         /* Remove the flag after REQUEST SENSE command is completed */
9227         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9228 
9229         if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9230                 /* Update the request sense data */
9231                 buf_dma_handle = *(ddi_dma_handle_t *)
9232                     (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9233                 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9234                     DDI_DMA_SYNC_FORKERNEL);
9235                 /* Copy the request sense data */
9236                 bcopy(rs_spkt->
9237                     satapkt_cmd.satacmd_bp->b_un.b_addr,
9238                     &sata_cmd->satacmd_rqsense,
9239                     SATA_ATAPI_MIN_RQSENSE_LEN);
9240 #if AHCI_DEBUG
9241                 rqsense = (struct scsi_extended_sense *)
9242                     sata_cmd->satacmd_rqsense;
9243 
9244                 /* Dump the sense data */
9245                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9246                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9247                     "Sense data for satapkt %p ATAPI cmd 0x%x",
9248                     spkt, sata_cmd->satacmd_acdb[0]);
9249                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9250                     "  es_code 0x%x es_class 0x%x "
9251                     "es_key 0x%x es_add_code 0x%x "
9252                     "es_qual_code 0x%x",
9253                     rqsense->es_code, rqsense->es_class,
9254                     rqsense->es_key, rqsense->es_add_code,
9255                     rqsense->es_qual_code);
9256 #endif
9257         }
9258 
9259         sata_free_error_retrieval_pkt(rs_spkt);
9260 }
9261 
9262 /*
9263  * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9264  * the port must be restarted. When the HBA detects thus error, it may try
9265  * to abort a transfer. And if the transfer was aborted, the device is
9266  * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9267  * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9268  * that the device is in a stable status and transfers may be restarted without
9269  * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9270  * then the software will send the COMRESET to do the port reset.
9271  *
9272  * Software should perform the appropriate error recovery actions based on
9273  * whether non-queued commands were being issued or natived command queuing
9274  * commands were being issued.
9275  *
9276  * And software will complete the command that had the error with error mark
9277  * to higher level software.
9278  *
9279  * Fatal errors include the following:
9280  *      PxIS.IFS - Interface Fatal Error Status
9281  *      PxIS.HBDS - Host Bus Data Error Status
9282  *      PxIS.HBFS - Host Bus Fatal Error Status
9283  *      PxIS.TFES - Task File Error Status
9284  */
9285 static void
9286 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9287     ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9288 {
9289         uint32_t        port_cmd_status;
9290         uint32_t        slot_status = 0;
9291         uint32_t        failed_tags = 0;
9292         int             failed_slot;
9293         int             reset_flag = 0, flag = 0;
9294         ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9295         sata_cmd_t      *sata_cmd = NULL;
9296         sata_pkt_t      *spkt = NULL;
9297 #if AHCI_DEBUG
9298         ahci_cmd_header_t *cmd_header;
9299 #endif
9300         uint8_t         port = addrp->aa_port;
9301         int             instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9302         int             rval;
9303 
9304         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9305 
9306         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9307             "ahci_fatal_error_recovery_handler enter: port %d", port);
9308 
9309         /* Port multiplier error */
9310         if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9311                 /* FBS code is neither completed nor tested. */
9312                 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9313                     port, intr_status);
9314 
9315                 /* Force a port reset */
9316                 flag = AHCI_PORT_RESET;
9317         }
9318 
9319         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9320             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9321 
9322                 /* Read PxCI to see which commands are still outstanding */
9323                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9324                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9325 
9326                 /*
9327                  * Read PxCMD.CCS to determine the slot that the HBA
9328                  * was processing when the error occurred.
9329                  */
9330                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9331                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9332                 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9333                     AHCI_CMD_STATUS_CCS_SHIFT;
9334 
9335                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9336                         spkt = ahci_portp->ahciport_err_retri_pkt;
9337                         ASSERT(spkt != NULL);
9338                 } else {
9339                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9340                         if (spkt == NULL) {
9341                                 /* May happen when interface errors occur? */
9342                                 goto next;
9343                         }
9344                 }
9345 
9346 #if AHCI_DEBUG
9347                 /*
9348                  * Debugging purpose...
9349                  */
9350                 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9351                         cmd_header =
9352                             &ahci_portp->ahciport_cmd_list[failed_slot];
9353                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9354                             "ahci_fatal_error_recovery_handler: port %d, "
9355                             "PRD Byte Count = 0x%x, "
9356                             "ahciport_prd_bytecounts = 0x%x", port,
9357                             cmd_header->ahcich_prd_byte_count,
9358                             ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9359                 }
9360 #endif
9361 
9362                 sata_cmd = &spkt->satapkt_cmd;
9363 
9364                 /* Fill out the status and error registers for PxIS.TFES */
9365                 if (intr_status & AHCI_INTR_STATUS_TFES) {
9366                         ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9367                             ahcirf_d2h_register_fis);
9368 
9369                         /* Copy the error context back to the sata_cmd */
9370                         ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9371                 }
9372 
9373                 /* The failed command must be one of the outstanding commands */
9374                 failed_tags = 0x1 << failed_slot;
9375                 ASSERT(failed_tags & slot_status);
9376 
9377                 /* Update the sata registers, especially PxSERR register */
9378                 ahci_update_sata_registers(ahci_ctlp, port,
9379                     &spkt->satapkt_device);
9380 
9381         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9382                 /* Read PxSACT to see which commands are still outstanding */
9383                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9384                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9385         }
9386 next:
9387 
9388 #if AHCI_DEBUG
9389         /*
9390          * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9391          * set, it means a fatal error happened after REQUEST SENSE command
9392          * or READ LOG EXT command is delivered to the HBA during the error
9393          * recovery process. At this time, the only outstanding command is
9394          * supposed to be REQUEST SENSE command or READ LOG EXT command.
9395          */
9396         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9397                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9398                     "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9399                     "command or READ LOG EXT command for error data retrieval "
9400                     "failed", port);
9401                 ASSERT(slot_status == 0x1);
9402                 ASSERT(failed_slot == 0);
9403                 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9404                     SCMD_REQUEST_SENSE ||
9405                     spkt->satapkt_cmd.satacmd_cmd_reg ==
9406                     SATAC_READ_LOG_EXT);
9407         }
9408 #endif
9409 
9410         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9411         ahci_portp->ahciport_mop_in_progress++;
9412 
9413         rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9414             port, flag, &reset_flag);
9415 
9416         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9417                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9418                 if (rval == AHCI_SUCCESS)
9419                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9420                             "succeed", instance, port);
9421                 else
9422                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9423                             "failed", instance, port);
9424         }
9425 
9426         /*
9427          * Won't retrieve error information:
9428          * 1. Port reset was involved to recover
9429          * 2. Device is gone
9430          * 3. IDENTIFY DEVICE command sent to ATAPI device
9431          * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9432          */
9433         if (reset_flag ||
9434             ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9435             spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9436             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9437                 goto out;
9438 
9439         /*
9440          * Deliver READ LOG EXT to gather information about the error when
9441          * a COMRESET has not been performed as part of the error recovery
9442          * during NCQ command processing.
9443          */
9444         if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9445                 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9446                     ahci_portp, port);
9447                 goto out;
9448         }
9449 
9450         /*
9451          * Deliver REQUEST SENSE for ATAPI command to gather information about
9452          * the error when a COMRESET has not been performed as part of the
9453          * error recovery.
9454          */
9455         if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9456                 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9457 out:
9458         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9459             "ahci_fatal_error_recovery_handler: port %d fatal error "
9460             "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9461             "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9462             port, slot_status, ahci_portp->ahciport_pending_tags,
9463             ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9464 
9465         ahci_mop_commands(ahci_ctlp,
9466             ahci_portp,
9467             slot_status,
9468             failed_tags, /* failed tags */
9469             0, /* timeout tags */
9470             0, /* aborted tags */
9471             0); /* reset tags */
9472 }
9473 
9474 /*
9475  * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9476  *      1. device specific.PxFBS.SDE=1
9477  *      2. Non-Deivce specific.
9478  * Nothing will be done when Command-based switching is employed.
9479  *
9480  * Currently code is neither completed nor tested.
9481  */
9482 static void
9483 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9484     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9485 {
9486 #ifndef __lock_lint
9487         _NOTE(ARGUNUSED(intr_status))
9488 #endif
9489         uint32_t        port_fbs_ctrl;
9490         int loop_count = 0;
9491         ahci_addr_t     addr;
9492 
9493         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9494 
9495         /* Nothing will be done under Command-based switching. */
9496         if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9497                 return;
9498 
9499         port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9500             (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9501 
9502         if (!(port_fbs_ctrl & AHCI_FBS_EN))
9503                 /* FBS is not enabled. */
9504                 return;
9505 
9506         /* Problem's getting complicated now. */
9507         /*
9508          * If FIS-based switching is used, we need to check
9509          * the PxFBS to see the error type.
9510          */
9511         port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9512             (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9513 
9514         /* Refer to spec(v1.2) 9.3.6.1 */
9515         if (port_fbs_ctrl & AHCI_FBS_SDE) {
9516                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9517                     "A Device Sepcific Error: port %d", port);
9518                 /*
9519                  * Controller has paused commands for all other
9520                  * sub-devices until PxFBS.DEC is set.
9521                  */
9522                 ahci_reject_all_abort_pkts(ahci_ctlp,
9523                     ahci_portp, 0);
9524 
9525                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9526                     (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9527                     port_fbs_ctrl | AHCI_FBS_DEC);
9528 
9529                 /*
9530                  * Wait controller clear PxFBS.DEC,
9531                  * then we can continue.
9532                  */
9533                 loop_count = 0;
9534                 do {
9535                         port_fbs_ctrl = ddi_get32(ahci_ctlp->
9536                             ahcictl_ahci_acc_handle, (uint32_t *)
9537                             AHCI_PORT_PxFBS(ahci_ctlp, port));
9538 
9539                         if (loop_count++ > 1000)
9540                                 /*
9541                                  * Esclate the error. Follow
9542                                  * non-device specific error
9543                                  * procedure.
9544                                  */
9545                                 return;
9546 
9547                         drv_usecwait(AHCI_100US_USECS);
9548                 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9549 
9550                 /*
9551                  * Issue a software reset to ensure drive is in
9552                  * a known state.
9553                  */
9554                 (void) ahci_software_reset(ahci_ctlp,
9555                     ahci_portp, &addr);
9556 
9557         } else {
9558 
9559                 /* Process Non-Device Specific Error. */
9560                 /* This will be handled later on. */
9561                 cmn_err(CE_NOTE, "!FBS is not supported now.");
9562         }
9563 }
9564 /*
9565  * Handle events - fatal error recovery
9566  */
9567 static void
9568 ahci_events_handler(void *args)
9569 {
9570         ahci_event_arg_t *ahci_event_arg;
9571         ahci_ctl_t *ahci_ctlp;
9572         ahci_port_t *ahci_portp;
9573         ahci_addr_t *addrp;
9574         uint32_t event;
9575         int instance;
9576 
9577         ahci_event_arg = (ahci_event_arg_t *)args;
9578 
9579         ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9580         ahci_portp = ahci_event_arg->ahciea_portp;
9581         addrp = ahci_event_arg->ahciea_addrp;
9582         event = ahci_event_arg->ahciea_event;
9583         instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9584 
9585         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9586             "ahci_events_handler enter: port %d intr_status = 0x%x",
9587             ahci_portp->ahciport_port_num, event);
9588 
9589         mutex_enter(&ahci_portp->ahciport_mutex);
9590 
9591         /*
9592          * ahci_intr_phyrdy_change() may have rendered it to
9593          * SATA_DTYPE_NONE.
9594          */
9595         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9596                 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9597                     "ahci_events_handler: port %d no device attached, "
9598                     "and just return without doing anything",
9599                     ahci_portp->ahciport_port_num);
9600 
9601                 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9602                         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9603                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9604                             "succeed", instance, ahci_portp->ahciport_port_num);
9605                 }
9606 
9607                 goto out;
9608         }
9609 
9610         if (event & (AHCI_INTR_STATUS_IFS |
9611             AHCI_INTR_STATUS_HBDS |
9612             AHCI_INTR_STATUS_HBFS |
9613             AHCI_INTR_STATUS_TFES))
9614                 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9615                     addrp, event);
9616 
9617 out:
9618         mutex_exit(&ahci_portp->ahciport_mutex);
9619 }
9620 
9621 /*
9622  * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9623  * detect there are some commands which are timed out.
9624  */
9625 static void
9626 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9627     uint8_t port, uint32_t tmp_timeout_tags)
9628 {
9629         uint32_t slot_status = 0;
9630         uint32_t finished_tags = 0;
9631         uint32_t timeout_tags = 0;
9632 
9633         AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9634             "ahci_timeout_pkts enter: port %d", port);
9635 
9636         mutex_enter(&ahci_portp->ahciport_mutex);
9637 
9638         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9639             RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9640             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9641                 /* Read PxCI to see which commands are still outstanding */
9642                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9643                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9644         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9645                 /* Read PxSACT to see which commands are still outstanding */
9646                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9647                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9648         }
9649 
9650 #if AHCI_DEBUG
9651         /*
9652          * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9653          * set, it means a fatal error happened after REQUEST SENSE command
9654          * or READ LOG EXT command is delivered to the HBA during the error
9655          * recovery process. At this time, the only outstanding command is
9656          * supposed to be REQUEST SENSE command or READ LOG EXT command.
9657          */
9658         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9659                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9660                     "ahci_timeout_pkts called while REQUEST SENSE "
9661                     "command or READ LOG EXT command for error recovery "
9662                     "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9663                     "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9664                     tmp_timeout_tags, slot_status,
9665                     ahci_portp->ahciport_pending_tags,
9666                     ahci_portp->ahciport_pending_ncq_tags);
9667                 ASSERT(slot_status == 0x1);
9668         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9669                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9670                     "ahci_timeout_pkts called while executing R/W PMULT "
9671                     "command timeout_tags = 0x%x, slot_status = 0x%x",
9672                     tmp_timeout_tags, slot_status);
9673                 ASSERT(slot_status == 0x1);
9674         }
9675 #endif
9676 
9677         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9678         ahci_portp->ahciport_mop_in_progress++;
9679 
9680         (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9681             port, AHCI_PORT_RESET, NULL);
9682 
9683         /*
9684          * Re-identify timeout tags because some previously checked commands
9685          * could already complete.
9686          */
9687         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9688                 finished_tags = ahci_portp->ahciport_pending_tags &
9689                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9690                 timeout_tags = tmp_timeout_tags & ~finished_tags;
9691 
9692                 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9693                     "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9694                     "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9695                     "pending_tags = 0x%x ",
9696                     port, finished_tags, timeout_tags,
9697                     slot_status, ahci_portp->ahciport_pending_tags);
9698         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9699                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9700                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9701                 timeout_tags = tmp_timeout_tags & ~finished_tags;
9702 
9703                 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9704                     "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9705                     "timeout_tags = 0x%x, port_sactive = 0x%x, "
9706                     "pending_ncq_tags = 0x%x ",
9707                     port, finished_tags, timeout_tags,
9708                     slot_status, ahci_portp->ahciport_pending_ncq_tags);
9709         } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9710             RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9711                 timeout_tags = tmp_timeout_tags;
9712         }
9713 
9714         ahci_mop_commands(ahci_ctlp,
9715             ahci_portp,
9716             slot_status,
9717             0,                  /* failed tags */
9718             timeout_tags,       /* timeout tags */
9719             0,                  /* aborted tags */
9720             0);                 /* reset tags */
9721 
9722         mutex_exit(&ahci_portp->ahciport_mutex);
9723 }
9724 
9725 /*
9726  * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9727  * for long time.
9728  */
9729 static void
9730 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9731 {
9732         ahci_port_t *ahci_portp;
9733         sata_pkt_t *spkt;
9734         uint32_t pending_tags;
9735         uint32_t timeout_tags;
9736         uint32_t port_cmd_status;
9737         uint32_t port_sactive;
9738         uint8_t port;
9739         int tmp_slot;
9740         int current_slot;
9741         uint32_t current_tags;
9742         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9743 
9744         mutex_enter(&ahci_ctlp->ahcictl_mutex);
9745 
9746         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9747             "ahci_watchdog_handler entered", NULL);
9748 
9749         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9750                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9751                         continue;
9752                 }
9753 
9754                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9755 
9756                 mutex_enter(&ahci_portp->ahciport_mutex);
9757                 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9758                         mutex_exit(&ahci_portp->ahciport_mutex);
9759                         continue;
9760                 }
9761 
9762                 /* Skip the check for those ports in error recovery */
9763                 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9764                     !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9765                         mutex_exit(&ahci_portp->ahciport_mutex);
9766                         continue;
9767                 }
9768 
9769                 pending_tags = 0;
9770                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9771                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9772 
9773                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9774                     RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9775                         current_slot = 0;
9776                         pending_tags = 0x1;
9777                 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9778                         current_slot =
9779                             (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9780                             AHCI_CMD_STATUS_CCS_SHIFT;
9781                         pending_tags = ahci_portp->ahciport_pending_tags;
9782                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9783                         port_sactive = ddi_get32(
9784                             ahci_ctlp->ahcictl_ahci_acc_handle,
9785                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9786                         current_tags = port_sactive &
9787                             ~port_cmd_status &
9788                             AHCI_NCQ_SLOT_MASK(ahci_portp);
9789                         pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9790                 }
9791 
9792                 timeout_tags = 0;
9793                 while (pending_tags) {
9794                         tmp_slot = ddi_ffs(pending_tags) - 1;
9795                         if (tmp_slot == -1) {
9796                                 break;
9797                         }
9798 
9799                         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9800                                 spkt = ahci_portp->ahciport_err_retri_pkt;
9801                         else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9802                                 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9803                         else
9804                                 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9805 
9806                         if ((spkt != NULL) && spkt->satapkt_time &&
9807                             !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9808                                 /*
9809                                  * If a packet has survived for more than it's
9810                                  * max life cycles, it is a candidate for time
9811                                  * out.
9812                                  */
9813                                 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9814                                     ahci_watchdog_timeout;
9815 
9816                                 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9817                                     > 0)
9818                                         goto next;
9819 
9820 #if AHCI_DEBUG
9821                                 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9822                                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9823                                             ahci_ctlp, "watchdog: the current "
9824                                             "tags is 0x%x", current_tags);
9825                                 } else {
9826                                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9827                                             ahci_ctlp, "watchdog: the current "
9828                                             "slot is %d", current_slot);
9829                                 }
9830 #endif
9831 
9832                                 /*
9833                                  * We need to check whether the HBA has
9834                                  * begun to execute the command, if not,
9835                                  * then re-set the timer of the command.
9836                                  */
9837                                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9838                                     (tmp_slot != current_slot) ||
9839                                     NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9840                                     ((0x1 << tmp_slot) & current_tags)) {
9841                                         ahci_portp->ahciport_slot_timeout \
9842                                             [tmp_slot] = spkt->satapkt_time;
9843                                 } else {
9844                                         timeout_tags |= (0x1 << tmp_slot);
9845                                         cmn_err(CE_WARN, "!ahci%d: watchdog "
9846                                             "port %d satapkt 0x%p timed out\n",
9847                                             instance, port, (void *)spkt);
9848                                 }
9849                         }
9850 next:
9851                         CLEAR_BIT(pending_tags, tmp_slot);
9852                 }
9853 
9854                 if (timeout_tags) {
9855                         mutex_exit(&ahci_portp->ahciport_mutex);
9856                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
9857                         ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9858                             port, timeout_tags);
9859                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
9860                         mutex_enter(&ahci_portp->ahciport_mutex);
9861                 }
9862 
9863                 mutex_exit(&ahci_portp->ahciport_mutex);
9864         }
9865 
9866         /* Re-install the watchdog timeout handler */
9867         if (ahci_ctlp->ahcictl_timeout_id != 0) {
9868                 ahci_ctlp->ahcictl_timeout_id =
9869                     timeout((void (*)(void *))ahci_watchdog_handler,
9870                     (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9871         }
9872 
9873         mutex_exit(&ahci_ctlp->ahcictl_mutex);
9874 }
9875 
9876 /*
9877  * Fill the error context into sata_cmd for non-queued command error.
9878  */
9879 static void
9880 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9881 {
9882         scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9883         scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9884         scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9885         scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9886         scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9887         scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9888         scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9889 
9890         if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9891                 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9892                 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9893                 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9894                 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9895         }
9896 }
9897 
9898 /*
9899  * Fill the ncq error page into sata_cmd for queued command error.
9900  */
9901 static void
9902 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9903     struct sata_ncq_error_recovery_page *ncq_err_page)
9904 {
9905         scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9906         scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9907         scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
9908         scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
9909         scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
9910         scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
9911         scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
9912         scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
9913         scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
9914         scmd->satacmd_status_reg = ncq_err_page->ncq_status;
9915         scmd->satacmd_error_reg = ncq_err_page->ncq_error;
9916 }
9917 
9918 /*
9919  * Put the respective register value to sata_cmd_t for satacmd_flags.
9920  */
9921 static void
9922 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9923 {
9924         if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
9925                 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9926         if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
9927                 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9928         if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
9929                 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9930         if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
9931                 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9932         if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
9933                 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9934         if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
9935                 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9936         if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
9937                 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9938         if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
9939                 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9940         if (scmd->satacmd_flags.sata_copy_out_device_reg)
9941                 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9942         if (scmd->satacmd_flags.sata_copy_out_error_reg)
9943                 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9944 }
9945 
9946 static void
9947 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
9948     uint32_t intr_status)
9949 {
9950         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9951 
9952         if (intr_status & AHCI_INTR_STATUS_IFS)
9953                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
9954                     "error", instance, port);
9955 
9956         if (intr_status & AHCI_INTR_STATUS_HBDS)
9957                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
9958                     instance, port);
9959 
9960         if (intr_status & AHCI_INTR_STATUS_HBFS)
9961                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
9962                     instance, port);
9963 
9964         if (intr_status & AHCI_INTR_STATUS_TFES)
9965                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
9966                     instance, port);
9967 
9968         cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
9969             "recovery", instance, port);
9970 }
9971 
9972 static void
9973 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
9974     uint32_t slot_tags)
9975 {
9976         ahci_port_t *ahci_portp;
9977         int tmp_slot;
9978         sata_pkt_t *spkt;
9979         sata_cmd_t cmd;
9980 
9981         ahci_portp = ahci_ctlp->ahcictl_ports[port];
9982         ASSERT(ahci_portp != NULL);
9983 
9984         while (slot_tags) {
9985                 tmp_slot = ddi_ffs(slot_tags) - 1;
9986                 if (tmp_slot == -1) {
9987                         break;
9988                 }
9989 
9990                 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9991                 ASSERT(spkt != NULL);
9992                 cmd = spkt->satapkt_cmd;
9993 
9994                 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
9995                     "features_reg = 0x%x sec_count_msb = 0x%x "
9996                     "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
9997                     "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
9998                     "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
9999                     "lba_high_lsb = 0x%x device_reg = 0x%x "
10000                     "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10001                     cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10002                     cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10003                     cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10004                     cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10005                     cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10006                     cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10007                     *((uint32_t *)&(cmd.satacmd_flags)));
10008 
10009                 CLEAR_BIT(slot_tags, tmp_slot);
10010         }
10011 }
10012 
10013 /*
10014  * Dump the serror message to the log.
10015  */
10016 static void
10017 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10018     uint32_t port_serror, int debug_only)
10019 {
10020         static char err_buf[512];
10021         static char err_msg_header[16];
10022         char *err_msg = err_buf;
10023 
10024         *err_buf = '\0';
10025         *err_msg_header = '\0';
10026 
10027         if (port_serror & SERROR_DATA_ERR_FIXED) {
10028                 err_msg = strcat(err_msg,
10029                     "\tRecovered Data Integrity Error (I)\n");
10030         }
10031 
10032         if (port_serror & SERROR_COMM_ERR_FIXED) {
10033                 err_msg = strcat(err_msg,
10034                     "\tRecovered Communication Error (M)\n");
10035         }
10036 
10037         if (port_serror & SERROR_DATA_ERR) {
10038                 err_msg = strcat(err_msg,
10039                     "\tTransient Data Integrity Error (T)\n");
10040         }
10041 
10042         if (port_serror & SERROR_PERSISTENT_ERR) {
10043                 err_msg = strcat(err_msg,
10044                     "\tPersistent Communication or Data Integrity Error (C)\n");
10045         }
10046 
10047         if (port_serror & SERROR_PROTOCOL_ERR) {
10048                 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10049         }
10050 
10051         if (port_serror & SERROR_INT_ERR) {
10052                 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10053         }
10054 
10055         if (port_serror & SERROR_PHY_RDY_CHG) {
10056                 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10057         }
10058 
10059         if (port_serror & SERROR_PHY_INT_ERR) {
10060                 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10061         }
10062 
10063         if (port_serror & SERROR_COMM_WAKE) {
10064                 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10065         }
10066 
10067         if (port_serror & SERROR_10B_TO_8B_ERR) {
10068                 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10069         }
10070 
10071         if (port_serror & SERROR_DISPARITY_ERR) {
10072                 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10073         }
10074 
10075         if (port_serror & SERROR_CRC_ERR) {
10076                 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10077         }
10078 
10079         if (port_serror & SERROR_HANDSHAKE_ERR) {
10080                 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10081         }
10082 
10083         if (port_serror & SERROR_LINK_SEQ_ERR) {
10084                 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10085         }
10086 
10087         if (port_serror & SERROR_TRANS_ERR) {
10088                 err_msg = strcat(err_msg,
10089                     "\tTransport state transition error (T)\n");
10090         }
10091 
10092         if (port_serror & SERROR_FIS_TYPE) {
10093                 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10094         }
10095 
10096         if (port_serror & SERROR_EXCHANGED_ERR) {
10097                 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10098         }
10099 
10100         if (*err_msg == '\0')
10101                 return;
10102 
10103         if (debug_only) {
10104                 (void) sprintf(err_msg_header, "port %d", port);
10105                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10106                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10107         } else if (ahci_ctlp) {
10108                 cmn_err(CE_WARN, "!ahci%d: %s %s",
10109                     ddi_get_instance(ahci_ctlp->ahcictl_dip),
10110                     err_msg_header, err_msg);
10111 
10112                 /* sata trace debug */
10113                 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10114                     "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10115                     err_msg_header, err_msg);
10116         } else {
10117                 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10118 
10119                 /* sata trace debug */
10120                 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10121         }
10122 }
10123 
10124 /*
10125  * Translate the sata_address_t type into the ahci_addr_t type.
10126  * sata_device.satadev_addr structure is used as source.
10127  */
10128 static void
10129 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10130     ahci_addr_t *ahci_addrp)
10131 {
10132         sata_address_t *sata_addrp = &sd->satadev_addr;
10133         ahci_addrp->aa_port =
10134             ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10135         ahci_addrp->aa_pmport = sata_addrp->pmport;
10136 
10137         switch (sata_addrp->qual) {
10138         case SATA_ADDR_DCPORT:
10139         case SATA_ADDR_CPORT:
10140                 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10141                 break;
10142         case SATA_ADDR_PMULT:
10143         case SATA_ADDR_PMULT_SPEC:
10144                 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10145                 break;
10146         case SATA_ADDR_DPMPORT:
10147         case SATA_ADDR_PMPORT:
10148                 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10149                 break;
10150         case SATA_ADDR_NULL:
10151         default:
10152                 /* something went wrong */
10153                 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10154                 break;
10155         }
10156 }
10157 
10158 /*
10159  * This routine is to calculate the total number of ports implemented
10160  * by the HBA.
10161  */
10162 static int
10163 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10164 {
10165         uint8_t i;
10166         int num = 0;
10167 
10168         for (i = 0; i < AHCI_MAX_PORTS; i++) {
10169                 if (((uint32_t)0x1 << i) & ports_implemented)
10170                         num++;
10171         }
10172 
10173         return (num);
10174 }
10175 
10176 #if AHCI_DEBUG
10177 static void
10178 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10179 {
10180         static char name[16];
10181         va_list ap;
10182 
10183         mutex_enter(&ahci_log_mutex);
10184 
10185         va_start(ap, fmt);
10186         if (ahci_ctlp) {
10187                 (void) sprintf(name, "ahci%d: ",
10188                     ddi_get_instance(ahci_ctlp->ahcictl_dip));
10189         } else {
10190                 (void) sprintf(name, "ahci: ");
10191         }
10192 
10193         (void) vsprintf(ahci_log_buf, fmt, ap);
10194         va_end(ap);
10195 
10196         cmn_err(level, "%s%s", name, ahci_log_buf);
10197 
10198         mutex_exit(&ahci_log_mutex);
10199 }
10200 #endif
10201 
10202 /*
10203  * quiesce(9E) entry point.
10204  *
10205  * This function is called when the system is single-threaded at high
10206  * PIL with preemption disabled. Therefore, this function must not be
10207  * blocked.
10208  *
10209  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10210  * DDI_FAILURE indicates an error condition and should almost never happen.
10211  */
10212 static int
10213 ahci_quiesce(dev_info_t *dip)
10214 {
10215         ahci_ctl_t *ahci_ctlp;
10216         ahci_port_t *ahci_portp;
10217         int instance, port;
10218 
10219         instance = ddi_get_instance(dip);
10220         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10221 
10222         if (ahci_ctlp == NULL)
10223                 return (DDI_FAILURE);
10224 
10225 #if AHCI_DEBUG
10226         ahci_debug_flags = 0;
10227 #endif
10228 
10229         ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10230 
10231         /* disable all the interrupts. */
10232         ahci_disable_all_intrs(ahci_ctlp);
10233 
10234         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10235                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10236                         continue;
10237                 }
10238 
10239                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10240 
10241                 /*
10242                  * Stop the port by clearing PxCMD.ST
10243                  *
10244                  * Here we must disable the port interrupt because
10245                  * ahci_disable_all_intrs only clear GHC.IE, and IS
10246                  * register will be still set if PxIE is enabled.
10247                  * When ahci shares one IRQ with other drivers, the
10248                  * intr handler may claim the intr mistakenly.
10249                  */
10250                 ahci_disable_port_intrs(ahci_ctlp, port);
10251                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10252                     ahci_portp, port);
10253         }
10254 
10255         ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10256 
10257         return (DDI_SUCCESS);
10258 }
10259 
10260 /*
10261  * The function will add a sata packet to the done queue.
10262  */
10263 static void
10264 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10265 {
10266         ASSERT(satapkt != NULL);
10267         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10268 
10269         /* set the reason for all packets */
10270         satapkt->satapkt_reason = reason;
10271         satapkt->satapkt_hba_driver_private = NULL;
10272 
10273         if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10274             satapkt->satapkt_comp) {
10275                 /*
10276                  * only add to queue when mode is not synch and there is
10277                  * completion callback
10278                  */
10279                 *ahci_portp->ahciport_doneqtail = satapkt;
10280                 ahci_portp->ahciport_doneqtail =
10281                     (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10282                 ahci_portp->ahciport_doneq_len++;
10283 
10284         } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10285             ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10286                 /*
10287                  * for sync/non-poll mode, just call cv_broadcast
10288                  */
10289                 cv_broadcast(&ahci_portp->ahciport_cv);
10290 }
10291 
10292 /*
10293  * The function will call completion callback of sata packet on the
10294  * completed queue
10295  */
10296 static void
10297 ahci_flush_doneq(ahci_port_t *ahci_portp)
10298 {
10299         sata_pkt_t *satapkt, *next;
10300 
10301         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10302 
10303         if (ahci_portp->ahciport_doneq) {
10304                 satapkt = ahci_portp->ahciport_doneq;
10305 
10306                 ahci_portp->ahciport_doneq = NULL;
10307                 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10308                 ahci_portp->ahciport_doneq_len = 0;
10309 
10310                 mutex_exit(&ahci_portp->ahciport_mutex);
10311 
10312                 while (satapkt != NULL) {
10313                         next = satapkt->satapkt_hba_driver_private;
10314                         satapkt->satapkt_hba_driver_private = NULL;
10315 
10316                         /* Call the callback */
10317                         (*satapkt->satapkt_comp)(satapkt);
10318 
10319                         satapkt = next;
10320                 }
10321 
10322                 mutex_enter(&ahci_portp->ahciport_mutex);
10323         }
10324 }