1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
  25  * Copyright (c) 2018, Joyent, Inc.
  26  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  27  */
  28 
  29 /*
  30  * AHCI (Advanced Host Controller Interface) SATA HBA Driver
  31  *
  32  * Power Management Support
  33  * ------------------------
  34  *
  35  * At the moment, the ahci driver only implements suspend/resume to
  36  * support Suspend to RAM on X86 feature. Device power management isn't
  37  * implemented, link power management is disabled, and hot plug isn't
  38  * allowed during the period from suspend to resume.
  39  *
  40  * For s/r support, the ahci driver only need to implement DDI_SUSPEND
  41  * and DDI_RESUME entries, and don't need to take care of new requests
  42  * sent down after suspend because the target driver (sd) has already
  43  * handled these conditions, and blocked these requests. For the detailed
  44  * information, please check with sdopen, sdclose and sdioctl routines.
  45  *
  46  *
  47  * Enclosure Management Support
  48  * ----------------------------
  49  *
  50  * The ahci driver has basic support for AHCI Enclosure Management (EM)
  51  * services. The AHCI specification provides an area in the primary ahci BAR for
  52  * posting data to send out to the enclosure management and provides a register
  53  * that provides both information and control about this. While the
  54  * specification allows for multiple forms of enclosure management, the only
  55  * supported, and commonly found form, is the AHCI specified LED format. The LED
  56  * format is often implemented as a one-way communication mechanism. Software
  57  * can write out what it cares about into the aforementioned data buffer and
  58  * then we wait for the transmission to be sent.
  59  *
  60  * This has some drawbacks. It means that we cannot know whether or not it has
  61  * succeeded. This means we cannot ask hardware what it thinks the LEDs are
  62  * set to. There's also the added unfortunate reality that firmware on the
  63  * microcontroller driving this will often not show the LEDs if no drive is
  64  * present and that actions taken may potentially cause this to get out of sync
  65  * with what we expect it to be. For example, the specification does not
  66  * describe what should happen if a drive is removed from the enclosure while
  67  * this is set and what should happen when it returns. We can only infer that it
  68  * should be the same.
  69  *
  70  * Because only a single command can be sent at any time and we don't want to
  71  * interfere with controller I/O, we create a taskq dedicated to this that has a
  72  * single thread. Both resets (which occur on attach and resume) and normal
  73  * changes to the LED state will be driven through this taskq. Because the taskq
  74  * has a single thread, this guarantees serial processing.
  75  *
  76  * Each userland-submitted task (basically not resets) has a reference counted
  77  * task structure. This allows the thread that called it to be cancelled and
  78  * have the system clean itself up. The user thread in ioctl blocks on a CV that
  79  * can receive signals as it waits for completion.  Note, there is no guarantee
  80  * provided by the kernel that the first thread to enter the kernel will be the
  81  * first one to change state.
  82  */
  83 
  84 #include <sys/note.h>
  85 #include <sys/scsi/scsi.h>
  86 #include <sys/pci.h>
  87 #include <sys/disp.h>
  88 #include <sys/sata/sata_hba.h>
  89 #include <sys/sata/adapters/ahci/ahcireg.h>
  90 #include <sys/sata/adapters/ahci/ahcivar.h>
  91 
  92 /*
  93  * FMA header files
  94  */
  95 #include <sys/ddifm.h>
  96 #include <sys/fm/protocol.h>
  97 #include <sys/fm/util.h>
  98 #include <sys/fm/io/ddi.h>
  99 
 100 /*
 101  * EM Control header files
 102  */
 103 #include <sys/types.h>
 104 #include <sys/file.h>
 105 #include <sys/errno.h>
 106 #include <sys/open.h>
 107 #include <sys/cred.h>
 108 #include <sys/ddi.h>
 109 #include <sys/sunddi.h>
 110 
 111 /*
 112  * This is the string displayed by modinfo, etc.
 113  */
 114 static char ahci_ident[] = "ahci driver";
 115 
 116 /*
 117  * Function prototypes for driver entry points
 118  */
 119 static  int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
 120 static  int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
 121 static  int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
 122 static  int ahci_quiesce(dev_info_t *);
 123 
 124 /*
 125  * Function prototypes for SATA Framework interfaces
 126  */
 127 static  int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
 128 static  int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
 129 
 130 static  int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
 131 static  int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
 132 static  int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
 133 static  int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
 134 static  int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
 135 static  int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
 136 #if defined(__lock_lint)
 137 static  int ahci_selftest(dev_info_t *, sata_device_t *);
 138 #endif
 139 
 140 /*
 141  * FMA Prototypes
 142  */
 143 static  void ahci_fm_init(ahci_ctl_t *);
 144 static  void ahci_fm_fini(ahci_ctl_t *);
 145 static  int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
 146 int     ahci_check_acc_handle(ddi_acc_handle_t);
 147 int     ahci_check_dma_handle(ddi_dma_handle_t);
 148 void    ahci_fm_ereport(ahci_ctl_t *, char *);
 149 static  int ahci_check_all_handle(ahci_ctl_t *);
 150 static  int ahci_check_ctl_handle(ahci_ctl_t *);
 151 static  int ahci_check_port_handle(ahci_ctl_t *, int);
 152 static  int ahci_check_slot_handle(ahci_port_t *, int);
 153 
 154 /*
 155  * Local function prototypes
 156  */
 157 static  int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
 158 static  int ahci_alloc_ports_state(ahci_ctl_t *);
 159 static  void ahci_dealloc_ports_state(ahci_ctl_t *);
 160 static  int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
 161 static  void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
 162 static  int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
 163 static  void ahci_dealloc_rcvd_fis(ahci_port_t *);
 164 static  int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
 165 static  void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
 166 static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
 167 static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
 168 static  void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
 169 static  void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
 170 
 171 static  int ahci_initialize_controller(ahci_ctl_t *);
 172 static  void ahci_uninitialize_controller(ahci_ctl_t *);
 173 static  int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 174 static  int ahci_config_space_init(ahci_ctl_t *);
 175 static  void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
 176 
 177 static  void ahci_drain_ports_taskq(ahci_ctl_t *);
 178 static  int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
 179     uint8_t);
 180 static  int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
 181 static  int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
 182 static  int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
 183     sata_device_t *);
 184 static  int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
 185     sata_pmult_gscr_t *);
 186 static  int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
 187     sata_device_t *);
 188 static  int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 189 static  int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 190 static  int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
 191     sata_device_t *);
 192 
 193 static  void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
 194 static  int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
 195 static  void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
 196     ahci_addr_t *);
 197 static  void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
 198 static  int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
 199     ahci_addr_t *, sata_pkt_t *);
 200 static  int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
 201     ahci_addr_t *, sata_pkt_t *);
 202 static  int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
 203     ahci_addr_t *, int);
 204 static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
 205 static  void ahci_copy_ncq_err_page(sata_cmd_t *,
 206     struct sata_ncq_error_recovery_page *);
 207 static  void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
 208 static  void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
 209 static  void ahci_flush_doneq(ahci_port_t *);
 210 
 211 static  int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 212 static  int ahci_hba_reset(ahci_ctl_t *);
 213 static  int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 214 static  int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 215 static  void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
 216 static  int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 217     ahci_addr_t *);
 218 static  int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 219     ahci_addr_t *);
 220 static  int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 221     ahci_addr_t *);
 222 static  int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
 223 static  int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
 224     uint8_t);
 225 static  int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
 226     uint8_t, int, int *);
 227 static  void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
 228     uint32_t, uint32_t, uint32_t, uint32_t);
 229 static  uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
 230 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
 231     uint8_t, sata_pkt_t *);
 232 static  void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
 233     ahci_addr_t *, uint32_t);
 234 static  void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
 235     uint8_t, uint32_t);
 236 static  void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
 237     uint8_t, uint32_t);
 238 static  void ahci_events_handler(void *);
 239 static  void ahci_watchdog_handler(ahci_ctl_t *);
 240 
 241 static  uint_t ahci_intr(caddr_t, caddr_t);
 242 static  void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
 243 static  int ahci_add_intrs(ahci_ctl_t *, int);
 244 static  void ahci_rem_intrs(ahci_ctl_t *);
 245 static  void ahci_enable_all_intrs(ahci_ctl_t *);
 246 static  void ahci_disable_all_intrs(ahci_ctl_t *);
 247 static  void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
 248 static  void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
 249 
 250 static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
 251 static  int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
 252 static  int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 253 static  int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
 254 static  int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
 255 static  int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
 256     ahci_port_t *, uint8_t);
 257 static  int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
 258 static  int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
 259     uint8_t, uint32_t);
 260 static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
 261     uint8_t, uint32_t);
 262 static  int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
 263 
 264 static  void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
 265 static  int ahci_get_num_implemented_ports(uint32_t);
 266 static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
 267 static  void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
 268 static  void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
 269 #if AHCI_DEBUG
 270 static  void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
 271 #endif
 272 
 273 static  boolean_t ahci_em_init(ahci_ctl_t *);
 274 static  void ahci_em_fini(ahci_ctl_t *);
 275 static  void ahci_em_suspend(ahci_ctl_t *);
 276 static  void ahci_em_resume(ahci_ctl_t *);
 277 static  int ahci_em_ioctl(dev_info_t *, int, intptr_t);
 278 
 279 
 280 /*
 281  * DMA attributes for the data buffer
 282  *
 283  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 284  * does not support 64-bit addressing
 285  */
 286 static ddi_dma_attr_t buffer_dma_attr = {
 287         DMA_ATTR_V0,            /* dma_attr_version */
 288         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 289         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 290         0x3fffffull,            /* dma_attr_count_max i.e. for one cookie */
 291         0x2ull,                 /* dma_attr_align: word aligned */
 292         1,                      /* dma_attr_burstsizes */
 293         1,                      /* dma_attr_minxfer */
 294         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 295         0xffffffffull,          /* dma_attr_seg */
 296         AHCI_PRDT_NUMBER,       /* dma_attr_sgllen */
 297         512,                    /* dma_attr_granular */
 298         0,                      /* dma_attr_flags */
 299 };
 300 
 301 /*
 302  * DMA attributes for the rcvd FIS
 303  *
 304  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 305  * does not support 64-bit addressing
 306  */
 307 static ddi_dma_attr_t rcvd_fis_dma_attr = {
 308         DMA_ATTR_V0,            /* dma_attr_version */
 309         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 310         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 311         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 312         0x100ull,               /* dma_attr_align: 256-byte aligned */
 313         1,                      /* dma_attr_burstsizes */
 314         1,                      /* dma_attr_minxfer */
 315         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 316         0xffffffffull,          /* dma_attr_seg */
 317         1,                      /* dma_attr_sgllen */
 318         1,                      /* dma_attr_granular */
 319         0,                      /* dma_attr_flags */
 320 };
 321 
 322 /*
 323  * DMA attributes for the command list
 324  *
 325  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 326  * does not support 64-bit addressing
 327  */
 328 static ddi_dma_attr_t cmd_list_dma_attr = {
 329         DMA_ATTR_V0,            /* dma_attr_version */
 330         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 331         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 332         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 333         0x400ull,               /* dma_attr_align: 1K-byte aligned */
 334         1,                      /* dma_attr_burstsizes */
 335         1,                      /* dma_attr_minxfer */
 336         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 337         0xffffffffull,          /* dma_attr_seg */
 338         1,                      /* dma_attr_sgllen */
 339         1,                      /* dma_attr_granular */
 340         0,                      /* dma_attr_flags */
 341 };
 342 
 343 /*
 344  * DMA attributes for cmd tables
 345  *
 346  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 347  * does not support 64-bit addressing
 348  */
 349 static ddi_dma_attr_t cmd_table_dma_attr = {
 350         DMA_ATTR_V0,            /* dma_attr_version */
 351         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 352         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 353         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 354         0x80ull,                /* dma_attr_align: 128-byte aligned */
 355         1,                      /* dma_attr_burstsizes */
 356         1,                      /* dma_attr_minxfer */
 357         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 358         0xffffffffull,          /* dma_attr_seg */
 359         1,                      /* dma_attr_sgllen */
 360         1,                      /* dma_attr_granular */
 361         0,                      /* dma_attr_flags */
 362 };
 363 
 364 
 365 /* Device access attributes */
 366 static ddi_device_acc_attr_t accattr = {
 367         DDI_DEVICE_ATTR_V1,
 368         DDI_STRUCTURE_LE_ACC,
 369         DDI_STRICTORDER_ACC,
 370         DDI_DEFAULT_ACC
 371 };
 372 
 373 static struct dev_ops ahcictl_dev_ops = {
 374         DEVO_REV,               /* devo_rev */
 375         0,                      /* refcnt  */
 376         ahci_getinfo,           /* info */
 377         nulldev,                /* identify */
 378         nulldev,                /* probe */
 379         ahci_attach,            /* attach */
 380         ahci_detach,            /* detach */
 381         nodev,                  /* no reset */
 382         NULL,                   /* driver operations */
 383         NULL,                   /* bus operations */
 384         NULL,                   /* power */
 385         ahci_quiesce,           /* quiesce */
 386 };
 387 
 388 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
 389         SATA_TRAN_HOTPLUG_OPS_REV_1,
 390         ahci_tran_hotplug_port_activate,
 391         ahci_tran_hotplug_port_deactivate
 392 };
 393 
 394 extern struct mod_ops mod_driverops;
 395 
 396 static  struct modldrv modldrv = {
 397         &mod_driverops,             /* driverops */
 398         ahci_ident,             /* short description */
 399         &ahcictl_dev_ops,   /* driver ops */
 400 };
 401 
 402 static  struct modlinkage modlinkage = {
 403         MODREV_1,
 404         &modldrv,
 405         NULL
 406 };
 407 
 408 /* The following variables are watchdog handler related */
 409 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
 410 static clock_t ahci_watchdog_tick;
 411 
 412 /*
 413  * This static variable indicates the size of command table,
 414  * and it's changeable with prdt number, which ahci_dma_prdt_number
 415  * indicates.
 416  */
 417 static size_t ahci_cmd_table_size;
 418 
 419 /*
 420  * The below global variables are tunable via /etc/system
 421  *
 422  *      ahci_dma_prdt_number
 423  *      ahci_msi_enabled
 424  *      ahci_buf_64bit_dma
 425  *      ahci_commu_64bit_dma
 426  */
 427 
 428 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
 429 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
 430 
 431 /* AHCI MSI is tunable */
 432 boolean_t ahci_msi_enabled = B_TRUE;
 433 
 434 /*
 435  * 64-bit dma addressing for data buffer is tunable
 436  *
 437  * The variable controls only the below value:
 438  *      DBAU (upper 32-bits physical address of data block)
 439  */
 440 boolean_t ahci_buf_64bit_dma = B_TRUE;
 441 
 442 /*
 443  * 64-bit dma addressing for communication system descriptors is tunable
 444  *
 445  * The variable controls the below three values:
 446  *
 447  *      PxCLBU (upper 32-bits for the command list base physical address)
 448  *      PxFBU (upper 32-bits for the received FIS base physical address)
 449  *      CTBAU (upper 32-bits of command table base)
 450  */
 451 boolean_t ahci_commu_64bit_dma = B_TRUE;
 452 
 453 /*
 454  * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
 455  * chipset. If the users want to have a try with 64-bit dma, please change
 456  * the below variable value to enable it.
 457  */
 458 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
 459 
 460 /*
 461  * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
 462  * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
 463  * please change the below value to enable it.
 464  */
 465 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
 466 
 467 /*
 468  * These values control the default delay and default number of times to wait
 469  * for an enclosure message to complete.
 470  */
 471 uint_t  ahci_em_reset_delay_ms = 1;
 472 uint_t  ahci_em_reset_delay_count = 1000;
 473 uint_t  ahci_em_tx_delay_ms = 1;
 474 uint_t  ahci_em_tx_delay_count = 1000;
 475 
 476 
 477 /*
 478  * End of global tunable variable definition
 479  */
 480 
 481 #if AHCI_DEBUG
 482 uint32_t ahci_debug_flags = 0;
 483 #else
 484 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
 485 #endif
 486 
 487 
 488 #if AHCI_DEBUG
 489 /* The following is needed for ahci_log() */
 490 static kmutex_t ahci_log_mutex;
 491 static char ahci_log_buf[512];
 492 #endif
 493 
 494 /* Opaque state pointer initialized by ddi_soft_state_init() */
 495 static void *ahci_statep = NULL;
 496 
 497 /*
 498  *  ahci module initialization.
 499  */
 500 int
 501 _init(void)
 502 {
 503         int     ret;
 504 
 505         ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
 506         if (ret != 0) {
 507                 goto err_out;
 508         }
 509 
 510 #if AHCI_DEBUG
 511         mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
 512 #endif
 513 
 514         if ((ret = sata_hba_init(&modlinkage)) != 0) {
 515 #if AHCI_DEBUG
 516                 mutex_destroy(&ahci_log_mutex);
 517 #endif
 518                 ddi_soft_state_fini(&ahci_statep);
 519                 goto err_out;
 520         }
 521 
 522         /* watchdog tick */
 523         ahci_watchdog_tick = drv_usectohz(
 524             (clock_t)ahci_watchdog_timeout * 1000000);
 525 
 526         ret = mod_install(&modlinkage);
 527         if (ret != 0) {
 528                 sata_hba_fini(&modlinkage);
 529 #if AHCI_DEBUG
 530                 mutex_destroy(&ahci_log_mutex);
 531 #endif
 532                 ddi_soft_state_fini(&ahci_statep);
 533                 goto err_out;
 534         }
 535 
 536         return (ret);
 537 
 538 err_out:
 539         cmn_err(CE_WARN, "!ahci: Module init failed");
 540         return (ret);
 541 }
 542 
 543 /*
 544  * ahci module uninitialize.
 545  */
 546 int
 547 _fini(void)
 548 {
 549         int     ret;
 550 
 551         ret = mod_remove(&modlinkage);
 552         if (ret != 0) {
 553                 return (ret);
 554         }
 555 
 556         /* Remove the resources allocated in _init(). */
 557         sata_hba_fini(&modlinkage);
 558 #if AHCI_DEBUG
 559         mutex_destroy(&ahci_log_mutex);
 560 #endif
 561         ddi_soft_state_fini(&ahci_statep);
 562 
 563         return (ret);
 564 }
 565 
 566 /*
 567  * _info entry point
 568  */
 569 int
 570 _info(struct modinfo *modinfop)
 571 {
 572         return (mod_info(&modlinkage, modinfop));
 573 }
 574 
 575 /*
 576  * The attach entry point for dev_ops.
 577  */
 578 static int
 579 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 580 {
 581         ahci_ctl_t *ahci_ctlp = NULL;
 582         int instance = ddi_get_instance(dip);
 583         int status;
 584         int attach_state;
 585         uint32_t cap_status, ahci_version;
 586         uint32_t ghc_control;
 587         int intr_types;
 588         int i;
 589         pci_regspec_t *regs;
 590         int regs_length;
 591         int rnumber;
 592 #if AHCI_DEBUG
 593         int speed;
 594 #endif
 595 
 596         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
 597             NULL);
 598 
 599         switch (cmd) {
 600         case DDI_ATTACH:
 601                 break;
 602 
 603         case DDI_RESUME:
 604 
 605                 /*
 606                  * During DDI_RESUME, the hardware state of the device
 607                  * (power may have been removed from the device) must be
 608                  * restored, allow pending requests to continue, and
 609                  * service new requests.
 610                  */
 611                 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
 612                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 613 
 614                 /*
 615                  * GHC.AE must be set to 1 before any other AHCI register
 616                  * is accessed
 617                  */
 618                 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 619                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
 620                 ghc_control |= AHCI_HBA_GHC_AE;
 621                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
 622                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
 623 
 624                 /* Restart watch thread */
 625                 if (ahci_ctlp->ahcictl_timeout_id == 0)
 626                         ahci_ctlp->ahcictl_timeout_id = timeout(
 627                             (void (*)(void *))ahci_watchdog_handler,
 628                             (caddr_t)ahci_ctlp, ahci_watchdog_tick);
 629 
 630                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 631 
 632                 /*
 633                  * Re-initialize the controller and enable the interrupts and
 634                  * restart all the ports.
 635                  *
 636                  * Note that so far we don't support hot-plug during
 637                  * suspend/resume.
 638                  */
 639                 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
 640                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
 641                             "Failed to initialize the controller "
 642                             "during DDI_RESUME", NULL);
 643                         return (DDI_FAILURE);
 644                 }
 645 
 646                 /*
 647                  * Reset the enclosure services.
 648                  */
 649                 ahci_em_resume(ahci_ctlp);
 650 
 651                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 652                 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
 653                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 654 
 655                 return (DDI_SUCCESS);
 656 
 657         default:
 658                 return (DDI_FAILURE);
 659         }
 660 
 661         attach_state = AHCI_ATTACH_STATE_NONE;
 662 
 663         /* Allocate soft state */
 664         status = ddi_soft_state_zalloc(ahci_statep, instance);
 665         if (status != DDI_SUCCESS) {
 666                 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
 667                     instance);
 668                 goto err_out;
 669         }
 670 
 671         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
 672         ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
 673         ahci_ctlp->ahcictl_dip = dip;
 674 
 675         /* Initialize the cport/port mapping */
 676         for (i = 0; i < AHCI_MAX_PORTS; i++) {
 677                 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
 678                 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
 679         }
 680 
 681         attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
 682 
 683         /* Initialize FMA properties */
 684         ahci_fm_init(ahci_ctlp);
 685 
 686         attach_state |= AHCI_ATTACH_STATE_FMA;
 687 
 688         /*
 689          * Now map the AHCI base address; which includes global
 690          * registers and port control registers
 691          *
 692          * According to the spec, the AHCI Base Address is BAR5,
 693          * but BAR0-BAR4 are optional, so we need to check which
 694          * rnumber is used for BAR5.
 695          */
 696 
 697         /*
 698          * search through DDI "reg" property for the AHCI register set
 699          */
 700         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
 701             DDI_PROP_DONTPASS, "reg", (int **)&regs,
 702             (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
 703                 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
 704                     instance);
 705                 goto err_out;
 706         }
 707 
 708         /* AHCI Base Address is located at 0x24 offset */
 709         for (rnumber = 0; rnumber < regs_length; ++rnumber) {
 710                 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
 711                     == AHCI_PCI_RNUM)
 712                         break;
 713         }
 714 
 715         ddi_prop_free(regs);
 716 
 717         if (rnumber == regs_length) {
 718                 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
 719                     instance);
 720                 goto err_out;
 721         }
 722 
 723         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
 724 
 725         status = ddi_regs_map_setup(dip,
 726             rnumber,
 727             (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
 728             0,
 729             0,
 730             &accattr,
 731             &ahci_ctlp->ahcictl_ahci_acc_handle);
 732         if (status != DDI_SUCCESS) {
 733                 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
 734                     instance);
 735                 goto err_out;
 736         }
 737 
 738         attach_state |= AHCI_ATTACH_STATE_REG_MAP;
 739 
 740         /*
 741          * GHC.AE must be set to 1 before any other AHCI register
 742          * is accessed
 743          */
 744         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 745             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
 746         ghc_control |= AHCI_HBA_GHC_AE;
 747         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
 748             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
 749 
 750         /* Get the AHCI version information */
 751         ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 752             (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
 753 
 754         cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
 755             (ahci_version & 0xffff0000) >> 16,
 756             ((ahci_version & 0x0000ff00) >> 4 |
 757             (ahci_version & 0x000000ff)));
 758 
 759         /* We don't support controllers whose versions are lower than 1.0 */
 760         if (!(ahci_version & 0xffff0000)) {
 761                 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
 762                     "than version 1.0", instance);
 763                 goto err_out;
 764         }
 765 
 766         /* Get the HBA capabilities information */
 767         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 768             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
 769 
 770         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
 771             cap_status);
 772 
 773         /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
 774         if (ahci_version >= 0x00010200) {
 775                 uint32_t cap2_status;
 776 
 777                 /* Get the HBA capabilities extended information */
 778                 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 779                     (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
 780 
 781                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 782                     "hba capabilities extended = 0x%x", cap2_status);
 783         }
 784 
 785         if (cap_status & AHCI_HBA_CAP_EMS) {
 786                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS;
 787                 ahci_ctlp->ahcictl_em_loc =
 788                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 789                     (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp));
 790                 ahci_ctlp->ahcictl_em_ctl =
 791                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 792                     (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
 793         }
 794 
 795 #if AHCI_DEBUG
 796         /* Get the interface speed supported by the HBA */
 797         speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
 798         if (speed == 0x01) {
 799                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 800                     "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
 801         } else if (speed == 0x10) {
 802                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 803                     "hba interface speed support: Gen 2 (3 Gbps)", NULL);
 804         } else if (speed == 0x11) {
 805                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 806                     "hba interface speed support: Gen 3 (6 Gbps)", NULL);
 807         }
 808 #endif
 809 
 810         /* Get the number of command slots supported by the HBA */
 811         ahci_ctlp->ahcictl_num_cmd_slots =
 812             ((cap_status & AHCI_HBA_CAP_NCS) >>
 813             AHCI_HBA_CAP_NCS_SHIFT) + 1;
 814 
 815         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
 816             ahci_ctlp->ahcictl_num_cmd_slots);
 817 
 818         /* Get the bit map which indicates ports implemented by the HBA */
 819         ahci_ctlp->ahcictl_ports_implemented =
 820             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 821             (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
 822 
 823         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
 824             ahci_ctlp->ahcictl_ports_implemented);
 825 
 826         /* Max port number implemented */
 827         ahci_ctlp->ahcictl_num_ports =
 828             ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
 829 
 830         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
 831             (cap_status & AHCI_HBA_CAP_NP) + 1);
 832 
 833         /* Get the number of implemented ports by the HBA */
 834         ahci_ctlp->ahcictl_num_implemented_ports =
 835             ahci_get_num_implemented_ports(
 836             ahci_ctlp->ahcictl_ports_implemented);
 837 
 838         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 839             "hba number of implemented ports: %d",
 840             ahci_ctlp->ahcictl_num_implemented_ports);
 841 
 842         /* Check whether HBA supports 64bit DMA addressing */
 843         if (!(cap_status & AHCI_HBA_CAP_S64A)) {
 844                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
 845                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
 846                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 847                     "hba does not support 64-bit addressing", NULL);
 848         }
 849 
 850         /* Checking for the support of Port Multiplier */
 851         if (cap_status & AHCI_HBA_CAP_SPM) {
 852                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
 853                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 854                     "hba supports port multiplier (CBSS)", NULL);
 855 
 856                 /* Support FIS-based switching ? */
 857                 if (cap_status & AHCI_HBA_CAP_FBSS) {
 858                         ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
 859                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 860                             "hba supports FIS-based switching (FBSS)", NULL);
 861                 }
 862         }
 863 
 864         /* Checking for Support Command List Override */
 865         if (cap_status & AHCI_HBA_CAP_SCLO) {
 866                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
 867                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 868                     "hba supports command list override.", NULL);
 869         }
 870 
 871         /* Checking for Asynchronous Notification */
 872         if (cap_status & AHCI_HBA_CAP_SSNTF) {
 873                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
 874                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 875                     "hba supports asynchronous notification.", NULL);
 876         }
 877 
 878         if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
 879             != DDI_SUCCESS) {
 880                 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
 881                     instance);
 882                 goto err_out;
 883         }
 884 
 885         attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
 886 
 887         /*
 888          * Check the pci configuration space, and set caps. We also
 889          * handle the hardware defect in this function.
 890          *
 891          * For example, force ATI SB600 to use 32-bit dma addressing
 892          * since it doesn't support 64-bit dma though its CAP register
 893          * declares it support.
 894          */
 895         if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
 896                 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
 897                     instance);
 898                 goto err_out;
 899         }
 900 
 901         /*
 902          * Disable the whole controller interrupts before adding
 903          * interrupt handlers(s).
 904          */
 905         ahci_disable_all_intrs(ahci_ctlp);
 906 
 907         /* Get supported interrupt types */
 908         if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
 909                 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
 910                     instance);
 911                 goto err_out;
 912         }
 913 
 914         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 915             "ddi_intr_get_supported_types() returned: 0x%x",
 916             intr_types);
 917 
 918         if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
 919                 /*
 920                  * Try MSI first, but fall back to FIXED if failed
 921                  */
 922                 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
 923                     DDI_SUCCESS) {
 924                         ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
 925                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 926                             "Using MSI interrupt type", NULL);
 927                         goto intr_done;
 928                 }
 929 
 930                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 931                     "MSI registration failed, "
 932                     "trying FIXED interrupts", NULL);
 933         }
 934 
 935         if (intr_types & DDI_INTR_TYPE_FIXED) {
 936                 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
 937                     DDI_SUCCESS) {
 938                         ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
 939                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 940                             "Using FIXED interrupt type", NULL);
 941                         goto intr_done;
 942                 }
 943 
 944                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 945                     "FIXED interrupt registration failed", NULL);
 946         }
 947 
 948         cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
 949 
 950         goto err_out;
 951 
 952 intr_done:
 953 
 954         attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
 955 
 956         /* Initialize the controller mutex */
 957         mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
 958             (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
 959 
 960         attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
 961 
 962         if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
 963                 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
 964         } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
 965                 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
 966         }
 967 
 968         ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
 969             (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
 970             sizeof (ahci_prdt_item_t));
 971 
 972         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 973             "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
 974             " ahci_cmd_table_size is 0x%x",
 975             ahci_dma_prdt_number, ahci_cmd_table_size);
 976 
 977         if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
 978                 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
 979                     ahci_dma_prdt_number;
 980 
 981         ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
 982         ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
 983         ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
 984         ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
 985 
 986         /*
 987          * enable 64bit dma for data buffer for SB600 if
 988          * sb600_buf_64bit_dma_disable is B_FALSE
 989          */
 990         if ((ahci_buf_64bit_dma == B_FALSE) ||
 991             ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
 992             !(sb600_buf_64bit_dma_disable == B_FALSE &&
 993             ahci_ctlp->ahcictl_venid == 0x1002 &&
 994             ahci_ctlp->ahcictl_devid == 0x4380))) {
 995                 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
 996                     0xffffffffull;
 997         }
 998 
 999         /*
1000          * enable 64bit dma for command buffer for SB600/700/710/800
1001          * if sbxxx_commu_64bit_dma_disable is B_FALSE
1002          */
1003         if ((ahci_commu_64bit_dma == B_FALSE) ||
1004             ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
1005             !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
1006             ahci_ctlp->ahcictl_venid == 0x1002 &&
1007             (ahci_ctlp->ahcictl_devid == 0x4380 ||
1008             ahci_ctlp->ahcictl_devid == 0x4391)))) {
1009                 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
1010                     0xffffffffull;
1011                 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
1012                     0xffffffffull;
1013                 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
1014                     0xffffffffull;
1015         }
1016 
1017         /* Allocate the ports structure */
1018         status = ahci_alloc_ports_state(ahci_ctlp);
1019         if (status != AHCI_SUCCESS) {
1020                 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
1021                     instance);
1022                 goto err_out;
1023         }
1024 
1025         attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
1026 
1027         /*
1028          * Initialize the controller and ports.
1029          */
1030         status = ahci_initialize_controller(ahci_ctlp);
1031         if (status != AHCI_SUCCESS) {
1032                 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
1033                     instance);
1034                 goto err_out;
1035         }
1036 
1037         attach_state |= AHCI_ATTACH_STATE_HW_INIT;
1038 
1039         /* Start one thread to check packet timeouts */
1040         ahci_ctlp->ahcictl_timeout_id = timeout(
1041             (void (*)(void *))ahci_watchdog_handler,
1042             (caddr_t)ahci_ctlp, ahci_watchdog_tick);
1043 
1044         attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
1045 
1046         if (!ahci_em_init(ahci_ctlp)) {
1047                 cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure "
1048                     "services", instance);
1049                 goto err_out;
1050         }
1051         attach_state |= AHCI_ATTACH_STATE_ENCLOSURE;
1052 
1053         if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
1054                 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
1055                     instance);
1056                 goto err_out;
1057         }
1058 
1059         /* Check all handles at the end of the attach operation. */
1060         if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
1061                 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
1062                     instance);
1063                 goto err_out;
1064         }
1065 
1066         ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
1067 
1068         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
1069 
1070         return (DDI_SUCCESS);
1071 
1072 err_out:
1073         /* FMA message */
1074         ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
1075         ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
1076 
1077         if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) {
1078                 ahci_em_fini(ahci_ctlp);
1079         }
1080 
1081         if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
1082                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1083                 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1084                 ahci_ctlp->ahcictl_timeout_id = 0;
1085                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1086         }
1087 
1088         if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1089                 ahci_uninitialize_controller(ahci_ctlp);
1090         }
1091 
1092         if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1093                 ahci_dealloc_ports_state(ahci_ctlp);
1094         }
1095 
1096         if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1097                 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1098         }
1099 
1100         if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1101                 ahci_rem_intrs(ahci_ctlp);
1102         }
1103 
1104         if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1105                 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1106         }
1107 
1108         if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1109                 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1110         }
1111 
1112         if (attach_state & AHCI_ATTACH_STATE_FMA) {
1113                 ahci_fm_fini(ahci_ctlp);
1114         }
1115 
1116         if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1117                 ddi_soft_state_free(ahci_statep, instance);
1118         }
1119 
1120         return (DDI_FAILURE);
1121 }
1122 
1123 /*
1124  * The detach entry point for dev_ops.
1125  */
1126 static int
1127 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1128 {
1129         ahci_ctl_t *ahci_ctlp;
1130         int instance;
1131         int ret;
1132 
1133         instance = ddi_get_instance(dip);
1134         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1135 
1136         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1137 
1138         switch (cmd) {
1139         case DDI_DETACH:
1140 
1141                 /* disable the interrupts for an uninterrupted detach */
1142                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1143                 ahci_disable_all_intrs(ahci_ctlp);
1144                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1145 
1146                 /* unregister from the sata framework. */
1147                 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1148                 if (ret != AHCI_SUCCESS) {
1149                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1150                         ahci_enable_all_intrs(ahci_ctlp);
1151                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1152                         return (DDI_FAILURE);
1153                 }
1154 
1155                 ahci_em_fini(ahci_ctlp);
1156 
1157                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1158 
1159                 /* stop the watchdog handler */
1160                 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1161                 ahci_ctlp->ahcictl_timeout_id = 0;
1162 
1163                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1164 
1165                 /* uninitialize the controller */
1166                 ahci_uninitialize_controller(ahci_ctlp);
1167 
1168                 /* remove the interrupts */
1169                 ahci_rem_intrs(ahci_ctlp);
1170 
1171                 /* deallocate the ports structures */
1172                 ahci_dealloc_ports_state(ahci_ctlp);
1173 
1174                 /* destroy mutex */
1175                 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1176 
1177                 /* teardown the pci config */
1178                 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1179 
1180                 /* remove the reg maps. */
1181                 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1182 
1183                 /* release fma resource */
1184                 ahci_fm_fini(ahci_ctlp);
1185 
1186                 /* free the soft state. */
1187                 ddi_soft_state_free(ahci_statep, instance);
1188 
1189                 return (DDI_SUCCESS);
1190 
1191         case DDI_SUSPEND:
1192 
1193                 /*
1194                  * The steps associated with suspension must include putting
1195                  * the underlying device into a quiescent state so that it
1196                  * will not generate interrupts or modify or access memory.
1197                  */
1198                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1199                 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1200                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1201                         return (DDI_SUCCESS);
1202                 }
1203 
1204                 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1205 
1206                 ahci_em_suspend(ahci_ctlp);
1207 
1208                 /* stop the watchdog handler */
1209                 if (ahci_ctlp->ahcictl_timeout_id) {
1210                         (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1211                         ahci_ctlp->ahcictl_timeout_id = 0;
1212                 }
1213 
1214                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1215 
1216                 /*
1217                  * drain the taskq
1218                  */
1219                 ahci_drain_ports_taskq(ahci_ctlp);
1220 
1221                 /*
1222                  * Disable the interrupts and stop all the ports.
1223                  */
1224                 ahci_uninitialize_controller(ahci_ctlp);
1225 
1226                 return (DDI_SUCCESS);
1227 
1228         default:
1229                 return (DDI_FAILURE);
1230         }
1231 }
1232 
1233 /*
1234  * The info entry point for dev_ops.
1235  *
1236  */
1237 static int
1238 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1239     void *arg, void **result)
1240 {
1241 #ifndef __lock_lint
1242         _NOTE(ARGUNUSED(dip))
1243 #endif /* __lock_lint */
1244 
1245         ahci_ctl_t *ahci_ctlp;
1246         int instance;
1247         dev_t dev;
1248 
1249         dev = (dev_t)arg;
1250         instance = getminor(dev);
1251 
1252         switch (infocmd) {
1253                 case DDI_INFO_DEVT2DEVINFO:
1254                         ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
1255                         if (ahci_ctlp != NULL) {
1256                                 *result = ahci_ctlp->ahcictl_dip;
1257                                 return (DDI_SUCCESS);
1258                         } else {
1259                                 *result = NULL;
1260                                 return (DDI_FAILURE);
1261                         }
1262                 case DDI_INFO_DEVT2INSTANCE:
1263                         *(int *)result = instance;
1264                         break;
1265                 default:
1266                         break;
1267         }
1268 
1269         return (DDI_SUCCESS);
1270 }
1271 
1272 /*
1273  * Registers the ahci with sata framework.
1274  */
1275 static int
1276 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1277 {
1278         struct  sata_hba_tran   *sata_hba_tran;
1279 
1280         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1281             "ahci_register_sata_hba_tran enter", NULL);
1282 
1283         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1284 
1285         /* Allocate memory for the sata_hba_tran  */
1286         sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1287 
1288         sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1289         sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1290         sata_hba_tran->sata_tran_hba_dma_attr =
1291             &ahci_ctlp->ahcictl_buffer_dma_attr;
1292 
1293         /* Report the number of implemented ports */
1294         sata_hba_tran->sata_tran_hba_num_cports =
1295             ahci_ctlp->ahcictl_num_implemented_ports;
1296 
1297         /* Support ATAPI device */
1298         sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1299 
1300         /* Get the data transfer capability for PIO command by the HBA */
1301         if (cap_status & AHCI_HBA_CAP_PMD) {
1302                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1303                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1304                     "DRQ block data transfer for PIO command protocol", NULL);
1305         }
1306 
1307         /*
1308          * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1309          * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1310          * QUEUED (EXT), and SERVICE commands). Queued operations are
1311          * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1312          * QUEUED commands when the HBA and device support native command
1313          * queuing(NCQ).
1314          *
1315          * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1316          * CAP register of the HBA indicates NCQ is supported.
1317          *
1318          * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1319          * set because the previous register content of PxCI can be re-written
1320          * in the register write.
1321          */
1322         if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1323             !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1324                 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1325                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1326                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1327                     "Command Queuing", NULL);
1328         }
1329 
1330         /* Support port multiplier? */
1331         if (cap_status & AHCI_HBA_CAP_SPM) {
1332                 sata_hba_tran->sata_tran_hba_features_support |=
1333                     SATA_CTLF_PORT_MULTIPLIER;
1334 
1335                 /* Support FIS-based switching for port multiplier? */
1336                 if (cap_status & AHCI_HBA_CAP_FBSS) {
1337                         sata_hba_tran->sata_tran_hba_features_support |=
1338                             SATA_CTLF_PMULT_FBS;
1339                 }
1340         }
1341 
1342         /* Report the number of command slots */
1343         sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1344 
1345         sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1346         sata_hba_tran->sata_tran_start = ahci_tran_start;
1347         sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1348         sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1349         sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1350 #ifdef __lock_lint
1351         sata_hba_tran->sata_tran_selftest = ahci_selftest;
1352 #endif
1353         /*
1354          * When SATA framework adds support for pwrmgt the
1355          * pwrmgt_ops needs to be updated
1356          */
1357         sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1358         sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl;
1359 
1360         ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1361 
1362         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1363 
1364         /* Attach it to SATA framework */
1365         if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1366             != DDI_SUCCESS) {
1367                 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1368                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1369                 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1370                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1371                 return (AHCI_FAILURE);
1372         }
1373 
1374         return (AHCI_SUCCESS);
1375 }
1376 
1377 /*
1378  * Unregisters the ahci with sata framework.
1379  */
1380 static int
1381 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1382 {
1383         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1384             "ahci_unregister_sata_hba_tran enter", NULL);
1385 
1386         /* Detach from the SATA framework. */
1387         if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1388             DDI_SUCCESS) {
1389                 return (AHCI_FAILURE);
1390         }
1391 
1392         /* Deallocate sata_hba_tran. */
1393         kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1394             sizeof (sata_hba_tran_t));
1395 
1396         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1397         ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1398         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1399 
1400         return (AHCI_SUCCESS);
1401 }
1402 
1403 #define SET_PORTSTR(str, addrp)                                         \
1404         if (AHCI_ADDR_IS_PORT(addrp))                                   \
1405                 (void) sprintf((str), "%d", (addrp)->aa_port);               \
1406         else if (AHCI_ADDR_IS_PMULT(addrp))                             \
1407                 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port);       \
1408         else                                                            \
1409                 (void) sprintf((str), "%d:%d", (addrp)->aa_port,     \
1410                     (addrp)->aa_pmport);
1411 
1412 /*
1413  * ahci_tran_probe_port is called by SATA framework. It returns port state,
1414  * port status registers and an attached device type via sata_device
1415  * structure.
1416  *
1417  * We return the cached information from a previous hardware probe. The
1418  * actual hardware probing itself was done either from within
1419  * ahci_initialize_controller() during the driver attach or from a phy
1420  * ready change interrupt handler.
1421  */
1422 static int
1423 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1424 {
1425         ahci_ctl_t *ahci_ctlp;
1426         ahci_port_t *ahci_portp;
1427         ahci_addr_t addr, pmult_addr;
1428         uint8_t cport = sd->satadev_addr.cport;
1429         char portstr[10];
1430         uint8_t device_type;
1431         uint32_t port_state;
1432         uint8_t port;
1433         int rval = SATA_SUCCESS, rval_init;
1434 
1435         ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1436         port = ahci_ctlp->ahcictl_cport_to_port[cport];
1437 
1438         ahci_portp = ahci_ctlp->ahcictl_ports[port];
1439 
1440         mutex_enter(&ahci_portp->ahciport_mutex);
1441 
1442         ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1443         ASSERT(AHCI_ADDR_IS_VALID(&addr));
1444         SET_PORTSTR(portstr, &addr);
1445 
1446         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1447             "ahci_tran_probe_port enter: port %s", portstr);
1448 
1449         if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1450             (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1451             ahci_portp->ahciport_pmult_info == NULL)) {
1452                 /* port mutliplier is removed. */
1453                 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1454                     "ahci_tran_probe_port: "
1455                     "pmult is removed from port %s", portstr);
1456                 mutex_exit(&ahci_portp->ahciport_mutex);
1457                 return (SATA_FAILURE);
1458         }
1459 
1460         /*
1461          * The sata_device may refer to
1462          * 1. A controller port.
1463          *    A controller port should be ready here.
1464          * 2. A port multiplier.
1465          *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1466          *    it and register the port multiplier to the framework.
1467          *    SATA_ADDR_PMULT - check the status of all its device ports.
1468          * 3. A port multiplier port.
1469          *    If it has not been initialized, initialized it.
1470          *
1471          * A port multiplier or a port multiplier port may require some
1472          * initialization because we cannot do these time-consuming jobs in an
1473          * interrupt context.
1474          */
1475         if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1476                 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1477                 /* Initialize registers on a port multiplier */
1478                 rval_init = ahci_initialize_pmult(ahci_ctlp,
1479                     ahci_portp, &pmult_addr, sd);
1480                 if (rval_init != AHCI_SUCCESS) {
1481                         AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1482                             "ahci_tran_probe_port: "
1483                             "pmult initialization failed.", NULL);
1484                         mutex_exit(&ahci_portp->ahciport_mutex);
1485                         return (SATA_FAILURE);
1486                 }
1487         } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1488                 /* Check pmports hotplug events */
1489                 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1490         } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1491             SATA_ADDR_DPMPORT)) {
1492                 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1493                     &addr, sd) != AHCI_SUCCESS) {
1494                         rval = SATA_FAILURE;
1495                         goto out;
1496                 }
1497         }
1498 
1499         /* Update port state and device type */
1500         port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1501 
1502         switch (port_state) {
1503 
1504         case SATA_PSTATE_FAILED:
1505                 sd->satadev_state = SATA_PSTATE_FAILED;
1506                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1507                     "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1508                 goto out;
1509 
1510         case SATA_PSTATE_SHUTDOWN:
1511                 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1512                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1513                     "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1514                 goto out;
1515 
1516         case SATA_PSTATE_PWROFF:
1517                 sd->satadev_state = SATA_PSTATE_PWROFF;
1518                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1519                     "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1520                 goto out;
1521 
1522         case SATA_PSTATE_PWRON:
1523                 sd->satadev_state = SATA_PSTATE_PWRON;
1524                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1525                     "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1526                 break;
1527 
1528         default:
1529                 sd->satadev_state = port_state;
1530                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1531                     "ahci_tran_probe_port: port %s PORT NORMAL %x",
1532                     portstr, port_state);
1533                 break;
1534         }
1535 
1536         device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1537 
1538         switch (device_type) {
1539 
1540         case SATA_DTYPE_ATADISK:
1541                 sd->satadev_type = SATA_DTYPE_ATADISK;
1542                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1543                     "ahci_tran_probe_port: port %s DISK found", portstr);
1544                 break;
1545 
1546         case SATA_DTYPE_ATAPI:
1547                 /*
1548                  * HBA driver only knows it's an ATAPI device, and don't know
1549                  * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1550                  * type need to be determined by checking IDENTIFY PACKET
1551                  * DEVICE data
1552                  */
1553                 sd->satadev_type = SATA_DTYPE_ATAPI;
1554                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1555                     "ahci_tran_probe_port: port %s ATAPI found", portstr);
1556                 break;
1557 
1558         case SATA_DTYPE_PMULT:
1559                 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1560                 sd->satadev_type = SATA_DTYPE_PMULT;
1561 
1562                 /* Update the number of pmports. */
1563                 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1564                 sd->satadev_add_info = ahci_portp->
1565                     ahciport_pmult_info->ahcipmi_num_dev_ports;
1566 
1567                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1568                     "ahci_tran_probe_port: port %s Port Multiplier found",
1569                     portstr);
1570                 break;
1571 
1572         case SATA_DTYPE_UNKNOWN:
1573                 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1574                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1575                     "ahci_tran_probe_port: port %s Unknown device found",
1576                     portstr);
1577                 break;
1578 
1579         default:
1580                 /* we don't support any other device types */
1581                 sd->satadev_type = SATA_DTYPE_NONE;
1582                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1583                     "ahci_tran_probe_port: port %s No device found", portstr);
1584                 break;
1585         }
1586 
1587 out:
1588         /* Register update only fails while probing a pmult/pmport */
1589         if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1590                 ahci_update_sata_registers(ahci_ctlp, port, sd);
1591         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1592                 if (port_state & SATA_STATE_READY)
1593                         if (ahci_update_pmult_pscr(ahci_ctlp,
1594                             &addr, sd) != AHCI_SUCCESS)
1595                                 rval = SATA_FAILURE;
1596         }
1597 
1598         /* Check handles for the sata registers access */
1599         if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1600             (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1601                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1602                     DDI_SERVICE_UNAFFECTED);
1603                 rval = SATA_FAILURE;
1604         }
1605 
1606         mutex_exit(&ahci_portp->ahciport_mutex);
1607         return (rval);
1608 }
1609 
1610 /*
1611  * There are four operation modes in sata framework:
1612  * SATA_OPMODE_INTERRUPTS
1613  * SATA_OPMODE_POLLING
1614  * SATA_OPMODE_ASYNCH
1615  * SATA_OPMODE_SYNCH
1616  *
1617  * Their combined meanings as following:
1618  *
1619  * SATA_OPMODE_SYNCH
1620  * The command has to be completed before sata_tran_start functions returns.
1621  * Either interrupts or polling could be used - it's up to the driver.
1622  * Mode used currently for internal, sata-module initiated operations.
1623  *
1624  * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1625  * It is the same as the one above.
1626  *
1627  * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1628  * The command has to be completed before sata_tran_start function returns.
1629  * No interrupt used, polling only. This should be the mode used for scsi
1630  * packets with FLAG_NOINTR.
1631  *
1632  * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1633  * The command may be queued (callback function specified). Interrupts could
1634  * be used. It's normal operation mode.
1635  */
1636 /*
1637  * Called by sata framework to transport a sata packet down stream.
1638  */
1639 static int
1640 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1641 {
1642         ahci_ctl_t *ahci_ctlp;
1643         ahci_port_t *ahci_portp;
1644         ahci_addr_t addr;
1645         uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1646         uint8_t port;
1647         char portstr[10];
1648 
1649         ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1650         port = ahci_ctlp->ahcictl_cport_to_port[cport];
1651 
1652         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1653             "ahci_tran_start enter: cport %d satapkt 0x%p",
1654             cport, (void *)spkt);
1655 
1656         ahci_portp = ahci_ctlp->ahcictl_ports[port];
1657 
1658         mutex_enter(&ahci_portp->ahciport_mutex);
1659         ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1660         SET_PORTSTR(portstr, &addr);
1661 
1662         /* Sanity check */
1663         if (AHCI_ADDR_IS_PMPORT(&addr)) {
1664                 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1665                     ahci_portp->ahciport_pmult_info == NULL) {
1666 
1667                         spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1668                         spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1669                         spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1670                         ahci_update_sata_registers(ahci_ctlp, port,
1671                             &spkt->satapkt_device);
1672                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1673                             "ahci_tran_start returning PORT_ERROR while "
1674                             "pmult removed: port: %s", portstr);
1675                         mutex_exit(&ahci_portp->ahciport_mutex);
1676                         return (SATA_TRAN_PORT_ERROR);
1677                 }
1678 
1679                 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1680                     SATA_STATE_READY)) {
1681                         if (!ddi_in_panic() ||
1682                             ahci_initialize_pmport(ahci_ctlp,
1683                             ahci_portp, &addr) != AHCI_SUCCESS) {
1684                                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1685                                 spkt->satapkt_device.satadev_type =
1686                                     AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1687                                 spkt->satapkt_device.satadev_state =
1688                                     AHCIPORT_GET_STATE(ahci_portp, &addr);
1689                                 ahci_update_sata_registers(ahci_ctlp, port,
1690                                     &spkt->satapkt_device);
1691                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1692                                     "ahci_tran_start returning PORT_ERROR "
1693                                     "while sub-link is not initialized "
1694                                     "at port: %s", portstr);
1695                                 mutex_exit(&ahci_portp->ahciport_mutex);
1696                                 return (SATA_TRAN_PORT_ERROR);
1697                         }
1698                 }
1699         }
1700 
1701         if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1702             AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1703             AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1704                 /*
1705                  * In case the target driver would send the packet before
1706                  * sata framework can have the opportunity to process those
1707                  * event reports.
1708                  */
1709                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1710                 spkt->satapkt_device.satadev_state =
1711                     ahci_portp->ahciport_port_state;
1712                 ahci_update_sata_registers(ahci_ctlp, port,
1713                     &spkt->satapkt_device);
1714                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1715                     "ahci_tran_start returning PORT_ERROR while "
1716                     "port in FAILED/SHUTDOWN/PWROFF state: "
1717                     "port: %s", portstr);
1718                 mutex_exit(&ahci_portp->ahciport_mutex);
1719                 return (SATA_TRAN_PORT_ERROR);
1720         }
1721 
1722         if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1723                 /*
1724                  * ahci_intr_phyrdy_change() may have rendered it to
1725                  * SATA_DTYPE_NONE.
1726                  */
1727                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1728                 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1729                 spkt->satapkt_device.satadev_state =
1730                     ahci_portp->ahciport_port_state;
1731                 ahci_update_sata_registers(ahci_ctlp, port,
1732                     &spkt->satapkt_device);
1733                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1734                     "ahci_tran_start returning PORT_ERROR while "
1735                     "no device attached: port: %s", portstr);
1736                 mutex_exit(&ahci_portp->ahciport_mutex);
1737                 return (SATA_TRAN_PORT_ERROR);
1738         }
1739 
1740         /* R/W PMULT command will occupy the whole HBA port */
1741         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1742                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1743                     "ahci_tran_start returning BUSY while "
1744                     "executing READ/WRITE PORT-MULT command: "
1745                     "port: %s", portstr);
1746                 spkt->satapkt_reason = SATA_PKT_BUSY;
1747                 mutex_exit(&ahci_portp->ahciport_mutex);
1748                 return (SATA_TRAN_BUSY);
1749         }
1750 
1751         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1752                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1753                     "ahci_tran_start returning BUSY while "
1754                     "hot-plug in progress: port: %s", portstr);
1755                 spkt->satapkt_reason = SATA_PKT_BUSY;
1756                 mutex_exit(&ahci_portp->ahciport_mutex);
1757                 return (SATA_TRAN_BUSY);
1758         }
1759 
1760         /*
1761          * SATA HBA driver should remember that a device was reset and it
1762          * is supposed to reject any packets which do not specify either
1763          * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1764          *
1765          * This is to prevent a race condition when a device was arbitrarily
1766          * reset by the HBA driver (and lost it's setting) and a target
1767          * driver sending some commands to a device before the sata framework
1768          * has a chance to restore the device setting (such as cache enable/
1769          * disable or other resettable stuff).
1770          */
1771         /*
1772          * It is unnecessary to use specific flags to indicate
1773          * reset_in_progress for a pmport. While mopping, all command will be
1774          * mopped so that the entire HBA port is being dealt as a single
1775          * object.
1776          */
1777         if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1778                 ahci_portp->ahciport_reset_in_progress = 0;
1779                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1780                     "ahci_tran_start [CLEAR] the "
1781                     "reset_in_progress for port: %d", port);
1782         }
1783 
1784         if (ahci_portp->ahciport_reset_in_progress &&
1785             ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1786             ! ddi_in_panic()) {
1787                 spkt->satapkt_reason = SATA_PKT_BUSY;
1788                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1789                     "ahci_tran_start returning BUSY while "
1790                     "reset in progress: port: %d", port);
1791                 mutex_exit(&ahci_portp->ahciport_mutex);
1792                 return (SATA_TRAN_BUSY);
1793         }
1794 
1795 #ifdef AHCI_DEBUG
1796         if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1797                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1798                     "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1799                     spkt, port);
1800         }
1801 #endif
1802 
1803         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1804                 spkt->satapkt_reason = SATA_PKT_BUSY;
1805                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1806                     "ahci_tran_start returning BUSY while "
1807                     "mopping in progress: port: %d", port);
1808                 mutex_exit(&ahci_portp->ahciport_mutex);
1809                 return (SATA_TRAN_BUSY);
1810         }
1811 
1812         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1813                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1814                     DDI_SERVICE_UNAFFECTED);
1815                 mutex_exit(&ahci_portp->ahciport_mutex);
1816                 return (SATA_TRAN_BUSY);
1817         }
1818 
1819         if (spkt->satapkt_op_mode &
1820             (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1821                 /*
1822                  * If a SYNC command to be executed in interrupt context,
1823                  * bounce it back to sata module.
1824                  */
1825                 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1826                     servicing_interrupt()) {
1827                         spkt->satapkt_reason = SATA_PKT_BUSY;
1828                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1829                             "ahci_tran_start returning BUSY while "
1830                             "sending SYNC mode under interrupt context: "
1831                             "port : %d", port);
1832                         mutex_exit(&ahci_portp->ahciport_mutex);
1833                         return (SATA_TRAN_BUSY);
1834                 }
1835 
1836                 /* We need to do the sync start now */
1837                 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1838                     spkt) == AHCI_FAILURE) {
1839                         goto fail_out;
1840                 }
1841         } else {
1842                 /* Async start, using interrupt */
1843                 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1844                     == AHCI_FAILURE) {
1845                         spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1846                         goto fail_out;
1847                 }
1848         }
1849 
1850         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1851             "sata tran accepted: port %s", portstr);
1852 
1853         mutex_exit(&ahci_portp->ahciport_mutex);
1854         return (SATA_TRAN_ACCEPTED);
1855 
1856 fail_out:
1857         /*
1858          * Failed to deliver packet to the controller.
1859          * Check if it's caused by invalid handles.
1860          */
1861         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1862             ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1863                 spkt->satapkt_device.satadev_type =
1864                     AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1865                 spkt->satapkt_device.satadev_state =
1866                     AHCIPORT_GET_STATE(ahci_portp, &addr);
1867                 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1868                 mutex_exit(&ahci_portp->ahciport_mutex);
1869                 return (SATA_TRAN_PORT_ERROR);
1870         }
1871 
1872         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1873             "return QUEUE_FULL: port %d", port);
1874         mutex_exit(&ahci_portp->ahciport_mutex);
1875         return (SATA_TRAN_QUEUE_FULL);
1876 }
1877 
1878 /*
1879  * SATA_OPMODE_SYNCH flag is set
1880  *
1881  * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1882  * without interrupt, otherwise we can still use the interrupt.
1883  */
1884 static int
1885 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1886     ahci_addr_t *addrp, sata_pkt_t *spkt)
1887 {
1888         int pkt_timeout_ticks;
1889         uint32_t timeout_tags;
1890         int rval;
1891         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1892         uint8_t port = addrp->aa_port;
1893 
1894         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1895 
1896         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1897             "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1898 
1899         if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1900                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1901                 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1902                     addrp, spkt)) == AHCI_FAILURE) {
1903                         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1904                         return (rval);
1905                 }
1906 
1907                 pkt_timeout_ticks =
1908                     drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1909 
1910                 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1911                         /* Simulate the interrupt */
1912                         mutex_exit(&ahci_portp->ahciport_mutex);
1913                         ahci_port_intr(ahci_ctlp, ahci_portp, port);
1914                         mutex_enter(&ahci_portp->ahciport_mutex);
1915 
1916                         if (spkt->satapkt_reason != SATA_PKT_BUSY)
1917                                 break;
1918 
1919                         mutex_exit(&ahci_portp->ahciport_mutex);
1920                         drv_usecwait(AHCI_1MS_USECS);
1921                         mutex_enter(&ahci_portp->ahciport_mutex);
1922 
1923                         pkt_timeout_ticks -= AHCI_1MS_TICKS;
1924                         if (pkt_timeout_ticks < 0) {
1925                                 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1926                                     "port %d satapkt 0x%p timed out\n",
1927                                     instance, port, (void *)spkt);
1928                                 timeout_tags = (0x1 << rval);
1929                                 mutex_exit(&ahci_portp->ahciport_mutex);
1930                                 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1931                                     port, timeout_tags);
1932                                 mutex_enter(&ahci_portp->ahciport_mutex);
1933                         }
1934                 }
1935 
1936                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1937                 return (AHCI_SUCCESS);
1938 
1939         } else {
1940                 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1941                     addrp, spkt)) == AHCI_FAILURE)
1942                         return (rval);
1943 
1944 #if AHCI_DEBUG
1945                 /*
1946                  * Note that the driver always uses the slot 0 to deliver
1947                  * REQUEST SENSE or READ LOG EXT command
1948                  */
1949                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1950                         ASSERT(rval == 0);
1951 #endif
1952 
1953                 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1954                         cv_wait(&ahci_portp->ahciport_cv,
1955                             &ahci_portp->ahciport_mutex);
1956 
1957                 return (AHCI_SUCCESS);
1958         }
1959 }
1960 
1961 /*
1962  * Searches for and claims a free command slot.
1963  *
1964  * Returns value:
1965  *
1966  * AHCI_FAILURE returned only if
1967  *      1. No empty slot left
1968  *      2. Non-queued command requested while queued command(s) is outstanding
1969  *      3. Queued command requested while non-queued command(s) is outstanding
1970  *      4. HBA doesn't support multiple-use of command list while already a
1971  *         non-queued command is oustanding
1972  *      5. Queued command requested while some queued command(s) has been
1973  *         outstanding on a different port multiplier port. (AHCI spec 1.2,
1974  *         9.1.2)
1975  *
1976  * claimed slot number returned if succeeded
1977  *
1978  * NOTE: it will always return slot 0 for following commands to simplify the
1979  * algorithm.
1980  *      1. REQUEST SENSE or READ LOG EXT command during error recovery process
1981  *      2. READ/WRITE PORTMULT command
1982  */
1983 static int
1984 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1985     ahci_addr_t *addrp, int command_type)
1986 {
1987         uint32_t port_cmd_issue;
1988         uint32_t free_slots;
1989         int slot;
1990 
1991         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1992 
1993         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1994             "ahciport_pending_tags = 0x%x "
1995             "ahciport_pending_ncq_tags = 0x%x",
1996             ahci_portp->ahciport_pending_tags,
1997             ahci_portp->ahciport_pending_ncq_tags);
1998 
1999         /*
2000          * According to the AHCI spec, system software is responsible to
2001          * ensure that queued and non-queued commands are not mixed in
2002          * the command list.
2003          */
2004         if (command_type == AHCI_NON_NCQ_CMD) {
2005                 /* Non-NCQ command request */
2006                 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2007                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2008                             "ahci_claim_free_slot: there is still pending "
2009                             "queued command(s) in the command list, "
2010                             "so no available slot for the non-queued "
2011                             "command", NULL);
2012                         return (AHCI_FAILURE);
2013                 }
2014                 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
2015                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2016                             "ahci_claim_free_slot: there is still pending "
2017                             "read/write port-mult command(s) in command list, "
2018                             "so no available slot for the non-queued command",
2019                             NULL);
2020                         return (AHCI_FAILURE);
2021                 }
2022                 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
2023                     NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2024                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2025                             "ahci_claim_free_slot: HBA cannot support multiple-"
2026                             "use of the command list for non-queued commands",
2027                             NULL);
2028                         return (AHCI_FAILURE);
2029                 }
2030                 free_slots = (~ahci_portp->ahciport_pending_tags) &
2031                     AHCI_SLOT_MASK(ahci_ctlp);
2032         } else if (command_type == AHCI_NCQ_CMD) {
2033                 /* NCQ command request */
2034                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2035                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2036                             "ahci_claim_free_slot: there is still pending "
2037                             "non-queued command(s) in the command list, "
2038                             "so no available slot for the queued command",
2039                             NULL);
2040                         return (AHCI_FAILURE);
2041                 }
2042 
2043                 /*
2044                  * NCQ commands cannot be sent to different port multiplier
2045                  * ports in Command-Based Switching mode
2046                  */
2047                 /*
2048                  * NOTE: In Command-Based Switching mode, AHCI controller
2049                  * usually reports a 'Handshake Error' when multiple NCQ
2050                  * commands are outstanding simultaneously.
2051                  */
2052                 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
2053                         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2054                         if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
2055                             NCQ_CMD_IN_PROGRESS(ahci_portp) &&
2056                             AHCIPORT_NCQ_PMPORT(ahci_portp) !=
2057                             addrp->aa_pmport) {
2058                                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2059                                     "ahci_claim_free_slot: there is still "
2060                                     "pending queued command(s) in the "
2061                                     "command list for another Port Multiplier "
2062                                     "port, so no available slot.", NULL);
2063                                 return (AHCI_FAILURE);
2064                         }
2065                 }
2066 
2067                 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
2068                     AHCI_NCQ_SLOT_MASK(ahci_portp);
2069         } else if (command_type == AHCI_ERR_RETRI_CMD) {
2070                 /* Error retrieval command request */
2071                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2072                     "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
2073                     "SENSE or READ LOG EXT command", NULL);
2074                 slot = 0;
2075                 goto out;
2076         } else if (command_type == AHCI_RDWR_PMULT_CMD) {
2077                 /*
2078                  * An extra check on PxCI. Sometimes PxCI bits may not be
2079                  * cleared during hot-plug or error recovery process.
2080                  */
2081                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2082                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
2083 
2084                 if (port_cmd_issue != 0) {
2085                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2086                             "ahci_claim_free_slot: there is still pending "
2087                             "command(s) in command list (0x%x/0x%x, PxCI %x),"
2088                             "so no available slot for R/W PMULT command.",
2089                             NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
2090                             NCQ_CMD_IN_PROGRESS(ahci_portp),
2091                             port_cmd_issue);
2092                         return (AHCI_FAILURE);
2093                 }
2094 
2095                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2096                     "ahci_claim_free_slot: slot 0 is allocated for "
2097                     "READ/WRITE PORTMULT command", NULL);
2098                 slot = 0;
2099                 goto out;
2100         }
2101 
2102         slot = ddi_ffs(free_slots) - 1;
2103         if (slot == -1) {
2104                 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2105                     "ahci_claim_free_slot: no empty slots", NULL);
2106                 return (AHCI_FAILURE);
2107         }
2108 
2109         /*
2110          * According to the AHCI spec, to allow a simple mechanism for the
2111          * HBA to map command list slots to queue entries, software must
2112          * match the tag number it uses to the slot it is placing the command
2113          * in. For example, if a queued command is placed in slot 5, the tag
2114          * for that command must be 5.
2115          */
2116         if (command_type == AHCI_NCQ_CMD) {
2117                 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2118                 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2119                         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2120                         AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2121                 }
2122         }
2123 
2124         ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2125 
2126 out:
2127         AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2128             "ahci_claim_free_slot: found slot: 0x%x", slot);
2129 
2130         return (slot);
2131 }
2132 
2133 /*
2134  * Builds the Command Table for the sata packet and delivers it to controller.
2135  *
2136  * Returns:
2137  *      slot number if we can obtain a slot successfully
2138  *      otherwise, return AHCI_FAILURE
2139  */
2140 static int
2141 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2142     ahci_addr_t *addrp, sata_pkt_t *spkt)
2143 {
2144         int cmd_slot;
2145         sata_cmd_t *scmd;
2146         ahci_fis_h2d_register_t *h2d_register_fisp;
2147         ahci_cmd_table_t *cmd_table;
2148         ahci_cmd_header_t *cmd_header;
2149         int ncookies;
2150         int i;
2151         int command_type = AHCI_NON_NCQ_CMD;
2152         int ncq_qdepth;
2153         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2154         uint8_t port, pmport;
2155 #if AHCI_DEBUG
2156         uint32_t *ptr;
2157         uint8_t *ptr2;
2158 #endif
2159 
2160         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2161 
2162         port = addrp->aa_port;
2163         pmport = addrp->aa_pmport;
2164 
2165         spkt->satapkt_reason = SATA_PKT_BUSY;
2166 
2167         scmd = &spkt->satapkt_cmd;
2168 
2169         /* Check if the command is a NCQ command */
2170         if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2171             scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2172                 command_type = AHCI_NCQ_CMD;
2173 
2174                 /*
2175                  * When NCQ is support, system software must determine the
2176                  * maximum tag allowed by the device and the HBA, and it
2177                  * must use a value not beyond of the lower bound of the two.
2178                  *
2179                  * Sata module is going to calculate the qdepth and send
2180                  * down to HBA driver via sata_cmd.
2181                  */
2182                 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2183 
2184                 /*
2185                  * At the moment, the driver doesn't support the dynamic
2186                  * setting of the maximum ncq depth, and the value can be
2187                  * set either during the attach or after hot-plug insertion.
2188                  */
2189                 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2190                         ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2191                         AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2192                             "ahci_deliver_satapkt: port %d the max tags for "
2193                             "NCQ command is %d", port, ncq_qdepth);
2194                 } else {
2195                         if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2196                                 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2197                                     " port %d the max tag for NCQ command is "
2198                                     "requested to change from %d to %d, at the"
2199                                     " moment the driver doesn't support the "
2200                                     "dynamic change so it's going to "
2201                                     "still use the previous tag value",
2202                                     instance, port,
2203                                     ahci_portp->ahciport_max_ncq_tags,
2204                                     ncq_qdepth);
2205                         }
2206                 }
2207         }
2208 
2209         /* Check if the command is an error retrieval command */
2210         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2211                 command_type = AHCI_ERR_RETRI_CMD;
2212 
2213         /* Check if the command is an read/write pmult command */
2214         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2215                 command_type = AHCI_RDWR_PMULT_CMD;
2216 
2217         /* Check if there is an empty command slot */
2218         cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2219             addrp, command_type);
2220         if (cmd_slot == AHCI_FAILURE) {
2221                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2222                 return (AHCI_FAILURE);
2223         }
2224 
2225         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2226             "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2227             "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2228             cmd_slot, port, (void *)spkt);
2229 
2230         cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2231         bzero((void *)cmd_table, ahci_cmd_table_size);
2232 
2233         /* For data transfer operations, it is the H2D Register FIS */
2234         h2d_register_fisp =
2235             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2236 
2237         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2238 
2239         /*
2240          * PMP field only make sense when target is a port multiplier or a
2241          * device behind a port multiplier. Otherwise should set it to 0.
2242          */
2243         if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2244                 SET_FIS_PMP(h2d_register_fisp, pmport);
2245 
2246         SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2247         SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2248         SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2249         SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2250 
2251         switch (scmd->satacmd_addr_type) {
2252 
2253         case 0:
2254                 /*
2255                  * satacmd_addr_type will be 0 for the commands below:
2256                  *      ATAPI command
2257                  *      SATAC_IDLE_IM
2258                  *      SATAC_STANDBY_IM
2259                  *      SATAC_DOWNLOAD_MICROCODE
2260                  *      SATAC_FLUSH_CACHE
2261                  *      SATAC_SET_FEATURES
2262                  *      SATAC_SMART
2263                  *      SATAC_ID_PACKET_DEVICE
2264                  *      SATAC_ID_DEVICE
2265                  *      SATAC_READ_PORTMULT
2266                  *      SATAC_WRITE_PORTMULT
2267                  */
2268                 /* FALLTHRU */
2269 
2270         case ATA_ADDR_LBA:
2271                 /* FALLTHRU */
2272 
2273         case ATA_ADDR_LBA28:
2274                 /* LBA[7:0] */
2275                 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2276 
2277                 /* LBA[15:8] */
2278                 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2279 
2280                 /* LBA[23:16] */
2281                 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2282 
2283                 /* LBA [27:24] (also called dev_head) */
2284                 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2285 
2286                 break;
2287 
2288         case ATA_ADDR_LBA48:
2289                 /* LBA[7:0] */
2290                 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2291 
2292                 /* LBA[15:8] */
2293                 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2294 
2295                 /* LBA[23:16] */
2296                 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2297 
2298                 /* LBA [31:24] */
2299                 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2300                     scmd->satacmd_lba_low_msb);
2301 
2302                 /* LBA [39:32] */
2303                 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2304                     scmd->satacmd_lba_mid_msb);
2305 
2306                 /* LBA [47:40] */
2307                 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2308                     scmd->satacmd_lba_high_msb);
2309 
2310                 /* Set dev_head */
2311                 SET_FIS_DEV_HEAD(h2d_register_fisp,
2312                     scmd->satacmd_device_reg);
2313 
2314                 /* Set the extended sector count and features */
2315                 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2316                     scmd->satacmd_sec_count_msb);
2317                 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2318                     scmd->satacmd_features_reg_ext);
2319                 break;
2320         }
2321 
2322         /*
2323          * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2324          * filled into features field, and sector count 8:15 is filled into
2325          * features (exp) field. The hba driver doesn't need to anything
2326          * special with regard to this, since sata framework has already
2327          * done so.
2328          *
2329          * However the driver needs to make sure TAG is filled into sector
2330          * field.
2331          */
2332         if (command_type == AHCI_NCQ_CMD) {
2333                 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2334                     (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2335         }
2336 
2337         ncookies = scmd->satacmd_num_dma_cookies;
2338         AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2339             "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2340             ncookies, ahci_dma_prdt_number);
2341 
2342         ASSERT(ncookies <= ahci_dma_prdt_number);
2343         ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2344 
2345         /* *** now fill the scatter gather list ******* */
2346         for (i = 0; i < ncookies; i++) {
2347                 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2348                     scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2349                 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2350                     scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2351                 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2352                     scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2353                 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2354                     scmd->satacmd_dma_cookie_list[i].dmac_size;
2355         }
2356 
2357         AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2358             "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2359             ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2360 
2361         /* The ACMD field is filled in for ATAPI command */
2362         if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2363                 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2364                     SATA_ATAPI_MAX_CDB_LEN);
2365         }
2366 
2367         /* Set Command Header in Command List */
2368         cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2369         BZERO_DESCR_INFO(cmd_header);
2370         BZERO_PRD_BYTE_COUNT(cmd_header);
2371 
2372         /* Set the number of entries in the PRD table */
2373         SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2374 
2375         /* Set the length of the command in the CFIS area */
2376         SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2377 
2378         /*
2379          * PMP field only make sense when target is a port multiplier or a
2380          * device behind a port multiplier. Otherwise should set it to 0.
2381          */
2382         if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2383                 SET_PORT_MULTI_PORT(cmd_header, pmport);
2384 
2385         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2386             "sata_data_direction = 0x%x",
2387             scmd->satacmd_flags.sata_data_direction);
2388 
2389         /* Set A bit if it is an ATAPI command */
2390         if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2391                 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2392 
2393         /* Set W bit if data is going to the device */
2394         if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2395                 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2396 
2397         /*
2398          * Set the prefetchable bit - this bit is only valid if the PRDTL
2399          * field is non-zero or the ATAPI 'A' bit is set in the command
2400          * header. This bit cannot be set when using native command
2401          * queuing commands or when using FIS-based switching with a Port
2402          * multiplier.
2403          */
2404         if (command_type != AHCI_NCQ_CMD)
2405                 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2406 
2407         /*
2408          * Now remember the sata packet in ahciport_slot_pkts[].
2409          * Error retrieval command and r/w port multiplier command will
2410          * be stored specifically for each port.
2411          */
2412         if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2413             !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2414                 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2415 
2416         /*
2417          * Keep the timeout value
2418          */
2419         ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2420 
2421         /*
2422          * If the intial timout is less than 1 tick, then make it longer by
2423          * 1 tick to avoid immediate timeout
2424          */
2425         if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2426             ahci_watchdog_timeout)
2427                 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2428                     ahci_watchdog_timeout;
2429 
2430 #if AHCI_DEBUG
2431         if (ahci_debug_flags & AHCIDBG_ATACMD &&
2432             scmd->satacmd_cmd_reg != SATAC_PACKET ||
2433             ahci_debug_flags & AHCIDBG_ATAPICMD &&
2434             scmd->satacmd_cmd_reg == SATAC_PACKET) {
2435 
2436                 /* Dump the command header and table */
2437                 ahci_log(ahci_ctlp, CE_WARN, "\n");
2438                 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2439                     "0x%p cmd_reg 0x%x port %d", spkt,
2440                     scmd->satacmd_cmd_reg, port);
2441                 ptr = (uint32_t *)cmd_header;
2442                 ahci_log(ahci_ctlp, CE_WARN,
2443                     "  Command Header:%8x %8x %8x %8x",
2444                     ptr[0], ptr[1], ptr[2], ptr[3]);
2445 
2446                 /* Dump the H2D register FIS */
2447                 ptr = (uint32_t *)h2d_register_fisp;
2448                 ahci_log(ahci_ctlp, CE_WARN,
2449                     "  Command FIS:   %8x %8x %8x %8x",
2450                     ptr[0], ptr[1], ptr[2], ptr[3]);
2451 
2452                 /* Dump the ACMD register FIS */
2453                 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2454                 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2455                         if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2456                                 ahci_log(ahci_ctlp, CE_WARN,
2457                                     "  ATAPI command: %2x %2x %2x %2x "
2458                                     "%2x %2x %2x %2x",
2459                                     ptr2[8 * i], ptr2[8 * i + 1],
2460                                     ptr2[8 * i + 2], ptr2[8 * i + 3],
2461                                     ptr2[8 * i + 4], ptr2[8 * i + 5],
2462                                     ptr2[8 * i + 6], ptr2[8 * i + 7]);
2463 
2464                 /* Dump the PRDT */
2465                 for (i = 0; i < ncookies; i++) {
2466                         ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2467                         ahci_log(ahci_ctlp, CE_WARN,
2468                             "  Cookie %d:      %8x %8x %8x %8x",
2469                             i, ptr[0], ptr[1], ptr[2], ptr[3]);
2470                 }
2471         }
2472 #endif
2473 
2474         (void) ddi_dma_sync(
2475             ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2476             0,
2477             ahci_cmd_table_size,
2478             DDI_DMA_SYNC_FORDEV);
2479 
2480         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2481             cmd_slot * sizeof (ahci_cmd_header_t),
2482             sizeof (ahci_cmd_header_t),
2483             DDI_DMA_SYNC_FORDEV);
2484 
2485         if ((ahci_check_dma_handle(ahci_portp->
2486             ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2487             ahci_check_dma_handle(ahci_portp->
2488             ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2489                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2490                     DDI_SERVICE_UNAFFECTED);
2491                 return (AHCI_FAILURE);
2492         }
2493 
2494         /* Set the corresponding bit in the PxSACT.DS for queued command */
2495         if (command_type == AHCI_NCQ_CMD) {
2496                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2497                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2498                     (0x1 << cmd_slot));
2499         }
2500 
2501         /* Indicate to the HBA that a command is active. */
2502         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2503             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2504             (0x1 << cmd_slot));
2505 
2506         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2507             "exit: port %d", port);
2508 
2509         /* Make sure the command is started by the PxSACT/PxCI */
2510         if (ahci_check_acc_handle(ahci_ctlp->
2511             ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2512                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2513                     DDI_SERVICE_UNAFFECTED);
2514                 return (AHCI_FAILURE);
2515         }
2516 
2517         return (cmd_slot);
2518 }
2519 
2520 /*
2521  * Called by the sata framework to abort the previously sent packet(s).
2522  *
2523  * Reset device to abort commands.
2524  */
2525 static int
2526 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2527 {
2528         ahci_ctl_t *ahci_ctlp;
2529         ahci_port_t *ahci_portp;
2530         uint32_t slot_status = 0;
2531         uint32_t aborted_tags = 0;
2532         uint32_t finished_tags = 0;
2533         uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2534         uint8_t port;
2535         int tmp_slot;
2536         int instance = ddi_get_instance(dip);
2537 
2538         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2539         port = ahci_ctlp->ahcictl_cport_to_port[cport];
2540 
2541         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2542             "ahci_tran_abort enter: port %d", port);
2543 
2544         ahci_portp = ahci_ctlp->ahcictl_ports[port];
2545         mutex_enter(&ahci_portp->ahciport_mutex);
2546 
2547         /*
2548          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2549          * commands are being mopped, therefore there is nothing else to do
2550          */
2551         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2552                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2553                     "ahci_tran_abort: port %d is in "
2554                     "mopping process, so just return directly ", port);
2555                 mutex_exit(&ahci_portp->ahciport_mutex);
2556                 return (SATA_SUCCESS);
2557         }
2558 
2559         /*
2560          * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2561          * command is being executed so no other commands is outstanding,
2562          * nothing to do.
2563          */
2564         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2565                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2566                     "ahci_tran_abort: port %d is reading/writing "
2567                     "port multiplier, so just return directly ", port);
2568                 mutex_exit(&ahci_portp->ahciport_mutex);
2569                 return (SATA_SUCCESS);
2570         }
2571 
2572         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2573             ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2574             ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2575                 /*
2576                  * In case the targer driver would send the request before
2577                  * sata framework can have the opportunity to process those
2578                  * event reports.
2579                  */
2580                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2581                 spkt->satapkt_device.satadev_state =
2582                     ahci_portp->ahciport_port_state;
2583                 ahci_update_sata_registers(ahci_ctlp, port,
2584                     &spkt->satapkt_device);
2585                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2586                     "ahci_tran_abort returning SATA_FAILURE while "
2587                     "port in FAILED/SHUTDOWN/PWROFF state: "
2588                     "port: %d", port);
2589                 mutex_exit(&ahci_portp->ahciport_mutex);
2590                 return (SATA_FAILURE);
2591         }
2592 
2593         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2594                 /*
2595                  * ahci_intr_phyrdy_change() may have rendered it to
2596                  * AHCI_PORT_TYPE_NODEV.
2597                  */
2598                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2599                 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2600                 spkt->satapkt_device.satadev_state =
2601                     ahci_portp->ahciport_port_state;
2602                 ahci_update_sata_registers(ahci_ctlp, port,
2603                     &spkt->satapkt_device);
2604                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2605                     "ahci_tran_abort returning SATA_FAILURE while "
2606                     "no device attached: port: %d", port);
2607                 mutex_exit(&ahci_portp->ahciport_mutex);
2608                 return (SATA_FAILURE);
2609         }
2610 
2611         if (flag == SATA_ABORT_ALL_PACKETS) {
2612                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2613                         aborted_tags = ahci_portp->ahciport_pending_tags;
2614                 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2615                         aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2616 
2617                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2618                     instance, port);
2619         } else {
2620                 aborted_tags = 0xffffffff;
2621                 /*
2622                  * Aborting one specific packet, first search the
2623                  * ahciport_slot_pkts[] list for matching spkt.
2624                  */
2625                 for (tmp_slot = 0;
2626                     tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2627                         if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2628                                 aborted_tags = (0x1 << tmp_slot);
2629                                 break;
2630                         }
2631                 }
2632 
2633                 if (aborted_tags == 0xffffffff) {
2634                         /* request packet is not on the pending list */
2635                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2636                             "Cannot find the aborting pkt 0x%p on the "
2637                             "pending list", (void *)spkt);
2638                         ahci_update_sata_registers(ahci_ctlp, port,
2639                             &spkt->satapkt_device);
2640                         mutex_exit(&ahci_portp->ahciport_mutex);
2641                         return (SATA_FAILURE);
2642                 }
2643                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2644                     instance, port, (void *)spkt);
2645         }
2646 
2647         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2648                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2649                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2650         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2651                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2652                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2653 
2654         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2655         ahci_portp->ahciport_mop_in_progress++;
2656 
2657         /*
2658          * To abort the packet(s), first we are trying to clear PxCMD.ST
2659          * to stop the port, and if the port can be stopped
2660          * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2661          * then we just send back the aborted packet(s) with ABORTED flag
2662          * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2663          * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2664          * perform a COMRESET.
2665          */
2666         (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2667             ahci_portp, port, NULL, NULL);
2668 
2669         /*
2670          * Compute which have finished and which need to be retried.
2671          *
2672          * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2673          * minus the slot_status. The aborted_tags has to be deducted by
2674          * finished_tags since we can't possibly abort a tag which had finished
2675          * already.
2676          */
2677         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2678                 finished_tags = ahci_portp->ahciport_pending_tags &
2679                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2680         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2681                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2682                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2683 
2684         aborted_tags &= ~finished_tags;
2685 
2686         ahci_mop_commands(ahci_ctlp,
2687             ahci_portp,
2688             slot_status,
2689             0, /* failed tags */
2690             0, /* timeout tags */
2691             aborted_tags,
2692             0); /* reset tags */
2693 
2694         ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2695         mutex_exit(&ahci_portp->ahciport_mutex);
2696 
2697         return (SATA_SUCCESS);
2698 }
2699 
2700 /*
2701  * Used to do device reset and reject all the pending packets on a device
2702  * during the reset operation.
2703  *
2704  * NOTE: ONLY called by ahci_tran_reset_dport
2705  */
2706 static int
2707 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2708     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2709 {
2710         uint32_t slot_status = 0;
2711         uint32_t reset_tags = 0;
2712         uint32_t finished_tags = 0;
2713         uint8_t port = addrp->aa_port;
2714         sata_device_t sdevice;
2715         int ret;
2716 
2717         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2718 
2719         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2720             "ahci_reset_device_reject_pkts on port: %d", port);
2721 
2722         /*
2723          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2724          * commands are being mopped, therefore there is nothing else to do
2725          */
2726         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2727                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2728                     "ahci_reset_device_reject_pkts: port %d is in "
2729                     "mopping process, so return directly ", port);
2730                 return (SATA_SUCCESS);
2731         }
2732 
2733         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2734                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2735                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2736                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2737         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2738                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2739                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2740                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2741         }
2742 
2743         if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2744             != AHCI_SUCCESS) {
2745                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2746                     "Try to do a port reset after software "
2747                     "reset failed", port);
2748                 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2749                 if (ret != AHCI_SUCCESS) {
2750                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2751                             "ahci_reset_device_reject_pkts: port %d "
2752                             "failed", port);
2753                         return (SATA_FAILURE);
2754                 }
2755         }
2756         /* Set the reset in progress flag */
2757         ahci_portp->ahciport_reset_in_progress = 1;
2758 
2759         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2760         ahci_portp->ahciport_mop_in_progress++;
2761 
2762         /* Indicate to the framework that a reset has happened */
2763         bzero((void *)&sdevice, sizeof (sata_device_t));
2764         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2765         sdevice.satadev_addr.pmport = 0;
2766         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2767         sdevice.satadev_state = SATA_DSTATE_RESET |
2768             SATA_DSTATE_PWR_ACTIVE;
2769         mutex_exit(&ahci_portp->ahciport_mutex);
2770         sata_hba_event_notify(
2771             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2772             &sdevice,
2773             SATA_EVNT_DEVICE_RESET);
2774         mutex_enter(&ahci_portp->ahciport_mutex);
2775 
2776         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2777             "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2778 
2779         /* Next try to mop the pending commands */
2780         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2781                 finished_tags = ahci_portp->ahciport_pending_tags &
2782                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2783         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2784                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2785                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2786 
2787         reset_tags &= ~finished_tags;
2788 
2789         ahci_mop_commands(ahci_ctlp,
2790             ahci_portp,
2791             slot_status,
2792             0, /* failed tags */
2793             0, /* timeout tags */
2794             0, /* aborted tags */
2795             reset_tags); /* reset tags */
2796 
2797         return (SATA_SUCCESS);
2798 }
2799 
2800 /*
2801  * Used to do device reset and reject all the pending packets on a device
2802  * during the reset operation.
2803  *
2804  * NOTE: ONLY called by ahci_tran_reset_dport
2805  */
2806 static int
2807 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2808     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2809 {
2810         uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2811         uint8_t port = addrp->aa_port;
2812         uint8_t pmport = addrp->aa_pmport;
2813         sata_device_t sdevice;
2814 
2815         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2816 
2817         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2818             "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2819 
2820         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2821                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2822                     "ahci_reset_pmdevice_reject_pkts: port %d is in "
2823                     "mopping process, so return directly ", port);
2824                 return (SATA_SUCCESS);
2825         }
2826 
2827         /* Checking for outstanding commands */
2828         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2829                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2830                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2831                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2832         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2833                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2834                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2835                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2836         }
2837 
2838         /* Issue SOFTWARE reset command. */
2839         if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2840             != AHCI_SUCCESS) {
2841                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2842                     "Try to do a port reset after software "
2843                     "reset failed", port);
2844                 return (SATA_FAILURE);
2845         }
2846 
2847         /* Set the reset in progress flag */
2848         ahci_portp->ahciport_reset_in_progress = 1;
2849 
2850         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2851         ahci_portp->ahciport_mop_in_progress++;
2852 
2853         /* Indicate to the framework that a reset has happened */
2854         bzero((void *)&sdevice, sizeof (sata_device_t));
2855         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2856         sdevice.satadev_addr.pmport = pmport;
2857         if (AHCI_ADDR_IS_PMULT(addrp))
2858                 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2859         else
2860                 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2861         sdevice.satadev_state = SATA_DSTATE_RESET |
2862             SATA_DSTATE_PWR_ACTIVE;
2863         mutex_exit(&ahci_portp->ahciport_mutex);
2864         sata_hba_event_notify(
2865             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2866             &sdevice,
2867             SATA_EVNT_DEVICE_RESET);
2868         mutex_enter(&ahci_portp->ahciport_mutex);
2869 
2870         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2871             "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2872             port, pmport);
2873 
2874         /* Next try to mop the pending commands */
2875         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2876                 finished_tags = ahci_portp->ahciport_pending_tags &
2877                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2878         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2879                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2880                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2881         reset_tags &= ~finished_tags;
2882 
2883         AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2884             "reset_tags = %x, finished_tags = %x, slot_status = %x",
2885             reset_tags, finished_tags, slot_status);
2886 
2887         /*
2888          * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2889          * try to reset a single device behind a port multiplier will
2890          * terminate all the commands on that HBA port. We need mop these
2891          * commands as well.
2892          */
2893         ahci_mop_commands(ahci_ctlp,
2894             ahci_portp,
2895             slot_status,
2896             0, /* failed tags */
2897             0, /* timeout tags */
2898             0, /* aborted tags */
2899             reset_tags); /* reset tags */
2900 
2901         return (SATA_SUCCESS);
2902 }
2903 
2904 /*
2905  * Used to do port reset and reject all the pending packets on a port during
2906  * the reset operation.
2907  */
2908 static int
2909 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2910     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2911 {
2912         uint32_t slot_status = 0;
2913         uint32_t reset_tags = 0;
2914         uint32_t finished_tags = 0;
2915         uint8_t port = addrp->aa_port;
2916 
2917         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2918 
2919         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2920             "ahci_reset_port_reject_pkts at port: %d", port);
2921 
2922         /*
2923          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2924          * commands are being mopped, therefore there is nothing else to do
2925          */
2926         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2927                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2928                     "ahci_reset_port_reject_pkts: port %d is in "
2929                     "mopping process, so return directly ", port);
2930                 return (SATA_SUCCESS);
2931         }
2932 
2933         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2934         ahci_portp->ahciport_mop_in_progress++;
2935 
2936         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2937                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2938                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2939                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2940         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2941                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2942                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2943                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2944         }
2945 
2946         if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2947             ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2948             NULL) != AHCI_SUCCESS) {
2949 
2950                 /* Clear mop flag */
2951                 ahci_portp->ahciport_mop_in_progress--;
2952                 if (ahci_portp->ahciport_mop_in_progress == 0)
2953                         ahci_portp->ahciport_flags &=
2954                             ~AHCI_PORT_FLAG_MOPPING;
2955                 return (SATA_FAILURE);
2956         }
2957 
2958         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2959                 finished_tags = ahci_portp->ahciport_pending_tags &
2960                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2961         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2962                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2963                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2964 
2965         reset_tags &= ~finished_tags;
2966 
2967         ahci_mop_commands(ahci_ctlp,
2968             ahci_portp,
2969             slot_status,
2970             0, /* failed tags */
2971             0, /* timeout tags */
2972             0, /* aborted tags */
2973             reset_tags); /* reset tags */
2974 
2975         return (SATA_SUCCESS);
2976 }
2977 
2978 /*
2979  * Used to do hba reset and reject all the pending packets on all ports
2980  * during the reset operation.
2981  */
2982 static int
2983 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2984 {
2985         ahci_port_t *ahci_portp;
2986         uint32_t slot_status[AHCI_MAX_PORTS];
2987         uint32_t reset_tags[AHCI_MAX_PORTS];
2988         uint32_t finished_tags[AHCI_MAX_PORTS];
2989         int port;
2990         int ret = SATA_SUCCESS;
2991 
2992         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2993             "ahci_reset_hba_reject_pkts enter", NULL);
2994 
2995         bzero(slot_status, sizeof (slot_status));
2996         bzero(reset_tags, sizeof (reset_tags));
2997         bzero(finished_tags, sizeof (finished_tags));
2998 
2999         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3000                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3001                         continue;
3002                 }
3003 
3004                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3005 
3006                 mutex_enter(&ahci_portp->ahciport_mutex);
3007                 ahci_portp->ahciport_reset_in_progress = 1;
3008                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3009                         slot_status[port] = ddi_get32(
3010                             ahci_ctlp->ahcictl_ahci_acc_handle,
3011                             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3012                         reset_tags[port] = slot_status[port] &
3013                             AHCI_SLOT_MASK(ahci_ctlp);
3014                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3015                             "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3016                             port, reset_tags[port],
3017                             ahci_portp->ahciport_pending_tags);
3018                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3019                         slot_status[port] = ddi_get32(
3020                             ahci_ctlp->ahcictl_ahci_acc_handle,
3021                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3022                         reset_tags[port] = slot_status[port] &
3023                             AHCI_NCQ_SLOT_MASK(ahci_portp);
3024                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3025                             "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3026                             port, reset_tags[port],
3027                             ahci_portp->ahciport_pending_tags);
3028                 }
3029                 mutex_exit(&ahci_portp->ahciport_mutex);
3030         }
3031 
3032         if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
3033                 ret = SATA_FAILURE;
3034         }
3035 
3036         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3037                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3038                         continue;
3039                 }
3040 
3041                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3042 
3043                 mutex_enter(&ahci_portp->ahciport_mutex);
3044                 /*
3045                  * To prevent recursive enter to ahci_mop_commands, we need
3046                  * check AHCI_PORT_FLAG_MOPPING flag.
3047                  */
3048                 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3049                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3050                             "ahci_reset_hba_reject_pkts: port %d is in "
3051                             "mopping process, so return directly ", port);
3052                         mutex_exit(&ahci_portp->ahciport_mutex);
3053                         continue;
3054                 }
3055 
3056                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3057                 ahci_portp->ahciport_mop_in_progress++;
3058 
3059                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
3060                         finished_tags[port]  =
3061                             ahci_portp->ahciport_pending_tags &
3062                             ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
3063                 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
3064                         finished_tags[port] =
3065                             ahci_portp->ahciport_pending_ncq_tags &
3066                             ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
3067 
3068                 reset_tags[port] &= ~finished_tags[port];
3069 
3070                 ahci_mop_commands(ahci_ctlp,
3071                     ahci_portp,
3072                     slot_status[port],
3073                     0, /* failed tags */
3074                     0, /* timeout tags */
3075                     0, /* aborted tags */
3076                     reset_tags[port]); /* reset tags */
3077                 mutex_exit(&ahci_portp->ahciport_mutex);
3078         }
3079 out:
3080         return (ret);
3081 }
3082 
3083 /*
3084  * Called by sata framework to reset a port(s) or device.
3085  */
3086 static int
3087 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
3088 {
3089         ahci_ctl_t *ahci_ctlp;
3090         ahci_port_t *ahci_portp;
3091         ahci_addr_t addr;
3092         uint8_t cport = sd->satadev_addr.cport;
3093         uint8_t pmport = sd->satadev_addr.pmport;
3094         uint8_t port;
3095         int ret = SATA_SUCCESS;
3096         int instance = ddi_get_instance(dip);
3097 
3098         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3099         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3100         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3101 
3102         ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3103 
3104         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3105             "ahci_tran_reset_dport enter: cport %d", cport);
3106 
3107         switch (sd->satadev_addr.qual) {
3108         case SATA_ADDR_PMPORT:
3109                 /*
3110                  * If we want to issue a COMRESET on a pmport, we need to
3111                  * reject the outstanding commands on that pmport. According
3112                  * to AHCI spec, PxCI register could only be cleared by
3113                  * clearing PxCMD.ST, which will halt the controller port - as
3114                  * well as other pmports.
3115                  *
3116                  * Therefore we directly reset the controller port for
3117                  * simplicity. ahci_tran_probe_port() will handle reset stuff
3118                  * like initializing the given pmport.
3119                  */
3120                 /* FALLTHRU */
3121         case SATA_ADDR_CPORT:
3122                 /* Port reset */
3123                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3124                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3125                     "port %d reset port", instance, port);
3126 
3127                 mutex_enter(&ahci_portp->ahciport_mutex);
3128                 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3129                 mutex_exit(&ahci_portp->ahciport_mutex);
3130 
3131                 break;
3132 
3133         case SATA_ADDR_DPMPORT:
3134                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3135                     "port %d:%d reset device", instance, port, pmport);
3136                 /* FALLTHRU */
3137         case SATA_ADDR_DCPORT:
3138                 /* Device reset */
3139                 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3140                         cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3141                             "port %d reset device", instance, port);
3142 
3143                 mutex_enter(&ahci_portp->ahciport_mutex);
3144                 /*
3145                  * software reset request must be sent to SATA_PMULT_HOSTPORT
3146                  * if target is a port multiplier:
3147                  */
3148                 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3149                     ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3150                         AHCI_ADDR_SET_PMULT(&addr, port);
3151 
3152                 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3153                     ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3154                     ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3155                         /*
3156                          * In case the targer driver would send the request
3157                          * before sata framework can have the opportunity to
3158                          * process those event reports.
3159                          */
3160                         sd->satadev_state = ahci_portp->ahciport_port_state;
3161                         ahci_update_sata_registers(ahci_ctlp, port, sd);
3162                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3163                             "ahci_tran_reset_dport returning SATA_FAILURE "
3164                             "while port in FAILED/SHUTDOWN/PWROFF state: "
3165                             "port: %d", port);
3166                         mutex_exit(&ahci_portp->ahciport_mutex);
3167                         ret = SATA_FAILURE;
3168                         break;
3169                 }
3170 
3171                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3172                     SATA_DTYPE_NONE) {
3173                         /*
3174                          * ahci_intr_phyrdy_change() may have rendered it to
3175                          * AHCI_PORT_TYPE_NODEV.
3176                          */
3177                         sd->satadev_type = SATA_DTYPE_NONE;
3178                         sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3179                             &addr);
3180                         ahci_update_sata_registers(ahci_ctlp, port, sd);
3181                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3182                             "ahci_tran_reset_dport returning SATA_FAILURE "
3183                             "while no device attached: port: %d", port);
3184                         mutex_exit(&ahci_portp->ahciport_mutex);
3185                         ret = SATA_FAILURE;
3186                         break;
3187                 }
3188 
3189                 if (AHCI_ADDR_IS_PORT(&addr)) {
3190                         ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3191                             ahci_portp, &addr);
3192                 } else {
3193                         ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3194                             ahci_portp, &addr);
3195                 }
3196 
3197                 mutex_exit(&ahci_portp->ahciport_mutex);
3198                 break;
3199 
3200         case SATA_ADDR_CNTRL:
3201                 /* Reset the whole controller */
3202                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3203                     "reset the whole hba", instance);
3204                 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3205                 break;
3206 
3207         default:
3208                 ret = SATA_FAILURE;
3209         }
3210 
3211         return (ret);
3212 }
3213 
3214 /*
3215  * Called by sata framework to activate a port as part of hotplug.
3216  * (cfgadm -c connect satax/y)
3217  * Support port multiplier.
3218  */
3219 static int
3220 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3221 {
3222         ahci_ctl_t *ahci_ctlp;
3223         ahci_port_t *ahci_portp;
3224         ahci_addr_t addr;
3225         uint8_t cport = satadev->satadev_addr.cport;
3226         uint8_t pmport = satadev->satadev_addr.pmport;
3227         uint8_t port;
3228         int instance = ddi_get_instance(dip);
3229 
3230         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3231         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3232 
3233         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3234             "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3235 
3236         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3237 
3238         mutex_enter(&ahci_portp->ahciport_mutex);
3239         ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3240         ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3241 
3242         if (AHCI_ADDR_IS_PORT(&addr)) {
3243                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3244                     instance, port);
3245 
3246                 /* Enable the interrupts on the port */
3247                 ahci_enable_port_intrs(ahci_ctlp, port);
3248 
3249                 /*
3250                  * Reset the port so that the PHY communication would be
3251                  * re-established.  But this reset is an internal operation
3252                  * and the sata module doesn't need to know about it.
3253                  * Moreover, the port with a device attached will be started
3254                  * too.
3255                  */
3256                 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3257                     ahci_portp, port,
3258                     AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3259                     NULL);
3260 
3261                 /*
3262                  * Need to check the link status and device status of the port
3263                  * and consider raising power if the port was in D3 state
3264                  */
3265                 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3266                 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3267                 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3268         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3269                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3270                     instance, port, pmport);
3271                 /* AHCI_ADDR_PMPORT */
3272                 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3273                 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3274                     ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3275         }
3276 
3277         satadev->satadev_state = ahci_portp->ahciport_port_state;
3278 
3279         ahci_update_sata_registers(ahci_ctlp, port, satadev);
3280 
3281         mutex_exit(&ahci_portp->ahciport_mutex);
3282         return (SATA_SUCCESS);
3283 }
3284 
3285 /*
3286  * Called by sata framework to deactivate a port as part of hotplug.
3287  * (cfgadm -c disconnect satax/y)
3288  * Support port multiplier.
3289  */
3290 static int
3291 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3292 {
3293         ahci_ctl_t *ahci_ctlp;
3294         ahci_port_t *ahci_portp;
3295         ahci_addr_t addr;
3296         uint8_t cport = satadev->satadev_addr.cport;
3297         uint8_t pmport = satadev->satadev_addr.pmport;
3298         uint8_t port;
3299         uint32_t port_scontrol;
3300         int instance = ddi_get_instance(dip);
3301 
3302         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3303         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3304 
3305         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3306             "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3307 
3308         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3309         mutex_enter(&ahci_portp->ahciport_mutex);
3310         ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3311         ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3312 
3313         if (AHCI_ADDR_IS_PORT(&addr)) {
3314                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3315                     instance, port);
3316 
3317                 /* Disable the interrupts on the port */
3318                 ahci_disable_port_intrs(ahci_ctlp, port);
3319 
3320                 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3321 
3322                         /* First to abort all the pending commands */
3323                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3324 
3325                         /* Then stop the port */
3326                         (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3327                             ahci_portp, port);
3328                 }
3329 
3330                 /* Next put the PHY offline */
3331                 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3332                     (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3333                 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3334                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3335                     AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3336         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3337                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3338                     instance, port, pmport);
3339 
3340                 ahci_disable_port_intrs(ahci_ctlp, port);
3341                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3342                     != SATA_DTYPE_NONE)
3343                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3344 
3345                 /* Re-enable the interrupts for the other pmports */
3346                 ahci_enable_port_intrs(ahci_ctlp, port);
3347         }
3348 
3349         /* Update port state */
3350         AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3351         satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3352 
3353         ahci_update_sata_registers(ahci_ctlp, port, satadev);
3354 
3355         mutex_exit(&ahci_portp->ahciport_mutex);
3356         return (SATA_SUCCESS);
3357 }
3358 
3359 /*
3360  * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3361  * when a device is unplugged or a port is deactivated.
3362  */
3363 static void
3364 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3365     ahci_port_t *ahci_portp, uint8_t port)
3366 {
3367         uint32_t slot_status = 0;
3368         uint32_t abort_tags = 0;
3369 
3370         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3371 
3372         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3373             "ahci_reject_all_abort_pkts at port: %d", port);
3374 
3375         /* Read/write port multiplier command takes highest priority */
3376         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3377                 slot_status = 0x1;
3378                 abort_tags = 0x1;
3379                 goto out;
3380         }
3381 
3382         /*
3383          * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3384          * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3385          * to get the error data, if yes when the device is removed, the
3386          * command needs to be aborted too.
3387          */
3388         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3389                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3390                         slot_status = 0x1;
3391                         abort_tags = 0x1;
3392                         goto out;
3393                 } else {
3394                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3395                             "ahci_reject_all_abort_pkts return directly "
3396                             "port %d no needs to reject any outstanding "
3397                             "commands", port);
3398                         return;
3399                 }
3400         }
3401 
3402         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3403                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3404                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3405                 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3406         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3407                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3408                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3409                 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3410         }
3411 
3412 out:
3413         /* No need to do mop when there is no outstanding commands */
3414         if (slot_status != 0) {
3415                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3416                 ahci_portp->ahciport_mop_in_progress++;
3417 
3418                 ahci_mop_commands(ahci_ctlp,
3419                     ahci_portp,
3420                     slot_status,
3421                     0, /* failed tags */
3422                     0, /* timeout tags */
3423                     abort_tags, /* aborting tags */
3424                     0); /* reset tags */
3425         }
3426 }
3427 
3428 #if defined(__lock_lint)
3429 static int
3430 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3431 {
3432         return (SATA_SUCCESS);
3433 }
3434 #endif
3435 
3436 /*
3437  * Initialize fma capabilities and register with IO fault services.
3438  */
3439 static void
3440 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3441 {
3442         /*
3443          * Need to change iblock to priority for new MSI intr
3444          */
3445         ddi_iblock_cookie_t fm_ibc;
3446 
3447         ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3448             ahci_ctlp->ahcictl_dip,
3449             DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3450             DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3451             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3452 
3453         /* Only register with IO Fault Services if we have some capability */
3454         if (ahci_ctlp->ahcictl_fm_cap) {
3455                 /* Adjust access and dma attributes for FMA */
3456                 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3457                 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3458                 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3459                 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3460                 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3461 
3462                 /*
3463                  * Register capabilities with IO Fault Services.
3464                  * ahcictl_fm_cap will be updated to indicate
3465                  * capabilities actually supported (not requested.)
3466                  */
3467                 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3468                     &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3469 
3470                 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3471                         cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3472                             ddi_get_instance(ahci_ctlp->ahcictl_dip));
3473                         return;
3474                 }
3475                 /*
3476                  * Initialize pci ereport capabilities if ereport
3477                  * capable (should always be.)
3478                  */
3479                 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3480                     DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3481                         pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3482                 }
3483 
3484                 /*
3485                  * Register error callback if error callback capable.
3486                  */
3487                 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3488                         ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3489                             ahci_fm_error_cb, (void *) ahci_ctlp);
3490                 }
3491 
3492                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3493                     "ahci_fm_fini: fma enabled.", NULL);
3494         }
3495 }
3496 
3497 /*
3498  * Releases fma capabilities and un-registers with IO fault services.
3499  */
3500 static void
3501 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3502 {
3503         /* Only unregister FMA capabilities if registered */
3504         if (ahci_ctlp->ahcictl_fm_cap) {
3505                 /*
3506                  * Un-register error callback if error callback capable.
3507                  */
3508                 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3509                         ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3510                 }
3511 
3512                 /*
3513                  * Release any resources allocated by pci_ereport_setup()
3514                  */
3515                 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3516                     DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3517                         pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3518                 }
3519 
3520                 /* Unregister from IO Fault Services */
3521                 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3522 
3523                 /* Adjust access and dma attributes for FMA */
3524                 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3525                 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3526                 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3527                 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3528                 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3529 
3530                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3531                     "ahci_fm_fini: fma disabled.", NULL);
3532         }
3533 }
3534 
3535 /*ARGSUSED*/
3536 static int
3537 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3538 {
3539         /*
3540          * as the driver can always deal with an error in any dma or
3541          * access handle, we can just return the fme_status value.
3542          */
3543         pci_ereport_post(dip, err, NULL);
3544         return (err->fme_status);
3545 }
3546 
3547 int
3548 ahci_check_acc_handle(ddi_acc_handle_t handle)
3549 {
3550         ddi_fm_error_t de;
3551 
3552         ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3553         return (de.fme_status);
3554 }
3555 
3556 int
3557 ahci_check_dma_handle(ddi_dma_handle_t handle)
3558 {
3559         ddi_fm_error_t de;
3560 
3561         ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3562         return (de.fme_status);
3563 }
3564 
3565 /*
3566  * Generate an ereport
3567  */
3568 void
3569 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3570 {
3571         uint64_t ena;
3572         char buf[FM_MAX_CLASS];
3573 
3574         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3575         ena = fm_ena_generate(0, FM_ENA_FMT1);
3576         if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3577                 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3578                     DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3579                     FM_EREPORT_VERSION, NULL);
3580         }
3581 }
3582 
3583 /*
3584  * Check if all handles are correctly allocated.
3585  */
3586 static int
3587 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3588 {
3589         int port;
3590 
3591         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3592                 return (DDI_FAILURE);
3593         }
3594 
3595         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3596                 ahci_port_t *ahci_portp;
3597 
3598                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3599                         continue;
3600 
3601                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3602 
3603                 mutex_enter(&ahci_portp->ahciport_mutex);
3604                 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3605                         mutex_exit(&ahci_portp->ahciport_mutex);
3606                         return (DDI_FAILURE);
3607                 }
3608                 mutex_exit(&ahci_portp->ahciport_mutex);
3609         }
3610 
3611         return (DDI_SUCCESS);
3612 }
3613 
3614 /*
3615  * Check the access handles for the controller. Note that
3616  * ahcictl_pci_conf_handle is only used in attach process.
3617  */
3618 static int
3619 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3620 {
3621         if ((ahci_check_acc_handle(ahci_ctlp->
3622             ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3623             (ahci_check_acc_handle(ahci_ctlp->
3624             ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3625                 return (DDI_FAILURE);
3626         }
3627         return (DDI_SUCCESS);
3628 }
3629 
3630 /*
3631  * Check the DMA handles and the access handles of a controller port.
3632  */
3633 static int
3634 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3635 {
3636         ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3637         int slot;
3638 
3639         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3640 
3641         if ((ahci_check_dma_handle(ahci_portp->
3642             ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3643             (ahci_check_dma_handle(ahci_portp->
3644             ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3645             (ahci_check_acc_handle(ahci_portp->
3646             ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3647             (ahci_check_acc_handle(ahci_portp->
3648             ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3649                 return (DDI_FAILURE);
3650         }
3651         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3652                 if (ahci_check_slot_handle(ahci_portp, slot)
3653                     != DDI_SUCCESS) {
3654                         return (DDI_FAILURE);
3655                 }
3656         }
3657         return (DDI_SUCCESS);
3658 }
3659 
3660 /*
3661  * Check the DMA handles and the access handles of a cmd table slot.
3662  */
3663 static int
3664 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3665 {
3666         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3667 
3668         if ((ahci_check_acc_handle(ahci_portp->
3669             ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3670             (ahci_check_dma_handle(ahci_portp->
3671             ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3672                 return (DDI_FAILURE);
3673         }
3674         return (DDI_SUCCESS);
3675 }
3676 
3677 /*
3678  * Allocate the ports structure, only called by ahci_attach
3679  */
3680 static int
3681 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3682 {
3683         int port, cport = 0;
3684 
3685         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3686             "ahci_alloc_ports_state enter", NULL);
3687 
3688         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3689 
3690         /* Allocate structures only for the implemented ports */
3691         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3692                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3693                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3694                             "hba port %d not implemented", port);
3695                         continue;
3696                 }
3697 
3698                 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3699                 ahci_ctlp->ahcictl_port_to_cport[port] =
3700                     (uint8_t)cport++;
3701 
3702                 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3703                         goto err_out;
3704                 }
3705         }
3706 
3707         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3708         return (AHCI_SUCCESS);
3709 
3710 err_out:
3711         for (port--; port >= 0; port--) {
3712                 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3713                         ahci_dealloc_port_state(ahci_ctlp, port);
3714                 }
3715         }
3716 
3717         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3718         return (AHCI_FAILURE);
3719 }
3720 
3721 /*
3722  * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3723  */
3724 static void
3725 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3726 {
3727         int port;
3728 
3729         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3730         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3731                 /* if this port is implemented by the HBA */
3732                 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3733                         ahci_dealloc_port_state(ahci_ctlp, port);
3734         }
3735         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3736 }
3737 
3738 /*
3739  * Drain the taskq.
3740  */
3741 static void
3742 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3743 {
3744         ahci_port_t *ahci_portp;
3745         int port;
3746 
3747         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3748                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3749                         continue;
3750                 }
3751 
3752                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3753 
3754                 mutex_enter(&ahci_portp->ahciport_mutex);
3755                 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3756                 mutex_exit(&ahci_portp->ahciport_mutex);
3757         }
3758 }
3759 
3760 /*
3761  * Initialize the controller and all ports. And then try to start the ports
3762  * if there are devices attached.
3763  *
3764  * This routine can be called from three seperate cases: DDI_ATTACH,
3765  * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3766  * other two cases; device signature probing are attempted only during
3767  * DDI_ATTACH case.
3768  */
3769 static int
3770 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3771 {
3772         ahci_port_t *ahci_portp;
3773         ahci_addr_t addr;
3774         int port;
3775 
3776         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3777             "ahci_initialize_controller enter", NULL);
3778 
3779         /* Disable the whole controller interrupts */
3780         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3781         ahci_disable_all_intrs(ahci_ctlp);
3782         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3783 
3784         /* Initialize the implemented ports and structures */
3785         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3786                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3787                         continue;
3788                 }
3789 
3790                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3791                 mutex_enter(&ahci_portp->ahciport_mutex);
3792 
3793                 /*
3794                  * Ensure that the controller is not in the running state
3795                  * by checking every implemented port's PxCMD register
3796                  */
3797                 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3798 
3799                 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3800                     != AHCI_SUCCESS) {
3801                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3802                             "ahci_initialize_controller: failed to "
3803                             "initialize port %d", port);
3804                         /*
3805                          * Set the port state to SATA_PSTATE_FAILED if
3806                          * failed to initialize it.
3807                          */
3808                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3809                 }
3810 
3811                 mutex_exit(&ahci_portp->ahciport_mutex);
3812         }
3813 
3814         /* Enable the whole controller interrupts */
3815         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3816         ahci_enable_all_intrs(ahci_ctlp);
3817         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3818 
3819         return (AHCI_SUCCESS);
3820 }
3821 
3822 /*
3823  * Reverse of ahci_initialize_controller()
3824  *
3825  * We only need to stop the ports and disable the interrupt.
3826  */
3827 static void
3828 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3829 {
3830         ahci_port_t *ahci_portp;
3831         int port;
3832 
3833         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3834             "ahci_uninitialize_controller enter", NULL);
3835 
3836         /* disable all the interrupts. */
3837         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3838         ahci_disable_all_intrs(ahci_ctlp);
3839         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3840 
3841         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3842                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3843                         continue;
3844                 }
3845 
3846                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3847 
3848                 /* Stop the port by clearing PxCMD.ST */
3849                 mutex_enter(&ahci_portp->ahciport_mutex);
3850 
3851                 /*
3852                  * Here we must disable the port interrupt because
3853                  * ahci_disable_all_intrs only clear GHC.IE, and IS
3854                  * register will be still set if PxIE is enabled.
3855                  * When ahci shares one IRQ with other drivers, the
3856                  * intr handler may claim the intr mistakenly.
3857                  */
3858                 ahci_disable_port_intrs(ahci_ctlp, port);
3859                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3860                     ahci_portp, port);
3861                 mutex_exit(&ahci_portp->ahciport_mutex);
3862         }
3863 }
3864 
3865 /*
3866  * ahci_alloc_pmult()
3867  * 1. Setting HBA port registers which are necessary for a port multiplier.
3868  *    (Set PxCMD.PMA while PxCMD.ST is '0')
3869  * 2. Allocate ahci_pmult_info structure.
3870  *
3871  * NOTE: Must stop port before the function is called.
3872  */
3873 static void
3874 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3875 {
3876         uint32_t port_cmd_status;
3877         uint8_t port = ahci_portp->ahciport_port_num;
3878 
3879         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3880 
3881         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3882             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3883 
3884         /* The port must have been stopped before. */
3885         ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3886 
3887         if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3888                 /* set PMA bit */
3889                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3890                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3891                     port_cmd_status|AHCI_CMD_STATUS_PMA);
3892 
3893                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3894                     "ahci_alloc_pmult: "
3895                     "PxCMD.PMA bit set at port %d.", port);
3896         }
3897 
3898         /* Allocate port multiplier information structure */
3899         if (ahci_portp->ahciport_pmult_info == NULL) {
3900                 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3901                     kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3902         }
3903 
3904         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3905 }
3906 
3907 /*
3908  * ahci_dealloc_pmult()
3909  * 1. Clearing related registers when a port multiplier is detached.
3910  *    (Clear PxCMD.PMA while PxCMD.ST is '0')
3911  * 2. Deallocate ahci_pmult_info structure.
3912  *
3913  * NOTE: Must stop port before the function is called.
3914  */
3915 static void
3916 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3917 {
3918         uint32_t port_cmd_status;
3919         uint8_t port = ahci_portp->ahciport_port_num;
3920 
3921         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3922 
3923         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3924             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3925 
3926         if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3927                 /* Clear PMA bit */
3928                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3929                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3930                     (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3931 
3932                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3933                     "ahci_dealloc_pmult: "
3934                     "PxCMD.PMA bit cleared at port %d.", port);
3935         }
3936 
3937         /* Release port multiplier information structure */
3938         if (ahci_portp->ahciport_pmult_info != NULL) {
3939                 kmem_free(ahci_portp->ahciport_pmult_info,
3940                     sizeof (ahci_pmult_info_t));
3941                 ahci_portp->ahciport_pmult_info = NULL;
3942         }
3943 }
3944 
3945 /*
3946  * Staggered Spin-up.
3947  */
3948 static void
3949 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3950 {
3951         uint32_t cap_status;
3952         uint32_t port_cmd_status;
3953 
3954         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3955 
3956         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3957             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3958 
3959         /* Check for staggered spin-up support */
3960         if (!(cap_status & AHCI_HBA_CAP_SSS))
3961                 return;
3962 
3963         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3964             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3965 
3966         /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3967         if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3968                 return;
3969 
3970         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3971 
3972         /* Set PxCMD.SUD */
3973         port_cmd_status |= AHCI_CMD_STATUS_SUD;
3974         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3975             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3976             port_cmd_status);
3977 }
3978 
3979 /*
3980  * The routine is to initialize a port. First put the port in NotRunning
3981  * state, then enable port interrupt and clear Serror register. And under
3982  * AHCI_ATTACH case, find device signature and then try to start the port.
3983  *
3984  * Called by
3985  *    1. ahci_initialize_controller
3986  *    2. ahci_intr_phyrdy_change (hotplug)
3987  */
3988 static int
3989 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3990     ahci_addr_t *addrp)
3991 {
3992         uint32_t port_sstatus, port_task_file, port_cmd_status;
3993         uint8_t port = addrp->aa_port;
3994         boolean_t resuming = B_TRUE;    /*  processing DDI_RESUME */
3995         int ret;
3996 
3997         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3998 
3999         /* AHCI_ADDR_PORT: We've no idea of the attached device here.  */
4000         ASSERT(AHCI_ADDR_IS_PORT(addrp));
4001 
4002         /*
4003          * At the time being, only probe ports/devices and get the types of
4004          * attached devices during DDI_ATTACH. In fact, the device can be
4005          * changed during power state changes, but at the time being, we
4006          * don't support the situation.
4007          */
4008         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4009                 resuming = B_FALSE;
4010         } else {
4011                 /* check for DDI_RESUME case */
4012                 mutex_exit(&ahci_portp->ahciport_mutex);
4013                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
4014                 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
4015                         resuming = B_FALSE;
4016                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
4017                 mutex_enter(&ahci_portp->ahciport_mutex);
4018         }
4019 
4020         if (resuming) {
4021                 /*
4022                  * During the resume, we need to set the PxCLB, PxCLBU, PxFB
4023                  * and PxFBU registers in case these registers were cleared
4024                  * during the suspend.
4025                  */
4026                 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4027                     "ahci_initialize_port: port %d "
4028                     "set PxCLB, PxCLBU, PxFB and PxFBU "
4029                     "during resume", port);
4030 
4031                 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
4032                     AHCI_SUCCESS)
4033                         return (AHCI_FAILURE);
4034         }
4035 
4036         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4037             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4038 
4039         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4040             "ahci_initialize_port: port %d ", port);
4041 
4042         /*
4043          * Check whether the port is in NotRunning state, if not,
4044          * put the port in NotRunning state
4045          */
4046         if (port_cmd_status &
4047             (AHCI_CMD_STATUS_ST |
4048             AHCI_CMD_STATUS_CR |
4049             AHCI_CMD_STATUS_FRE |
4050             AHCI_CMD_STATUS_FR)) {
4051                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
4052                     ahci_portp, port);
4053         }
4054 
4055         /* Make sure the drive is spun-up */
4056         ahci_staggered_spin_up(ahci_ctlp, port);
4057 
4058         /* Disable interrupt */
4059         ahci_disable_port_intrs(ahci_ctlp, port);
4060 
4061         /* Device is unknown at first */
4062         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4063 
4064         /* Disable the interface power management */
4065         ahci_disable_interface_pm(ahci_ctlp, port);
4066 
4067         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4068             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
4069         port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4070             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4071 
4072         /* Check physcial link status */
4073         if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
4074             SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
4075 
4076             /* Check interface status */
4077             port_task_file & AHCI_TFD_STS_BSY ||
4078             port_task_file & AHCI_TFD_STS_DRQ ||
4079 
4080             /* Check whether port reset must be executed */
4081             ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
4082 
4083             /* Always reset port on RESUME */
4084             resuming != B_FALSE) {
4085 
4086                 /* Something went wrong, we need do some reset things */
4087                 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
4088 
4089                 /* Does port reset succeed on HBA port? */
4090                 if (ret != AHCI_SUCCESS) {
4091                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4092                             "ahci_initialize_port:"
4093                             "port reset failed at port %d", port);
4094                         return (AHCI_FAILURE);
4095                 }
4096 
4097                 /* Is port failed? */
4098                 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4099                     SATA_PSTATE_FAILED) {
4100                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4101                             "ahci_initialize_port: port %d state 0x%x",
4102                             port, ahci_portp->ahciport_port_state);
4103                         return (AHCI_FAILURE);
4104                 }
4105         }
4106 
4107         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4108         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4109 
4110         /*
4111          * Try to get the device signature if the port is not empty.
4112          */
4113         if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4114             SATA_DTYPE_NONE)
4115                 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4116 
4117         /* Return directly if no device connected */
4118         if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4119                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4120                     "No device connected to port %d", port);
4121                 goto out;
4122         }
4123 
4124         /* If this is a port multiplier, we need do some initialization */
4125         if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4126                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4127                     "Port multiplier found at port %d", port);
4128                 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4129         }
4130 
4131         /* Try to start the port */
4132         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4133             != AHCI_SUCCESS) {
4134                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4135                     "failed to start port %d", port);
4136                 return (AHCI_FAILURE);
4137         }
4138 out:
4139         /* Enable port interrupts */
4140         ahci_enable_port_intrs(ahci_ctlp, port);
4141 
4142         return (AHCI_SUCCESS);
4143 }
4144 
4145 /*
4146  *  Handle hardware defect, and check the capabilities. For example,
4147  *  power management capabilty and MSI capability.
4148  */
4149 static int
4150 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4151 {
4152         ushort_t caps_ptr, cap_count, cap;
4153 #if AHCI_DEBUG
4154         ushort_t pmcap, pmcsr;
4155         ushort_t msimc;
4156 #endif
4157         uint8_t revision;
4158 
4159         ahci_ctlp->ahcictl_venid =
4160             pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4161             PCI_CONF_VENID);
4162 
4163         ahci_ctlp->ahcictl_devid =
4164             pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4165             PCI_CONF_DEVID);
4166 
4167         /*
4168          * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4169          * controllers with 0x00 revision id work on 4-byte aligned buffer,
4170          * which is a bug and was fixed after 0x00 revision id controllers.
4171          *
4172          * Moreover, VT8251 cannot use multiple command slots in the command
4173          * list for non-queued commands because the previous register content
4174          * of PxCI can be re-written in the register write, so a flag will be
4175          * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4176          *
4177          * For VT8251, software reset also has the same defect as the below
4178          * AMD/ATI chipset. That is, software reset will get failed if 0xf
4179          * is filled in pmport field. Therefore, another software reset need
4180          * to be done with 0 filled in pmport field.
4181          */
4182         if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4183                 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4184                     PCI_CONF_REVID);
4185                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4186                     "revision id = 0x%x", revision);
4187                 if (revision == 0x00) {
4188                         ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4189                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4190                             "change ddi_attr_align to 0x4", NULL);
4191                 }
4192 
4193                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4194                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4195                     "VT8251 cannot use multiple command lists for "
4196                     "non-queued commands", NULL);
4197 
4198                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4199         }
4200 
4201         /*
4202          * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4203          * DMA addressing for communication memory descriptors though S64A bit
4204          * of CAP register declares it supports. Even though 64-bit DMA for
4205          * data buffer works on ASUS M2A-VM with newer BIOS, three other
4206          * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4207          * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4208          *
4209          * Due to certain hardware issue, the chipset must do port reset during
4210          * initialization, otherwise, when retrieving device signature,
4211          * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4212          * need to set.
4213          *
4214          * For this chipset software reset will get failure if the pmport of
4215          * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4216          * multiplier is connected to the port. In order to fix the issue,
4217          * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4218          * reset got failure, the driver will try to do another software reset
4219          * with pmport 0.
4220          */
4221         if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4222             ahci_ctlp->ahcictl_devid == 0x4380) {
4223                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4224                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4225                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4226                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4227 
4228                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4229                     "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4230                     "communication memory descriptors though CAP indicates "
4231                     "support, so force it to use 32-bit DMA", NULL);
4232                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4233                     "ATI SB600 need to do a port reset during initialization",
4234                     NULL);
4235                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4236                     "ATI SB600 will get software reset failure if pmport "
4237                     "is set 0xf and no port multiplier is attached", NULL);
4238         }
4239 
4240         /*
4241          * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4242          * vendor ID and device ID (0x1002,0x4391).
4243          *
4244          * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4245          * DMA addressing for communication memory descriptors though S64A bit
4246          * of CAP register declares the support. However, it does support
4247          * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4248          * set for this controller.
4249          *
4250          * SB710 has the same initialization issue as SB600, so it also need
4251          * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4252          *
4253          * SB700 also has the same issue about software reset, and thus
4254          * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4255          */
4256         if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4257             ahci_ctlp->ahcictl_devid == 0x4391) {
4258                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4259                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4260                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4261 
4262                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4263                     "ATI SB700/750 cannot do 64-bit DMA for communication "
4264                     "memory descriptors though CAP indicates support, "
4265                     "so force it to use 32-bit DMA", NULL);
4266                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4267                     "ATI SB710 need to do a port reset during initialization",
4268                     NULL);
4269                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4270                     "ATI SB700 will get software reset failure if pmport "
4271                     "is set 0xf and no port multiplier is attached", NULL);
4272         }
4273 
4274         /*
4275          * Check if capabilities list is supported and if so,
4276          * get initial capabilities pointer and clear bits 0,1.
4277          */
4278         if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4279             PCI_CONF_STAT) & PCI_STAT_CAP) {
4280                 caps_ptr = P2ALIGN(pci_config_get8(
4281                     ahci_ctlp->ahcictl_pci_conf_handle,
4282                     PCI_CONF_CAP_PTR), 4);
4283         } else {
4284                 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4285         }
4286 
4287         /*
4288          * Walk capabilities if supported.
4289          */
4290         for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4291 
4292                 /*
4293                  * Check that we haven't exceeded the maximum number of
4294                  * capabilities and that the pointer is in a valid range.
4295                  */
4296                 if (++cap_count > PCI_CAP_MAX_PTR) {
4297                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4298                             "too many device capabilities", NULL);
4299                         return (AHCI_FAILURE);
4300                 }
4301                 if (caps_ptr < PCI_CAP_PTR_OFF) {
4302                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4303                             "capabilities pointer 0x%x out of range",
4304                             caps_ptr);
4305                         return (AHCI_FAILURE);
4306                 }
4307 
4308                 /*
4309                  * Get next capability and check that it is valid.
4310                  * For now, we only support power management.
4311                  */
4312                 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4313                     caps_ptr);
4314                 switch (cap) {
4315                 case PCI_CAP_ID_PM:
4316 
4317                         /* power management supported */
4318                         ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4319 
4320                         /* Save PMCSR offset */
4321                         ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4322 
4323 #if AHCI_DEBUG
4324                         pmcap = pci_config_get16(
4325                             ahci_ctlp->ahcictl_pci_conf_handle,
4326                             caps_ptr + PCI_PMCAP);
4327                         pmcsr = pci_config_get16(
4328                             ahci_ctlp->ahcictl_pci_conf_handle,
4329                             ahci_ctlp->ahcictl_pmcsr_offset);
4330                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4331                             "Power Management capability found PCI_PMCAP "
4332                             "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4333                         if ((pmcap & 0x3) == 0x3)
4334                                 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4335                                     "PCI Power Management Interface "
4336                                     "spec 1.2 compliant", NULL);
4337 #endif
4338                         break;
4339 
4340                 case PCI_CAP_ID_MSI:
4341 #if AHCI_DEBUG
4342                         msimc = pci_config_get16(
4343                             ahci_ctlp->ahcictl_pci_conf_handle,
4344                             caps_ptr + PCI_MSI_CTRL);
4345                         AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4346                             "Message Signaled Interrupt capability found "
4347                             "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4348 #endif
4349                         AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4350                             "MSI capability found", NULL);
4351                         break;
4352 
4353                 case PCI_CAP_ID_PCIX:
4354                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4355                             "PCI-X capability found", NULL);
4356                         break;
4357 
4358                 case PCI_CAP_ID_PCI_E:
4359                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4360                             "PCI Express capability found", NULL);
4361                         break;
4362 
4363                 case PCI_CAP_ID_MSI_X:
4364                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4365                             "MSI-X capability found", NULL);
4366                         break;
4367 
4368                 case PCI_CAP_ID_SATA:
4369                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4370                             "SATA capability found", NULL);
4371                         break;
4372 
4373                 case PCI_CAP_ID_VS:
4374                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4375                             "Vendor Specific capability found", NULL);
4376                         break;
4377 
4378                 default:
4379                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4380                             "unrecognized capability 0x%x", cap);
4381                         break;
4382                 }
4383 
4384                 /*
4385                  * Get next capabilities pointer and clear bits 0,1.
4386                  */
4387                 caps_ptr = P2ALIGN(pci_config_get8(
4388                     ahci_ctlp->ahcictl_pci_conf_handle,
4389                     (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4390         }
4391 
4392         return (AHCI_SUCCESS);
4393 }
4394 
4395 /*
4396  * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4397  * PORTMULT command. SYNC & POLLING mode is used.
4398  */
4399 static int
4400 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4401     uint8_t regn, uint32_t *pregv, uint8_t type)
4402 {
4403         ahci_port_t *ahci_portp;
4404         ahci_addr_t pmult_addr;
4405         sata_pkt_t *spkt;
4406         sata_cmd_t *scmd;
4407         sata_device_t sata_device;
4408         uint8_t port = addrp->aa_port;
4409         uint8_t pmport = addrp->aa_pmport;
4410         uint8_t cport;
4411         uint32_t intr_mask;
4412         int rval;
4413         char portstr[10];
4414 
4415         SET_PORTSTR(portstr, addrp);
4416         cport = ahci_ctlp->ahcictl_port_to_cport[port];
4417         ahci_portp = ahci_ctlp->ahcictl_ports[port];
4418 
4419         ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4420         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4421 
4422         /* Check the existence of the port multiplier */
4423         if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4424                 return (AHCI_FAILURE);
4425 
4426         /* Request a READ/WRITE PORTMULT sata packet. */
4427         bzero(&sata_device, sizeof (sata_device_t));
4428         sata_device.satadev_addr.cport = cport;
4429         sata_device.satadev_addr.pmport = pmport;
4430         sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4431         sata_device.satadev_rev = SATA_DEVICE_REV;
4432 
4433         /*
4434          * Make sure no command is outstanding here. All R/W PMULT requests
4435          * come from
4436          *
4437          * 1. ahci_attach()
4438          *    The port should be empty.
4439          *
4440          * 2. ahci_tran_probe_port()
4441          *    Any request from SATA framework (via ahci_tran_start) should be
4442          *    rejected if R/W PMULT command is outstanding.
4443          *
4444          *    If we are doing mopping, do not check those flags because no
4445          *    command will be actually outstanding.
4446          *
4447          *    If the port has been occupied by any other commands, the probe
4448          *    function will return a SATA_RETRY. SATA framework will retry
4449          *    later.
4450          */
4451         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4452                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4453                     "R/W PMULT failed: R/W PMULT in progress at port %d.",
4454                     port, ahci_portp->ahciport_flags);
4455                 return (AHCI_FAILURE);
4456         }
4457 
4458         if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4459             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4460             NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4461             NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4462                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4463                     "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4464                     port, ahci_portp->ahciport_flags);
4465                 return (AHCI_FAILURE);
4466         }
4467 
4468         /*
4469          * The port multiplier is gone. This may happen when
4470          * 1. Cutting off the power of an enclosure. The device lose the power
4471          *    before port multiplier.
4472          * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4473          *
4474          * The issued command should be aborted and the following command
4475          * should not be continued.
4476          */
4477         if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4478                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4479                     "READ/WRITE PMULT failed: "
4480                     "port-mult is removed from port %d", port);
4481                 return (AHCI_FAILURE);
4482         }
4483 
4484         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4485 
4486         spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4487             &sata_device, regn, *pregv, type);
4488 
4489         /*
4490          * READ/WRITE PORTMULT command is intended to sent to the control port
4491          * of the port multiplier.
4492          */
4493         AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4494 
4495         ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4496 
4497         /* No interrupt here. Store the interrupt enable mask. */
4498         intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4499             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4500         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4501             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4502 
4503         rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4504 
4505         if (rval == AHCI_SUCCESS &&
4506             spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4507                 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4508                         scmd = &spkt->satapkt_cmd;
4509                         *pregv = scmd->satacmd_lba_high_lsb << 24 |
4510                             scmd->satacmd_lba_mid_lsb << 16 |
4511                             scmd->satacmd_lba_low_lsb << 8 |
4512                             scmd->satacmd_sec_count_lsb;
4513                 }
4514         } else {
4515                 /* Failed or not completed. */
4516                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4517                     "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4518                     type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4519                     AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4520                 rval = AHCI_FAILURE;
4521         }
4522 out:
4523         /* Restore the interrupt mask */
4524         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4525             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4526 
4527         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4528         ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4529         sata_free_rdwr_pmult_pkt(spkt);
4530         return (rval);
4531 }
4532 
4533 static int
4534 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4535     uint8_t regn, uint32_t *pregv)
4536 {
4537         return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4538             SATA_RDWR_PMULT_PKT_TYPE_READ);
4539 }
4540 
4541 static int
4542 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4543     uint8_t regn, uint32_t regv)
4544 {
4545         return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, &regv,
4546             SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4547 }
4548 
4549 #define READ_PMULT(addrp, r, pv, out)                                   \
4550         if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS)   \
4551                 goto out;
4552 
4553 #define WRITE_PMULT(addrp, r, v, out)                                   \
4554         if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS)   \
4555                 goto out;
4556 
4557 /*
4558  * Update sata registers on port multiplier, including GSCR/PSCR registers.
4559  * ahci_update_pmult_gscr()
4560  * ahci_update_pmult_pscr()
4561  */
4562 static int
4563 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4564     sata_pmult_gscr_t *sg)
4565 {
4566         ASSERT(MUTEX_HELD(
4567             &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4568 
4569         READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4570         READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4571         READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4572         READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4573 
4574         return (AHCI_SUCCESS);
4575 
4576 err:    /* R/W PMULT error */
4577         return (AHCI_FAILURE);
4578 }
4579 
4580 static int
4581 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4582     sata_device_t *sd)
4583 {
4584         ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4585         ASSERT(MUTEX_HELD(
4586             &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4587 
4588         READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4589         READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4590         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4591         READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4592 
4593         return (AHCI_SUCCESS);
4594 
4595 err:    /* R/W PMULT error */
4596         return (AHCI_FAILURE);
4597 }
4598 
4599 /*
4600  * ahci_initialize_pmult()
4601  *
4602  * Initialize a port multiplier, including
4603  * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4604  * 2. Redefine MASK register. (SATA Chap 16.?)
4605  */
4606 static int
4607 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4608     ahci_addr_t *addrp, sata_device_t *sd)
4609 {
4610         sata_pmult_gscr_t sg;
4611         uint32_t gscr64;
4612         uint8_t port = addrp->aa_port;
4613 
4614         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4615 
4616         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4617             "[Initialize] Port-multiplier at port %d.", port);
4618 
4619         /*
4620          * Enable features of port multiplier. Currently only
4621          * Asynchronous Notification is enabled.
4622          */
4623         /* Check gscr64 for supported features. */
4624         READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4625 
4626         if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4627                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4628                     "port %d: Port Multiplier supports "
4629                     "Asynchronous Notification.", port);
4630 
4631                 /* Write to gscr96 to enabled features */
4632                 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4633                     SATA_PMULT_CAP_SNOTIF, err);
4634 
4635                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4636                     (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4637                     AHCI_SNOTIF_CLEAR_ALL);
4638                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4639                     "port %d: PMult PxSNTF cleared.", port);
4640 
4641         }
4642 
4643         /*
4644          * Now we need to update gscr33 register to enable hot-plug interrupt
4645          * for sub devices behind port multiplier.
4646          */
4647         WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4648         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4649             "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4650 
4651         /*
4652          * Fetch the number of device ports of the port multiplier
4653          */
4654         if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4655                 return (AHCI_FAILURE);
4656 
4657         /* Register the port multiplier to SATA Framework. */
4658         mutex_exit(&ahci_portp->ahciport_mutex);
4659         sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4660         mutex_enter(&ahci_portp->ahciport_mutex);
4661 
4662         ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4663             sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4664 
4665         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4666             "port %d: pmult sub-port number updated to %x.", port,
4667             ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4668 
4669         /* Till now port-mult is successfully initialized */
4670         ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4671         return (AHCI_SUCCESS);
4672 
4673 err:    /* R/W PMULT error */
4674         return (AHCI_FAILURE);
4675 }
4676 
4677 /*
4678  * Initialize a port multiplier port. According to spec, firstly we need
4679  * issue a COMRESET, then a software reset to get its signature.
4680  *
4681  * NOTE: This function should only be called in ahci_probe_pmport()
4682  */
4683 static int
4684 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4685     ahci_addr_t *addrp)
4686 {
4687         uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4688         uint8_t port = addrp->aa_port;
4689         uint8_t pmport = addrp->aa_pmport;
4690         int ret = AHCI_FAILURE;
4691 
4692         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4693         ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4694 
4695         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4696             "ahci_initialize_pmport: port %d:%d", port, pmport);
4697 
4698         /* Check HBA port state */
4699         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4700                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4701                     "ahci_initialize_pmport:"
4702                     "port %d:%d Port Multiplier is failed.",
4703                     port, pmport);
4704                 return (AHCI_FAILURE);
4705         }
4706 
4707         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4708                 return (AHCI_FAILURE);
4709         }
4710         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4711 
4712         /* Checking for outstanding commands */
4713         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4714                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4715                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4716                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4717         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4718                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4719                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4720                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4721         }
4722 
4723         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4724         ahci_portp->ahciport_mop_in_progress++;
4725 
4726         /* Clear status */
4727         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4728 
4729         /* Firstly assume an unknown device */
4730         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4731 
4732         ahci_disable_port_intrs(ahci_ctlp, port);
4733 
4734         /* port reset is necessary for port multiplier port */
4735         if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4736                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4737                     "ahci_initialize_pmport:"
4738                     "port reset failed at port %d:%d",
4739                     port, pmport);
4740                 goto out;
4741         }
4742 
4743         /* Is port failed? */
4744         if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4745             SATA_PSTATE_FAILED) {
4746                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4747                     "ahci_initialize_pmport: port %d:%d failed. "
4748                     "state = 0x%x", port, pmport,
4749                     ahci_portp->ahciport_port_state);
4750                 goto out;
4751         }
4752 
4753         /* Is there any device attached? */
4754         if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4755             == SATA_DTYPE_NONE) {
4756                 /* Do not waste time on an empty port */
4757                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4758                     "ahci_initialize_pmport: No device is found "
4759                     "at port %d:%d", port, pmport);
4760                 ret = AHCI_SUCCESS;
4761                 goto out;
4762         }
4763 
4764         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4765         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4766             "port %d:%d is ready now.", port, pmport);
4767 
4768         /*
4769          * Till now we can assure a device attached to that HBA port and work
4770          * correctly. Now try to get the device signature. This is an optional
4771          * step. If failed, unknown device is assumed, then SATA module will
4772          * continue to use IDENTIFY DEVICE to get the information of the
4773          * device.
4774          */
4775         ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4776 
4777         ret = AHCI_SUCCESS;
4778 
4779 out:
4780         /* Next try to mop the pending commands */
4781         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4782                 finished_tags = ahci_portp->ahciport_pending_tags &
4783                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4784         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4785                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4786                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4787         reset_tags &= ~finished_tags;
4788 
4789         ahci_mop_commands(ahci_ctlp,
4790             ahci_portp,
4791             slot_status,
4792             0, /* failed tags */
4793             0, /* timeout tags */
4794             0, /* aborted tags */
4795             reset_tags); /* reset tags */
4796 
4797         /* Clear PxSNTF register if supported. */
4798         if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4799                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4800                     (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4801                     AHCI_SNOTIF_CLEAR_ALL);
4802         }
4803 
4804         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4805         ahci_enable_port_intrs(ahci_ctlp, port);
4806         return (ret);
4807 }
4808 
4809 /*
4810  * ahci_probe_pmult()
4811  *
4812  * This function will be called to probe a port multiplier, which will
4813  * handle hotplug events on port multiplier ports.
4814  *
4815  * NOTE: Only called from ahci_tran_probe_port()
4816  */
4817 static int
4818 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4819     ahci_addr_t *addrp)
4820 {
4821         sata_device_t sdevice;
4822         ahci_addr_t pmport_addr;
4823         uint32_t gscr32, port_hotplug_tags;
4824         uint32_t pmport_sstatus;
4825         int dev_exists_now = 0, dev_existed_previously = 0;
4826         uint8_t port = addrp->aa_port;
4827         int npmport;
4828 
4829         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4830 
4831         /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4832         READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4833         port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4834 
4835         do {
4836                 npmport = ddi_ffs(port_hotplug_tags) - 1;
4837                 if (npmport == -1)
4838                         /* no pending hot plug events. */
4839                         return (AHCI_SUCCESS);
4840 
4841                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4842                     "hot-plug event at port %d:%d", port, npmport);
4843 
4844                 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4845 
4846                 /* Check previous device at that port */
4847                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4848                     != SATA_DTYPE_NONE)
4849                         dev_existed_previously = 1;
4850 
4851                 /* PxSStatus tells the presence of device. */
4852                 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4853                     &pmport_sstatus, err);
4854 
4855                 if (SSTATUS_GET_DET(pmport_sstatus) ==
4856                     SSTATUS_DET_DEVPRE_PHYCOM)
4857                         dev_exists_now = 1;
4858 
4859                 /*
4860                  * Clear PxSERR is critical. The transition from 0 to 1 will
4861                  * emit a FIS which generates an asynchronous notification
4862                  * event at controller. If we fail to clear the PxSERR, the
4863                  * Async Notif events will no longer be activated on this
4864                  * pmport.
4865                  */
4866                 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4867                     AHCI_SERROR_CLEAR_ALL, err);
4868 
4869                 bzero((void *)&sdevice, sizeof (sata_device_t));
4870                 sdevice.satadev_addr.cport = ahci_ctlp->
4871                     ahcictl_port_to_cport[port];
4872                 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4873                 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4874                 sdevice.satadev_state = SATA_PSTATE_PWRON;
4875 
4876                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4877                     "[Existence] %d -> %d", dev_existed_previously,
4878                     dev_exists_now);
4879 
4880                 if (dev_exists_now) {
4881                         if (dev_existed_previously) {
4882                                 /* Link (may) not change: Exist -> Exist * */
4883                                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4884                                     "ahci_probe_pmult: port %d:%d "
4885                                     "device link lost/established",
4886                                     port, npmport);
4887 
4888                                 mutex_exit(&ahci_portp->ahciport_mutex);
4889                                 sata_hba_event_notify(
4890                                     ahci_ctlp->ahcictl_sata_hba_tran->
4891                                     sata_tran_hba_dip,
4892                                     &sdevice,
4893                                     SATA_EVNT_LINK_LOST|
4894                                     SATA_EVNT_LINK_ESTABLISHED);
4895                                 mutex_enter(&ahci_portp->ahciport_mutex);
4896                         } else {
4897                                 /* Link change: None -> Exist */
4898                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4899                                     "ahci_probe_pmult: port %d:%d "
4900                                     "device link established", port, npmport);
4901 
4902                                 /* Clear port state */
4903                                 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4904                                     SATA_STATE_UNKNOWN);
4905                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4906                                     "ahci_probe_pmult: port %d "
4907                                     "ahciport_port_state [Cleared].", port);
4908 
4909                                 mutex_exit(&ahci_portp->ahciport_mutex);
4910                                 sata_hba_event_notify(
4911                                     ahci_ctlp->ahcictl_sata_hba_tran->
4912                                     sata_tran_hba_dip,
4913                                     &sdevice,
4914                                     SATA_EVNT_LINK_ESTABLISHED);
4915                                 mutex_enter(&ahci_portp->ahciport_mutex);
4916                         }
4917                 } else { /* No device exists now */
4918                         if (dev_existed_previously) {
4919 
4920                                 /* Link change: Exist -> None */
4921                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4922                                     "ahci_probe_pmult: port %d:%d "
4923                                     "device link lost", port, npmport);
4924 
4925                                 /* An existing device is lost. */
4926                                 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4927                                     SATA_STATE_UNKNOWN);
4928                                 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4929                                     SATA_DTYPE_NONE);
4930 
4931                                 mutex_exit(&ahci_portp->ahciport_mutex);
4932                                 sata_hba_event_notify(
4933                                     ahci_ctlp->ahcictl_sata_hba_tran->
4934                                     sata_tran_hba_dip,
4935                                     &sdevice,
4936                                     SATA_EVNT_LINK_LOST);
4937                                 mutex_enter(&ahci_portp->ahciport_mutex);
4938                         }
4939                 }
4940 
4941                 CLEAR_BIT(port_hotplug_tags, npmport);
4942         } while (port_hotplug_tags != 0);
4943 
4944         return (AHCI_SUCCESS);
4945 
4946 err:    /* R/W PMULT error */
4947         return (AHCI_FAILURE);
4948 }
4949 
4950 /*
4951  * Probe and initialize a port multiplier port.
4952  * A port multiplier port could only be initilaizer here.
4953  */
4954 static int
4955 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4956     ahci_addr_t *addrp, sata_device_t *sd)
4957 {
4958         uint32_t port_state;
4959         uint8_t port = addrp->aa_port;
4960         ahci_addr_t addr_pmult;
4961 
4962         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4963 
4964         /*
4965          * Check the parent - port multiplier first.
4966          */
4967 
4968         /*
4969          * Parent port multiplier might have been removed. This event will be
4970          * ignored and failure.
4971          */
4972         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4973             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4974                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4975                     "ahci_tran_probe_port: "
4976                     "parent device removed, ignore event.", NULL);
4977 
4978                 return (AHCI_FAILURE);
4979         }
4980 
4981         /* The port is ready? */
4982         port_state = ahci_portp->ahciport_port_state;
4983         if (!(port_state & SATA_STATE_READY)) {
4984                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4985                     "ahci_tran_probe_port: "
4986                     "parent port-mult is NOT ready.", NULL);
4987 
4988                 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4989                     ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4990                     AHCI_SUCCESS) {
4991                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4992                             "ahci_tran_probe_port: "
4993                             "restart port-mult failed.", NULL);
4994                         return (AHCI_FAILURE);
4995                 }
4996         }
4997 
4998         /*
4999          * If port-mult is restarted due to some reason, we need
5000          * re-initialized the PMult.
5001          */
5002         if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
5003                 /* Initialize registers on a port multiplier */
5004                 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
5005                 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
5006                     &addr_pmult, sd) != AHCI_SUCCESS)
5007                         return (AHCI_FAILURE);
5008         }
5009 
5010         /*
5011          * Then we check the port-mult port
5012          */
5013         /* Is this pmport initialized? */
5014         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5015         if (!(port_state & SATA_STATE_READY)) {
5016 
5017                 /* ahci_initialize_pmport() will set READY state */
5018                 if (ahci_initialize_pmport(ahci_ctlp,
5019                     ahci_portp, addrp) != AHCI_SUCCESS)
5020                         return (AHCI_FAILURE);
5021         }
5022 
5023         return (AHCI_SUCCESS);
5024 }
5025 
5026 /*
5027  * AHCI device reset ...; a single device on one of the ports is reset,
5028  * but the HBA and physical communication remain intact. This is the
5029  * least intrusive.
5030  *
5031  * When issuing a software reset sequence, there should not be other
5032  * commands in the command list, so we will first clear and then re-set
5033  * PxCMD.ST to clear PxCI. And before issuing the software reset,
5034  * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
5035  * cleared unless command list override (PxCMD.CLO) is supported.
5036  */
5037 static int
5038 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5039     ahci_addr_t *addrp)
5040 {
5041         ahci_fis_h2d_register_t *h2d_register_fisp;
5042         ahci_cmd_table_t *cmd_table;
5043         ahci_cmd_header_t *cmd_header;
5044         uint32_t port_cmd_status, port_cmd_issue, port_task_file;
5045         int slot, loop_count;
5046         uint8_t port = addrp->aa_port;
5047         uint8_t pmport = addrp->aa_pmport;
5048         int rval = AHCI_FAILURE;
5049 
5050         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5051 
5052         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5053             "port %d:%d device software resetting (FIS)", port, pmport);
5054 
5055         /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
5056         if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5057             port) != AHCI_SUCCESS) {
5058                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5059                     "ahci_software_reset: cannot stop HBA port %d.", port);
5060                 goto out;
5061         }
5062 
5063         /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
5064         port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5065             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5066 
5067         if (port_task_file & AHCI_TFD_STS_BSY ||
5068             port_task_file & AHCI_TFD_STS_DRQ) {
5069                 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
5070                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5071                             "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
5072                             "cannot issue a software reset.", port_task_file);
5073                         goto out;
5074                 }
5075 
5076                 /*
5077                  * If HBA Support CLO, as Command List Override (CAP.SCLO is
5078                  * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
5079                  * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
5080                  */
5081                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5082                     "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
5083 
5084                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5085                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5086                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5087                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5088                     port_cmd_status|AHCI_CMD_STATUS_CLO);
5089 
5090                 /* Waiting till PxCMD.SCLO bit is cleared */
5091                 loop_count = 0;
5092                 do {
5093                         /* Wait for 10 millisec */
5094                         drv_usecwait(AHCI_10MS_USECS);
5095 
5096                         /* We are effectively timing out after 1 sec. */
5097                         if (loop_count++ > 100) {
5098                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5099                                     "SCLO time out. port %d is busy.", port);
5100                                 goto out;
5101                         }
5102 
5103                         port_cmd_status =
5104                             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5105                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5106                 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5107 
5108                 /* Re-check */
5109                 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5110                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5111                 if (port_task_file & AHCI_TFD_STS_BSY ||
5112                     port_task_file & AHCI_TFD_STS_DRQ) {
5113                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5114                             "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5115                             port_task_file);
5116                         goto out;
5117                 }
5118         }
5119 
5120         /* Then start port */
5121         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5122             != AHCI_SUCCESS) {
5123                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5124                     "ahci_software_reset: cannot start AHCI port %d.", port);
5125                 goto out;
5126         }
5127 
5128         /*
5129          * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5130          * ahci_port.ahciport_pending_ncq_tags may fail
5131          * ahci_claim_free_slot(). Actually according to spec, by clearing
5132          * PxCMD.ST there is no command outstanding while executing software
5133          * reseting. Hence we directly use slot 0 instead of
5134          * ahci_claim_free_slot().
5135          */
5136         slot = 0;
5137 
5138         /* Now send the first H2D Register FIS with SRST set to 1 */
5139         cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5140         bzero((void *)cmd_table, ahci_cmd_table_size);
5141 
5142         h2d_register_fisp =
5143             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5144 
5145         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5146         SET_FIS_PMP(h2d_register_fisp, pmport);
5147         SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5148 
5149         /* Set Command Header in Command List */
5150         cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5151         BZERO_DESCR_INFO(cmd_header);
5152         BZERO_PRD_BYTE_COUNT(cmd_header);
5153         SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5154         SET_PORT_MULTI_PORT(cmd_header, pmport);
5155 
5156         SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5157         SET_RESET(cmd_header, 1);
5158         SET_WRITE(cmd_header, 1);
5159 
5160         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5161             0,
5162             ahci_cmd_table_size,
5163             DDI_DMA_SYNC_FORDEV);
5164 
5165         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5166             slot * sizeof (ahci_cmd_header_t),
5167             sizeof (ahci_cmd_header_t),
5168             DDI_DMA_SYNC_FORDEV);
5169 
5170         /* Indicate to the HBA that a command is active. */
5171         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5172             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5173             (0x1 << slot));
5174 
5175         loop_count = 0;
5176 
5177         /* Loop till the first command is finished */
5178         do {
5179                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5180                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5181 
5182                 /* We are effectively timing out after 1 sec. */
5183                 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5184                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5185                             "the first SRST FIS is timed out, "
5186                             "loop_count = %d", loop_count);
5187                         goto out;
5188                 }
5189                 /* Wait for 10 millisec */
5190                 drv_usecwait(AHCI_10MS_USECS);
5191         } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5192 
5193         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5194             "ahci_software_reset: 1st loop count: %d, "
5195             "port_cmd_issue = 0x%x, slot = 0x%x",
5196             loop_count, port_cmd_issue, slot);
5197 
5198         /* According to ATA spec, we need wait at least 5 microsecs here. */
5199         drv_usecwait(AHCI_1MS_USECS);
5200 
5201         /* Now send the second H2D Register FIS with SRST cleard to zero */
5202         cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5203         bzero((void *)cmd_table, ahci_cmd_table_size);
5204 
5205         h2d_register_fisp =
5206             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5207 
5208         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5209         SET_FIS_PMP(h2d_register_fisp, pmport);
5210 
5211         /* Set Command Header in Command List */
5212         cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5213         BZERO_DESCR_INFO(cmd_header);
5214         BZERO_PRD_BYTE_COUNT(cmd_header);
5215         SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5216         SET_PORT_MULTI_PORT(cmd_header, pmport);
5217 
5218         SET_WRITE(cmd_header, 1);
5219 
5220         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5221             0,
5222             ahci_cmd_table_size,
5223             DDI_DMA_SYNC_FORDEV);
5224 
5225         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5226             slot * sizeof (ahci_cmd_header_t),
5227             sizeof (ahci_cmd_header_t),
5228             DDI_DMA_SYNC_FORDEV);
5229 
5230         /* Indicate to the HBA that a command is active. */
5231         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5232             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5233             (0x1 << slot));
5234 
5235         loop_count = 0;
5236 
5237         /* Loop till the second command is finished */
5238         do {
5239                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5240                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5241 
5242                 /* We are effectively timing out after 1 sec. */
5243                 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5244                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5245                             "the second SRST FIS is timed out, "
5246                             "loop_count = %d", loop_count);
5247                         goto out;
5248                 }
5249 
5250                 /* Wait for 10 millisec */
5251                 drv_usecwait(AHCI_10MS_USECS);
5252         } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5253 
5254         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5255             "ahci_software_reset: 2nd loop count: %d, "
5256             "port_cmd_issue = 0x%x, slot = 0x%x",
5257             loop_count, port_cmd_issue, slot);
5258 
5259         if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5260             (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5261                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5262                     DDI_SERVICE_UNAFFECTED);
5263                 goto out;
5264         }
5265 
5266         rval = AHCI_SUCCESS;
5267 out:
5268         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5269             "ahci_software_reset: %s at port %d:%d",
5270             rval == AHCI_SUCCESS ? "succeed" : "failed",
5271             port, pmport);
5272 
5273         return (rval);
5274 }
5275 
5276 /*
5277  * AHCI port reset ...; the physical communication between the HBA and device
5278  * on a port are disabled. This is more intrusive.
5279  *
5280  * When an HBA or port reset occurs, Phy communication is going to
5281  * be re-established with the device through a COMRESET followed by the
5282  * normal out-of-band communication sequence defined in Serial ATA. At
5283  * the end of reset, the device, if working properly, will send a D2H
5284  * Register FIS, which contains the device signature. When the HBA receives
5285  * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5286  * the PxSIG register with the signature.
5287  *
5288  * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5289  * function is called. If not, it is assumed the interface is in hung
5290  * condition.
5291  */
5292 static int
5293 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5294     ahci_addr_t *addrp)
5295 {
5296         ahci_addr_t pmult_addr;
5297         uint32_t port_cmd_status;
5298         uint32_t port_scontrol, port_sstatus;
5299         uint32_t port_task_file;
5300         uint32_t port_state;
5301         uint8_t port = addrp->aa_port;
5302 
5303         int loop_count;
5304         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5305 
5306         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5307 
5308         /* Target is a port multiplier port? */
5309         if (AHCI_ADDR_IS_PMPORT(addrp))
5310                 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5311 
5312         /* Otherwise it must be an HBA port. */
5313         ASSERT(AHCI_ADDR_IS_PORT(addrp));
5314 
5315         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5316             "Port %d port resetting...", port);
5317         ahci_portp->ahciport_port_state = 0;
5318 
5319         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5320             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5321 
5322         /*
5323          * According to the spec, SUD bit should be set here,
5324          * but JMicron JMB363 doesn't follow it, so print
5325          * a debug message.
5326          */
5327         if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5328                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5329                     "ahci_port_reset: port %d SUD bit not set", port);
5330 
5331         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5332             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5333         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5334 
5335         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5336             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5337             port_scontrol);
5338 
5339         /* Enable PxCMD.FRE to read device */
5340         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5341             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5342             port_cmd_status|AHCI_CMD_STATUS_FRE);
5343 
5344         /*
5345          * The port enters P:StartComm state, and the HBA tells the link layer
5346          * to start communication, which involves sending COMRESET to the
5347          * device. And the HBA resets PxTFD.STS to 7Fh.
5348          *
5349          * Give time for COMRESET to percolate, according to the AHCI
5350          * spec, software shall wait at least 1 millisecond before
5351          * clearing PxSCTL.DET
5352          */
5353         drv_usecwait(AHCI_1MS_USECS * 2);
5354 
5355         /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5356         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5357             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5358         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5359         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5360             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5361             port_scontrol);
5362 
5363         /*
5364          * When a COMINIT is received from the device, then the port enters
5365          * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5366          * PxSSTS.DET to 1h to indicate a device is detected but communication
5367          * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5368          * a COMINIT has been received.
5369          */
5370         /*
5371          * The DET field is valid only if IPM field indicates
5372          * that the interface is in active state.
5373          */
5374         loop_count = 0;
5375         for (;;) {
5376                 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5377                     (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5378 
5379                 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5380                         /*
5381                          * If the interface is not active, the DET field
5382                          * is considered not accurate. So we want to
5383                          * continue looping.
5384                          */
5385                         SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5386                 }
5387 
5388                 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5389                         break;
5390 
5391                 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5392                         /*
5393                          * We are effectively timing out after 0.1 sec.
5394                          */
5395                         break;
5396                 }
5397 
5398                 /* Wait for 10 millisec */
5399                 drv_usecwait(AHCI_10MS_USECS);
5400         }
5401 
5402         AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5403             "ahci_port_reset: 1st loop count: %d, "
5404             "port_sstatus = 0x%x port %d",
5405             loop_count, port_sstatus, port);
5406 
5407         if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5408                 /*
5409                  * Either the port is not active or there
5410                  * is no device present.
5411                  */
5412                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5413                 return (AHCI_SUCCESS);
5414         }
5415 
5416         /* Clear port serror register for the port */
5417         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5418             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5419             AHCI_SERROR_CLEAR_ALL);
5420 
5421         /*
5422          * Devices should return a FIS contains its signature to HBA after
5423          * COMINIT signal. Check whether a D2H Register FIS is received by
5424          * polling PxTFD.STS.
5425          */
5426         loop_count = 0;
5427         for (;;) {
5428                 port_task_file =
5429                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5430                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5431 
5432                 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5433                     AHCI_TFD_STS_ERR)) == 0)
5434                         break;
5435 
5436                 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5437                         /*
5438                          * We are effectively timing out after 11 sec.
5439                          */
5440                         cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5441                             "the device hardware has been initialized and "
5442                             "the power-up diagnostics failed",
5443                             instance, port);
5444 
5445                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5446                             "port %d: some or all of BSY, DRQ and ERR in "
5447                             "PxTFD.STS are not clear. We need another "
5448                             "software reset.", port);
5449 
5450                         /* Clear port serror register for the port */
5451                         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5452                             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5453                             AHCI_SERROR_CLEAR_ALL);
5454 
5455                         AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5456 
5457                         /* Try another software reset. */
5458                         if (ahci_software_reset(ahci_ctlp, ahci_portp,
5459                             &pmult_addr) != AHCI_SUCCESS) {
5460                                 AHCIPORT_SET_STATE(ahci_portp, addrp,
5461                                     SATA_PSTATE_FAILED);
5462                                 return (AHCI_FAILURE);
5463                         }
5464                         break;
5465                 }
5466 
5467                 /* Wait for 10 millisec */
5468                 drv_usecwait(AHCI_10MS_USECS);
5469         }
5470 
5471         AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5472             "ahci_port_reset: 2nd loop count: %d, "
5473             "port_task_file = 0x%x port %d",
5474             loop_count, port_task_file, port);
5475 
5476         /* Clear port serror register for the port */
5477         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5478             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5479             AHCI_SERROR_CLEAR_ALL);
5480 
5481         /* Set port as ready */
5482         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5483         AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5484 
5485         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5486             "ahci_port_reset: succeed at port %d.", port);
5487         return (AHCI_SUCCESS);
5488 }
5489 
5490 /*
5491  * COMRESET on a port multiplier port.
5492  *
5493  * NOTE: Only called in ahci_port_reset()
5494  */
5495 static int
5496 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5497     ahci_addr_t *addrp)
5498 {
5499         uint32_t port_scontrol, port_sstatus, port_serror;
5500         uint32_t port_cmd_status, port_intr_status;
5501         uint32_t port_state;
5502         uint8_t port = addrp->aa_port;
5503         uint8_t pmport = addrp->aa_pmport;
5504         int loop_count;
5505         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5506 
5507         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5508             "port %d:%d: pmport resetting", port, pmport);
5509 
5510         /* Initialize pmport state */
5511         AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5512 
5513         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5514         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5515         WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5516 
5517         /* PxCMD.FRE should be set before. */
5518         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5519             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5520         ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5521         if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5522                 return (AHCI_FAILURE);
5523 
5524         /*
5525          * Give time for COMRESET to percolate, according to the AHCI
5526          * spec, software shall wait at least 1 millisecond before
5527          * clearing PxSCTL.DET
5528          */
5529         drv_usecwait(AHCI_1MS_USECS*2);
5530 
5531         /*
5532          * Fetch the SCONTROL again and rewrite the DET part with 0
5533          * This will generate an Asychronous Notification events.
5534          */
5535         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5536         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5537         WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5538 
5539         /*
5540          * The port enters P:StartComm state, and HBA tells link layer to
5541          * start communication, which involves sending COMRESET to device.
5542          * And the HBA resets PxTFD.STS to 7Fh.
5543          *
5544          * When a COMINIT is received from the device, then the port enters
5545          * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5546          * PxSSTS.DET to 1h to indicate a device is detected but communication
5547          * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5548          * a COMINIT has been received.
5549          */
5550         /*
5551          * The DET field is valid only if IPM field indicates
5552          * that the interface is in active state.
5553          */
5554         loop_count = 0;
5555         do {
5556                 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5557 
5558                 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5559                         /*
5560                          * If the interface is not active, the DET field
5561                          * is considered not accurate. So we want to
5562                          * continue looping.
5563                          */
5564                         SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5565                 }
5566 
5567                 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5568                         /*
5569                          * We are effectively timing out after 0.1 sec.
5570                          */
5571                         break;
5572                 }
5573 
5574                 /* Wait for 10 millisec */
5575                 drv_usecwait(AHCI_10MS_USECS);
5576 
5577         } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5578 
5579         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5580             "ahci_pmport_reset: 1st loop count: %d, "
5581             "port_sstatus = 0x%x port %d:%d",
5582             loop_count, port_sstatus, port, pmport);
5583 
5584         if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5585             (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5586                 /*
5587                  * Either the port is not active or there
5588                  * is no device present.
5589                  */
5590                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5591                     "ahci_pmport_reset: "
5592                     "no device attached to port %d:%d",
5593                     port, pmport);
5594                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5595                 return (AHCI_SUCCESS);
5596         }
5597 
5598         /* Now we can make sure there is a device connected to the port */
5599         /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5600         READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5601 
5602         if (!(port_serror & (1 << 26))) {
5603                 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5604                     "COMINIT signal from the device not received port %d:%d",
5605                     instance, port, pmport);
5606 
5607                 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5608                 return (AHCI_FAILURE);
5609         }
5610 
5611         /*
5612          * After clear PxSERR register, we will receive a D2H FIS.
5613          * Normally this FIS will cause a IPMS error according to AHCI spec
5614          * v1.2 because there is no command outstanding for it. So we need
5615          * to ignore this error.
5616          */
5617         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5618         WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5619 
5620         /* Now we need to check the D2H FIS by checking IPMS error. */
5621         loop_count = 0;
5622         do {
5623                 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5624                     (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5625 
5626                 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5627                         /*
5628                          * No D2H FIS received. This is possible according
5629                          * to SATA 2.6 spec.
5630                          */
5631                         cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5632                             "PxIS.IPMS is not set, we need another "
5633                             "software reset.", port, pmport);
5634 
5635                         break;
5636                 }
5637 
5638                 /* Wait for 10 millisec */
5639                 mutex_exit(&ahci_portp->ahciport_mutex);
5640                 delay(AHCI_10MS_TICKS);
5641                 mutex_enter(&ahci_portp->ahciport_mutex);
5642 
5643         } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5644 
5645         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5646             "ahci_pmport_reset: 2st loop count: %d, "
5647             "port_sstatus = 0x%x port %d:%d",
5648             loop_count, port_sstatus, port, pmport);
5649 
5650         /* Clear IPMS */
5651         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5652             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5653             AHCI_INTR_STATUS_IPMS);
5654         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5655 
5656         /* This pmport is now ready for ahci_tran_start() */
5657         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5658         AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5659 
5660         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5661             "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5662         return (AHCI_SUCCESS);
5663 
5664 err:    /* R/W PMULT error */
5665         /* IPMS flags might be set before. */
5666         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5667         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5668             "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5669 
5670         return (AHCI_FAILURE);
5671 }
5672 
5673 /*
5674  * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5675  * This is the most intrusive.
5676  *
5677  * When an HBA reset occurs, Phy communication will be re-established with
5678  * the device through a COMRESET followed by the normal out-of-band
5679  * communication sequence defined in Serial ATA. At the end of reset, the
5680  * device, if working properly, will send a D2H Register FIS, which contains
5681  * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5682  * and PxTFD.ERR register fields, and updates the PxSIG register with the
5683  * signature.
5684  *
5685  * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5686  */
5687 static int
5688 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5689 {
5690         ahci_port_t *ahci_portp;
5691         uint32_t ghc_control;
5692         uint8_t port;
5693         int loop_count;
5694         int rval = AHCI_SUCCESS;
5695 
5696         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5697             NULL);
5698 
5699         mutex_enter(&ahci_ctlp->ahcictl_mutex);
5700 
5701         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5702             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5703 
5704         /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5705         ghc_control |= AHCI_HBA_GHC_HR;
5706         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5707             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5708 
5709         /*
5710          * Wait until HBA Reset complete or timeout
5711          */
5712         loop_count = 0;
5713         do {
5714                 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5715                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5716 
5717                 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5718                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5719                             "ahci hba reset is timing out, "
5720                             "ghc_control = 0x%x", ghc_control);
5721                         /* We are effectively timing out after 1 sec. */
5722                         break;
5723                 }
5724 
5725                 /* Wait for 10 millisec */
5726                 drv_usecwait(AHCI_10MS_USECS);
5727         } while (ghc_control & AHCI_HBA_GHC_HR);
5728 
5729         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5730             "ahci_hba_reset: 1st loop count: %d, "
5731             "ghc_control = 0x%x", loop_count, ghc_control);
5732 
5733         if (ghc_control & AHCI_HBA_GHC_HR) {
5734                 /* The hba is not reset for some reasons */
5735                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5736                     "hba reset failed: HBA in a hung or locked state", NULL);
5737                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5738                 return (AHCI_FAILURE);
5739         }
5740 
5741         /*
5742          * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5743          */
5744         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5745             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5746         ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5747         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5748             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5749 
5750         mutex_exit(&ahci_ctlp->ahcictl_mutex);
5751 
5752         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5753                 /* Only check implemented ports */
5754                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5755                         continue;
5756                 }
5757 
5758                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5759                 mutex_enter(&ahci_portp->ahciport_mutex);
5760 
5761                 /* Make sure the drive is spun-up */
5762                 ahci_staggered_spin_up(ahci_ctlp, port);
5763 
5764                 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5765                     port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5766                     AHCI_SUCCESS) {
5767                         rval = AHCI_FAILURE;
5768                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5769                             "ahci_hba_reset: port %d failed", port);
5770                         /*
5771                          * Set the port state to SATA_PSTATE_FAILED if
5772                          * failed to initialize it.
5773                          */
5774                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5775                 }
5776 
5777                 mutex_exit(&ahci_portp->ahciport_mutex);
5778         }
5779 
5780         return (rval);
5781 }
5782 
5783 /*
5784  * This routine is only called from AHCI_ATTACH or phyrdy change
5785  * case. It first calls software reset, then stop the port and try to
5786  * read PxSIG register to find the type of device attached to the port.
5787  *
5788  * The caller should make sure a valid device exists on specified port and
5789  * physical communication has been established so that the signature could
5790  * be retrieved by software reset.
5791  *
5792  * NOTE: The port interrupts should be disabled before the function is called.
5793  */
5794 static void
5795 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5796     ahci_addr_t *addrp)
5797 {
5798         ahci_addr_t dev_addr;
5799         uint32_t signature;
5800         uint8_t port = addrp->aa_port;
5801         uint8_t pmport = addrp->aa_pmport;
5802         int rval;
5803 
5804         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5805         ASSERT(AHCI_ADDR_IS_VALID(addrp));
5806 
5807         /*
5808          * If the HBA doesn't support port multiplier, then the driver
5809          * doesn't need to bother to check port multiplier device.
5810          *
5811          * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5812          * connected to Silicon Image 4723, to which the two sata drives
5813          * attached can be set with RAID1, RAID0 or Spanning mode.
5814          *
5815          * We found software reset will get failure if port multiplier address
5816          * 0xf is used by software reset, so just ignore the check since
5817          * ICH7 doesn't support port multiplier device at all.
5818          */
5819         if (AHCI_ADDR_IS_PORT(addrp) &&
5820             (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5821                 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5822                     "ahci_find_dev_signature enter: port %d", port);
5823 
5824                 /*
5825                  * NOTE: when the ahci address is a HBA port, we do not know
5826                  * it is a device or a port multiplier that attached. we need
5827                  * try a software reset at port multiplier address (0xf
5828                  * pmport)
5829                  */
5830                 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5831         } else {
5832                 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5833                     "ahci_find_dev_signature enter: port %d:%d",
5834                     port, pmport);
5835                 dev_addr = *addrp;
5836         }
5837 
5838         /* Assume it is unknown. */
5839         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5840 
5841         /* Issue a software reset to get the signature */
5842         rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5843         if (rval != AHCI_SUCCESS) {
5844 
5845                 /*
5846                  * Try to do software reset again with pmport set with 0 if
5847                  * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5848                  * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5849                  */
5850                 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5851                     (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5852                         dev_addr.aa_pmport = 0;
5853                         rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5854                             &dev_addr);
5855                 }
5856 
5857                 if (rval != AHCI_SUCCESS) {
5858                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5859                             "ahci_find_dev_signature: software reset failed "
5860                             "at port %d:%d, cannot get signature.",
5861                             port, pmport);
5862 
5863                         AHCIPORT_SET_STATE(ahci_portp, addrp,
5864                             SATA_PSTATE_FAILED);
5865                         return;
5866                 }
5867         }
5868 
5869         /*
5870          * ahci_software_reset has started the port, so we need manually stop
5871          * the port again.
5872          */
5873         if (AHCI_ADDR_IS_PORT(addrp)) {
5874                 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5875                     ahci_portp, port) != AHCI_SUCCESS) {
5876                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5877                             "ahci_find_dev_signature: cannot stop port %d.",
5878                             port);
5879                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5880                         return;
5881                 }
5882         }
5883 
5884         /* Now we can make sure that a valid signature is received. */
5885         signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5886             (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5887 
5888         if (AHCI_ADDR_IS_PMPORT(addrp)) {
5889                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5890                     "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5891                     signature, port, pmport);
5892         } else {
5893                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5894                     "ahci_find_dev_signature: signature = 0x%x at port %d",
5895                     signature, port);
5896         }
5897 
5898         /* NOTE: Only support ATAPI device at controller port. */
5899         if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5900                 signature = SATA_DTYPE_UNKNOWN;
5901 
5902         switch (signature) {
5903 
5904         case AHCI_SIGNATURE_DISK:
5905                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5906                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5907                     "Disk is found at port: %d", port);
5908                 break;
5909 
5910         case AHCI_SIGNATURE_ATAPI:
5911                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5912                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5913                     "ATAPI device is found at port: %d", port);
5914                 break;
5915 
5916         case AHCI_SIGNATURE_PORT_MULTIPLIER:
5917                 /* Port Multiplier cannot recursively attached. */
5918                 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5919                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5920                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5921                     "Port Multiplier is found at port: %d", port);
5922                 break;
5923 
5924         default:
5925                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5926                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5927                     "Unknown device is found at port: %d", port);
5928         }
5929 }
5930 
5931 /*
5932  * According to the spec, to reliably detect hot plug removals, software
5933  * must disable interface power management. Software should perform the
5934  * following initialization on a port after a device is attached:
5935  *   Set PxSCTL.IPM to 3h to disable interface state transitions
5936  *   Set PxCMD.ALPE to '0' to disable aggressive power management
5937  *   Disable device initiated interface power management by SET FEATURE
5938  *
5939  * We can ignore the last item because by default the feature is disabled
5940  */
5941 static void
5942 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5943 {
5944         uint32_t port_scontrol, port_cmd_status;
5945 
5946         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5947             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5948         SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5949         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5950             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5951 
5952         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5953             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5954         port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5955         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5956             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5957 }
5958 
5959 /*
5960  * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5961  * to 1, then set it firstly.
5962  *
5963  * Each port contains two major DMA engines. One DMA engine walks through
5964  * the command list, and is controlled by PxCMD.ST. The second DMA engine
5965  * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5966  *
5967  * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5968  * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5969  * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5970  *
5971  * Software shall not set PxCMD.ST to '1' unless a functional device is
5972  * present on the port(as determined by PxTFD.STS.BSY = '0',
5973  * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5974  */
5975 static int
5976 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5977 {
5978         uint32_t port_cmd_status;
5979 
5980         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5981 
5982         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5983 
5984         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5985                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5986                     "the state for port %d is 0x%x",
5987                     port, ahci_portp->ahciport_port_state);
5988                 return (AHCI_FAILURE);
5989         }
5990 
5991         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5992                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5993                     "no device is attached at port %d", port);
5994                 return (AHCI_FAILURE);
5995         }
5996 
5997         /* First to set PxCMD.FRE before setting PxCMD.ST. */
5998         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5999             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6000 
6001         if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
6002                 port_cmd_status |= AHCI_CMD_STATUS_FRE;
6003                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6004                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6005                     port_cmd_status);
6006         }
6007 
6008         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6009             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6010 
6011         port_cmd_status |= AHCI_CMD_STATUS_ST;
6012 
6013         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6014             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6015             port_cmd_status);
6016 
6017         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
6018 
6019         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
6020             "PxCMD.ST set to '1' at port %d", port);
6021 
6022         return (AHCI_SUCCESS);
6023 }
6024 
6025 /*
6026  * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
6027  * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
6028  * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
6029  */
6030 static int
6031 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6032 {
6033         uint8_t port = ahci_portp->ahciport_port_num;
6034         uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6035             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6036 
6037         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6038 
6039         /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
6040         if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
6041                 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
6042                     port) != AHCI_SUCCESS)
6043                         return (AHCI_FAILURE);
6044 
6045                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6046                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6047         }
6048 
6049         /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
6050         if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
6051                 int loop_count = 0;
6052 
6053                 /* Clear PxCMD.FRE */
6054                 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
6055                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6056                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6057                     port_cmd_status);
6058 
6059                 /* Wait until PxCMD.FR is cleared */
6060                 for (;;) {
6061                         port_cmd_status =
6062                             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6063                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6064 
6065                         if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
6066                                 break;
6067 
6068                         if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
6069                                 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
6070                                     "ahci_setup_port_base_addresses: cannot "
6071                                     "clear PxCMD.FR for port %d.", port);
6072 
6073                                 /*
6074                                  * We are effectively timing out after 0.5 sec.
6075                                  * This value is specified in AHCI spec.
6076                                  */
6077                                 return (AHCI_FAILURE);
6078                         }
6079 
6080                         /* Wait for 1 millisec */
6081                         drv_usecwait(AHCI_1MS_USECS);
6082                 }
6083         }
6084 
6085         /* Step 3: Config Port Command List Base Address */
6086         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6087             (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
6088             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6089 
6090         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6091             (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
6092             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
6093 
6094         /* Step 4: Config Port Received FIS Base Address */
6095         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6096             (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
6097             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6098 
6099         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6100             (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6101             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6102 
6103         return (AHCI_SUCCESS);
6104 }
6105 
6106 /*
6107  * Allocate the ahci_port_t including Received FIS and Command List.
6108  * The argument - port is the physical port number, and not logical
6109  * port number seen by the SATA framework.
6110  */
6111 static int
6112 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6113 {
6114         dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6115         ahci_port_t *ahci_portp;
6116         char taskq_name[64] = "event_handle_taskq";
6117 
6118         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6119 
6120         ahci_portp =
6121             (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6122 
6123         ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6124         ahci_portp->ahciport_port_num = port;
6125 
6126         /* Initialize the port condition variable */
6127         cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6128 
6129         /* Initialize the port mutex */
6130         mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6131             (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6132 
6133         mutex_enter(&ahci_portp->ahciport_mutex);
6134 
6135         /*
6136          * Allocate memory for received FIS structure and
6137          * command list for this port
6138          */
6139         if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6140                 goto err_case1;
6141         }
6142 
6143         if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6144                 goto err_case2;
6145         }
6146 
6147         /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6148         if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6149             AHCI_SUCCESS) {
6150                 goto err_case3;
6151         }
6152 
6153         (void) snprintf(taskq_name + strlen(taskq_name),
6154             sizeof (taskq_name) - strlen(taskq_name),
6155             "_port%d", port);
6156 
6157         /* Create the taskq for the port */
6158         if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6159             taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6160                 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6161                     "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6162                 goto err_case3;
6163         }
6164 
6165         /* Allocate the argument for the taskq */
6166         ahci_portp->ahciport_event_args =
6167             kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6168 
6169         ahci_portp->ahciport_event_args->ahciea_addrp =
6170             kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6171 
6172         /* Initialize the done queue */
6173         ahci_portp->ahciport_doneq = NULL;
6174         ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6175         ahci_portp->ahciport_doneq_len = 0;
6176 
6177         mutex_exit(&ahci_portp->ahciport_mutex);
6178 
6179         return (AHCI_SUCCESS);
6180 
6181 err_case3:
6182         ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6183 
6184 err_case2:
6185         ahci_dealloc_rcvd_fis(ahci_portp);
6186 
6187 err_case1:
6188         mutex_exit(&ahci_portp->ahciport_mutex);
6189         mutex_destroy(&ahci_portp->ahciport_mutex);
6190         cv_destroy(&ahci_portp->ahciport_cv);
6191 
6192         kmem_free(ahci_portp, sizeof (ahci_port_t));
6193 
6194         return (AHCI_FAILURE);
6195 }
6196 
6197 /*
6198  * Reverse of ahci_alloc_port_state().
6199  */
6200 static void
6201 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6202 {
6203         ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6204 
6205         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6206         ASSERT(ahci_portp != NULL);
6207 
6208         mutex_enter(&ahci_portp->ahciport_mutex);
6209         kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6210             sizeof (ahci_addr_t));
6211         ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6212         kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6213         ahci_portp->ahciport_event_args = NULL;
6214         ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6215         ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6216         ahci_dealloc_rcvd_fis(ahci_portp);
6217         ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6218         mutex_exit(&ahci_portp->ahciport_mutex);
6219 
6220         mutex_destroy(&ahci_portp->ahciport_mutex);
6221         cv_destroy(&ahci_portp->ahciport_cv);
6222 
6223         kmem_free(ahci_portp, sizeof (ahci_port_t));
6224 
6225         ahci_ctlp->ahcictl_ports[port] = NULL;
6226 }
6227 
6228 /*
6229  * Allocates memory for the Received FIS Structure
6230  */
6231 static int
6232 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6233 {
6234         size_t rcvd_fis_size;
6235         size_t ret_len;
6236         uint_t cookie_count;
6237 
6238         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6239 
6240         rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6241 
6242         /* allocate rcvd FIS dma handle. */
6243         if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6244             &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6245             DDI_DMA_SLEEP,
6246             NULL,
6247             &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6248             DDI_SUCCESS) {
6249                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6250                     "rcvd FIS dma handle alloc failed", NULL);
6251 
6252                 return (AHCI_FAILURE);
6253         }
6254 
6255         if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6256             rcvd_fis_size,
6257             &accattr,
6258             DDI_DMA_CONSISTENT,
6259             DDI_DMA_SLEEP,
6260             NULL,
6261             (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6262             &ret_len,
6263             &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6264 
6265                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6266                     "rcvd FIS dma mem alloc fail", NULL);
6267                 /* error.. free the dma handle. */
6268                 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6269                 return (AHCI_FAILURE);
6270         }
6271 
6272         if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6273             NULL,
6274             (caddr_t)ahci_portp->ahciport_rcvd_fis,
6275             rcvd_fis_size,
6276             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6277             DDI_DMA_SLEEP,
6278             NULL,
6279             &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6280             &cookie_count) !=  DDI_DMA_MAPPED) {
6281 
6282                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6283                     "rcvd FIS dma handle bind fail", NULL);
6284                 /*  error.. free the dma handle & free the memory. */
6285                 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6286                 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6287                 return (AHCI_FAILURE);
6288         }
6289 
6290         bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6291 
6292         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6293             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6294         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6295             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6296 
6297         return (AHCI_SUCCESS);
6298 }
6299 
6300 /*
6301  * Deallocates the Received FIS Structure
6302  */
6303 static void
6304 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6305 {
6306         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6307 
6308         /* Unbind the cmd list dma handle first. */
6309         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6310 
6311         /* Then free the underlying memory. */
6312         ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6313 
6314         /* Now free the handle itself. */
6315         ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6316 }
6317 
6318 /*
6319  * Allocates memory for the Command List, which contains up to 32 entries.
6320  * Each entry contains a command header, which is a 32-byte structure that
6321  * includes the pointer to the command table.
6322  */
6323 static int
6324 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6325 {
6326         size_t cmd_list_size;
6327         size_t ret_len;
6328         uint_t cookie_count;
6329 
6330         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6331 
6332         cmd_list_size =
6333             ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6334 
6335         /* allocate cmd list dma handle. */
6336         if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6337             &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6338             DDI_DMA_SLEEP,
6339             NULL,
6340             &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6341 
6342                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6343                     "cmd list dma handle alloc failed", NULL);
6344                 return (AHCI_FAILURE);
6345         }
6346 
6347         if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6348             cmd_list_size,
6349             &accattr,
6350             DDI_DMA_CONSISTENT,
6351             DDI_DMA_SLEEP,
6352             NULL,
6353             (caddr_t *)&ahci_portp->ahciport_cmd_list,
6354             &ret_len,
6355             &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6356 
6357                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6358                     "cmd list dma mem alloc fail", NULL);
6359                 /* error.. free the dma handle. */
6360                 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6361                 return (AHCI_FAILURE);
6362         }
6363 
6364         if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6365             NULL,
6366             (caddr_t)ahci_portp->ahciport_cmd_list,
6367             cmd_list_size,
6368             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6369             DDI_DMA_SLEEP,
6370             NULL,
6371             &ahci_portp->ahciport_cmd_list_dma_cookie,
6372             &cookie_count) !=  DDI_DMA_MAPPED) {
6373 
6374                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6375                     "cmd list dma handle bind fail", NULL);
6376                 /*  error.. free the dma handle & free the memory. */
6377                 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6378                 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6379                 return (AHCI_FAILURE);
6380         }
6381 
6382         bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6383 
6384         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6385             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6386 
6387         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6388             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6389 
6390         if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6391                 goto err_out;
6392         }
6393 
6394         return (AHCI_SUCCESS);
6395 
6396 err_out:
6397         /* Unbind the cmd list dma handle first. */
6398         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6399 
6400         /* Then free the underlying memory. */
6401         ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6402 
6403         /* Now free the handle itself. */
6404         ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6405 
6406         return (AHCI_FAILURE);
6407 }
6408 
6409 /*
6410  * Deallocates the Command List
6411  */
6412 static void
6413 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6414 {
6415         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6416 
6417         /* First dealloc command table */
6418         ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6419 
6420         /* Unbind the cmd list dma handle first. */
6421         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6422 
6423         /* Then free the underlying memory. */
6424         ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6425 
6426         /* Now free the handle itself. */
6427         ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6428 }
6429 
6430 /*
6431  * Allocates memory for all Command Tables, which contains Command FIS,
6432  * ATAPI Command and Physical Region Descriptor Table.
6433  */
6434 static int
6435 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6436 {
6437         size_t ret_len;
6438         ddi_dma_cookie_t cmd_table_dma_cookie;
6439         uint_t cookie_count;
6440         int slot;
6441 
6442         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6443 
6444         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6445             "ahci_alloc_cmd_tables: port %d enter",
6446             ahci_portp->ahciport_port_num);
6447 
6448         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6449                 /* Allocate cmd table dma handle. */
6450                 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6451                     &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6452                     DDI_DMA_SLEEP,
6453                     NULL,
6454                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6455                     DDI_SUCCESS) {
6456 
6457                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6458                             "cmd table dma handle alloc failed", NULL);
6459 
6460                         goto err_out;
6461                 }
6462 
6463                 if (ddi_dma_mem_alloc(
6464                     ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6465                     ahci_cmd_table_size,
6466                     &accattr,
6467                     DDI_DMA_CONSISTENT,
6468                     DDI_DMA_SLEEP,
6469                     NULL,
6470                     (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6471                     &ret_len,
6472                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6473                     NULL) {
6474 
6475                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6476                             "cmd table dma mem alloc fail", NULL);
6477 
6478                         /* error.. free the dma handle. */
6479                         ddi_dma_free_handle(
6480                             &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6481                         goto err_out;
6482                 }
6483 
6484                 if (ddi_dma_addr_bind_handle(
6485                     ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6486                     NULL,
6487                     (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6488                     ahci_cmd_table_size,
6489                     DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6490                     DDI_DMA_SLEEP,
6491                     NULL,
6492                     &cmd_table_dma_cookie,
6493                     &cookie_count) !=  DDI_DMA_MAPPED) {
6494 
6495                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6496                             "cmd table dma handle bind fail", NULL);
6497                         /*  error.. free the dma handle & free the memory. */
6498                         ddi_dma_mem_free(
6499                             &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6500                         ddi_dma_free_handle(
6501                             &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6502                         goto err_out;
6503                 }
6504 
6505                 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6506                     ahci_cmd_table_size);
6507 
6508                 /* Config Port Command Table Base Address */
6509                 SET_COMMAND_TABLE_BASE_ADDR(
6510                     (&ahci_portp->ahciport_cmd_list[slot]),
6511                     cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6512 
6513 #ifndef __lock_lint
6514                 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6515                     (&ahci_portp->ahciport_cmd_list[slot]),
6516                     cmd_table_dma_cookie.dmac_laddress >> 32);
6517 #endif
6518         }
6519 
6520         return (AHCI_SUCCESS);
6521 err_out:
6522 
6523         for (slot--; slot >= 0; slot--) {
6524                 /* Unbind the cmd table dma handle first */
6525                 (void) ddi_dma_unbind_handle(
6526                     ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6527 
6528                 /* Then free the underlying memory */
6529                 ddi_dma_mem_free(
6530                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6531 
6532                 /* Now free the handle itself */
6533                 ddi_dma_free_handle(
6534                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6535         }
6536 
6537         return (AHCI_FAILURE);
6538 }
6539 
6540 /*
6541  * Deallocates memory for all Command Tables.
6542  */
6543 static void
6544 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6545 {
6546         int slot;
6547 
6548         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6549 
6550         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6551             "ahci_dealloc_cmd_tables: %d enter",
6552             ahci_portp->ahciport_port_num);
6553 
6554         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6555                 /* Unbind the cmd table dma handle first. */
6556                 (void) ddi_dma_unbind_handle(
6557                     ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6558 
6559                 /* Then free the underlying memory. */
6560                 ddi_dma_mem_free(
6561                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6562 
6563                 /* Now free the handle itself. */
6564                 ddi_dma_free_handle(
6565                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6566         }
6567 }
6568 
6569 /*
6570  * Update SATA registers at controller ports
6571  */
6572 static void
6573 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6574     sata_device_t *sd)
6575 {
6576         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6577 
6578         sd->satadev_scr.sstatus =
6579             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6580             (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6581         sd->satadev_scr.serror =
6582             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6583             (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6584         sd->satadev_scr.scontrol =
6585             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6586             (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6587         sd->satadev_scr.sactive =
6588             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6589             (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6590 }
6591 
6592 /*
6593  * For poll mode, ahci_port_intr will be called to emulate the interrupt
6594  */
6595 static void
6596 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6597 {
6598         uint32_t port_intr_status;
6599         uint32_t port_intr_enable;
6600 
6601         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6602             "ahci_port_intr enter: port %d", port);
6603 
6604         mutex_enter(&ahci_portp->ahciport_mutex);
6605         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6606                 /* For SATA_OPMODE_POLLING commands */
6607                 port_intr_enable =
6608                     (AHCI_INTR_STATUS_DHRS |
6609                     AHCI_INTR_STATUS_PSS |
6610                     AHCI_INTR_STATUS_SDBS |
6611                     AHCI_INTR_STATUS_UFS |
6612                     AHCI_INTR_STATUS_PCS |
6613                     AHCI_INTR_STATUS_PRCS |
6614                     AHCI_INTR_STATUS_OFS |
6615                     AHCI_INTR_STATUS_INFS |
6616                     AHCI_INTR_STATUS_IFS |
6617                     AHCI_INTR_STATUS_HBDS |
6618                     AHCI_INTR_STATUS_HBFS |
6619                     AHCI_INTR_STATUS_TFES);
6620         } else {
6621                 /*
6622                  * port_intr_enable indicates that the corresponding interrrupt
6623                  * reporting is enabled.
6624                  */
6625                 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6626                     (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6627         }
6628 
6629         /* IPMS error in port reset should be ignored according AHCI spec. */
6630         if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6631                 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6632         mutex_exit(&ahci_portp->ahciport_mutex);
6633 
6634         /*
6635          * port_intr_stats indicates that the corresponding interrupt
6636          * condition is active.
6637          */
6638         port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6639             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6640 
6641         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6642             "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6643             "port_intr_enable = 0x%x",
6644             port, port_intr_status, port_intr_enable);
6645 
6646         port_intr_status &= port_intr_enable;
6647 
6648         /*
6649          * Pending interrupt events are indicated by the PxIS register.
6650          * Make sure we don't miss any event.
6651          */
6652         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6653                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6654                     DDI_SERVICE_UNAFFECTED);
6655                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6656                     DDI_FME_VERSION);
6657                 return;
6658         }
6659 
6660         /* First clear the port interrupts status */
6661         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6662             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6663             port_intr_status);
6664 
6665         /* Check the completed non-queued commands */
6666         if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6667             AHCI_INTR_STATUS_PSS)) {
6668                 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6669                     ahci_portp, port);
6670         }
6671 
6672         /* Check the completed queued commands */
6673         if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6674                 (void) ahci_intr_set_device_bits(ahci_ctlp,
6675                     ahci_portp, port);
6676         }
6677 
6678         /* Check the port connect change status interrupt bit */
6679         if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6680                 (void) ahci_intr_port_connect_change(ahci_ctlp,
6681                     ahci_portp, port);
6682         }
6683 
6684         /* Check the device mechanical presence status interrupt bit */
6685         if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6686                 (void) ahci_intr_device_mechanical_presence_status(
6687                     ahci_ctlp, ahci_portp, port);
6688         }
6689 
6690         /* Check the PhyRdy change status interrupt bit */
6691         if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6692                 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6693                     port);
6694         }
6695 
6696         /*
6697          * Check the non-fatal error interrupt bits, there are four
6698          * kinds of non-fatal errors at the time being:
6699          *
6700          *    PxIS.UFS - Unknown FIS Error
6701          *    PxIS.OFS - Overflow Error
6702          *    PxIS.INFS - Interface Non-Fatal Error
6703          *    PxIS.IPMS - Incorrect Port Multiplier Status Error
6704          *
6705          * For these non-fatal errors, the HBA can continue to operate,
6706          * so the driver just log the error messages.
6707          */
6708         if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6709             AHCI_INTR_STATUS_OFS |
6710             AHCI_INTR_STATUS_IPMS |
6711             AHCI_INTR_STATUS_INFS)) {
6712                 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6713                     port, port_intr_status);
6714         }
6715 
6716         /*
6717          * Check the fatal error interrupt bits, there are four kinds
6718          * of fatal errors for AHCI controllers:
6719          *
6720          *    PxIS.HBFS - Host Bus Fatal Error
6721          *    PxIS.HBDS - Host Bus Data Error
6722          *    PxIS.IFS - Interface Fatal Error
6723          *    PxIS.TFES - Task File Error
6724          *
6725          * The fatal error means the HBA can not recover from it by
6726          * itself, and it will try to abort the transfer, and the software
6727          * must intervene to restart the port.
6728          */
6729         if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6730             AHCI_INTR_STATUS_HBDS |
6731             AHCI_INTR_STATUS_HBFS |
6732             AHCI_INTR_STATUS_TFES))
6733                 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6734                     port, port_intr_status);
6735 
6736         /* Check the cold port detect interrupt bit */
6737         if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6738                 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6739         }
6740 
6741         /* Second clear the corresponding bit in IS.IPS */
6742         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6743             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6744 
6745         /* Try to recover at the end of the interrupt handler. */
6746         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6747             DDI_FM_OK) {
6748                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6749                     DDI_SERVICE_UNAFFECTED);
6750                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6751                     DDI_FME_VERSION);
6752         }
6753 }
6754 
6755 /*
6756  * Interrupt service handler
6757  */
6758 static uint_t
6759 ahci_intr(caddr_t arg1, caddr_t arg2)
6760 {
6761 #ifndef __lock_lint
6762         _NOTE(ARGUNUSED(arg2))
6763 #endif
6764         /* LINTED */
6765         ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6766         ahci_port_t *ahci_portp;
6767         int32_t global_intr_status;
6768         uint8_t port;
6769 
6770         /*
6771          * global_intr_status indicates that the corresponding port has
6772          * an interrupt pending.
6773          */
6774         global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6775             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6776 
6777         if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6778                 /* The interrupt is not ours */
6779                 return (DDI_INTR_UNCLAIMED);
6780         }
6781 
6782         /*
6783          * Check the handle after reading global_intr_status - we don't want
6784          * to miss any port with pending interrupts.
6785          */
6786         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6787             DDI_FM_OK) {
6788                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6789                     DDI_SERVICE_UNAFFECTED);
6790                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6791                     DDI_FME_VERSION);
6792                 return (DDI_INTR_UNCLAIMED);
6793         }
6794 
6795         /* Loop for all the ports */
6796         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6797                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6798                         continue;
6799                 }
6800                 if (!((0x1 << port) & global_intr_status)) {
6801                         continue;
6802                 }
6803 
6804                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6805 
6806                 /* Call ahci_port_intr */
6807                 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6808         }
6809 
6810         return (DDI_INTR_CLAIMED);
6811 }
6812 
6813 /*
6814  * For non-queued commands, when the corresponding bit in the PxCI register
6815  * is cleared, it means the command is completed successfully. And according
6816  * to the HBA state machine, there are three conditions which possibly will
6817  * try to clear the PxCI register bit.
6818  *      1. Receive one D2H Register FIS which is with 'I' bit set
6819  *      2. Update PIO Setup FIS
6820  *      3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6821  *
6822  * Process completed non-queued commands when the interrupt status bit -
6823  * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6824  *
6825  * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6826  * with the 'I' bit set. And the following commands will send thus
6827  * FIS with 'I' bit set upon the successful completion:
6828  *      1. Non-data commands
6829  *      2. DMA data-in command
6830  *      3. DMA data-out command
6831  *      4. PIO data-out command
6832  *      5. PACKET non-data commands
6833  *      6. PACKET PIO data-in command
6834  *      7. PACKET PIO data-out command
6835  *      8. PACKET DMA data-in command
6836  *      9. PACKET DMA data-out command
6837  *
6838  * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6839  * with the 'I' bit set. And the following commands will send this
6840  * FIS upon the successful completion:
6841  *      1. PIO data-in command
6842  */
6843 static int
6844 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6845     ahci_port_t *ahci_portp, uint8_t port)
6846 {
6847         uint32_t port_cmd_issue = 0;
6848         uint32_t finished_tags;
6849         int finished_slot;
6850         sata_pkt_t *satapkt;
6851         ahci_fis_d2h_register_t *rcvd_fisp;
6852 #if AHCI_DEBUG
6853         ahci_cmd_header_t *cmd_header;
6854         uint32_t cmd_dmacount;
6855 #endif
6856 
6857         mutex_enter(&ahci_portp->ahciport_mutex);
6858 
6859         if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6860             !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6861             !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6862                 /*
6863                  * Spurious interrupt. Nothing to be done.
6864                  */
6865                 mutex_exit(&ahci_portp->ahciport_mutex);
6866                 return (AHCI_SUCCESS);
6867         }
6868 
6869         port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6870             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6871 
6872         /* If the PxCI corrupts, don't complete the commmands. */
6873         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6874             != DDI_FM_OK) {
6875                 mutex_exit(&ahci_portp->ahciport_mutex);
6876                 return (AHCI_FAILURE);
6877         }
6878 
6879         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6880                 /* Slot 0 is always used during error recovery */
6881                 finished_tags = 0x1 & ~port_cmd_issue;
6882                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6883                     "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6884                     "retrieval is finished, and finished_tags = 0x%x",
6885                     port, finished_tags);
6886         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6887                 finished_tags = 0x1 & ~port_cmd_issue;
6888                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6889                     "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6890                     "port multiplier is finished, and finished_tags = 0x%x",
6891                     port, finished_tags);
6892 
6893         } else {
6894 
6895                 finished_tags = ahci_portp->ahciport_pending_tags &
6896                     ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6897         }
6898 
6899         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6900             "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6901             "port_cmd_issue = 0x%x finished_tags = 0x%x",
6902             ahci_portp->ahciport_pending_tags, port_cmd_issue,
6903             finished_tags);
6904 
6905         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6906             (finished_tags == 0x1)) {
6907                 satapkt = ahci_portp->ahciport_err_retri_pkt;
6908                 ASSERT(satapkt != NULL);
6909 
6910                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6911                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6912                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6913 
6914                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6915                 goto out;
6916         }
6917 
6918         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6919             (finished_tags == 0x1)) {
6920                 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6921                 ASSERT(satapkt != NULL);
6922 
6923                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6924                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6925                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6926 
6927                 /* READ PORTMULT need copy out FIS content. */
6928                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6929                         rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6930                             ahcirf_d2h_register_fis);
6931                         satapkt->satapkt_cmd.satacmd_status_reg =
6932                             GET_RFIS_STATUS(rcvd_fisp);
6933                         ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6934                 }
6935 
6936                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6937                 goto out;
6938         }
6939 
6940         while (finished_tags) {
6941                 finished_slot = ddi_ffs(finished_tags) - 1;
6942                 if (finished_slot == -1) {
6943                         goto out;
6944                 }
6945 
6946                 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6947                 ASSERT(satapkt != NULL);
6948 #if AHCI_DEBUG
6949                 /*
6950                  * For non-native queued commands, the PRD byte count field
6951                  * shall contain an accurate count of the number of bytes
6952                  * transferred for the command before the PxCI bit is cleared
6953                  * to '0' for the command.
6954                  *
6955                  * The purpose of this field is to let software know how many
6956                  * bytes transferred for a given operation in order to
6957                  * determine if underflow occurred. When issuing native command
6958                  * queuing commands, this field should not be used and is not
6959                  * required to be valid since in this case underflow is always
6960                  * illegal.
6961                  *
6962                  * For data reads, the HBA will update its PRD byte count with
6963                  * the total number of bytes received from the last FIS, and
6964                  * may be able to continue normally. For data writes, the
6965                  * device will detect an error, and HBA most likely will get
6966                  * a fatal error.
6967                  *
6968                  * Therefore, here just put code to debug part. And please
6969                  * refer to the comment above ahci_intr_fatal_error for the
6970                  * definition of underflow error.
6971                  */
6972                 cmd_dmacount =
6973                     ahci_portp->ahciport_prd_bytecounts[finished_slot];
6974                 if (cmd_dmacount) {
6975                         cmd_header =
6976                             &ahci_portp->ahciport_cmd_list[finished_slot];
6977                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6978                             "ahci_intr_cmd_cmplt: port %d, "
6979                             "PRD Byte Count = 0x%x, "
6980                             "ahciport_prd_bytecounts = 0x%x", port,
6981                             cmd_header->ahcich_prd_byte_count,
6982                             cmd_dmacount);
6983 
6984                         if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6985                                 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6986                                     "ahci_intr_cmd_cmplt: port %d, "
6987                                     "an underflow occurred", port);
6988                         }
6989                 }
6990 #endif
6991 
6992                 /*
6993                  * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6994                  * feature, sata_special_regs flag will be set, and the
6995                  * driver should copy the status and the other corresponding
6996                  * register values in the D2H Register FIS received (It's
6997                  * working on Non-data protocol) from the device back to
6998                  * the sata_cmd.
6999                  *
7000                  * For every AHCI port, there is only one Received FIS
7001                  * structure, which contains the FISes received from the
7002                  * device, So we're trying to copy the content of D2H
7003                  * Register FIS in the Received FIS structure back to
7004                  * the sata_cmd.
7005                  */
7006                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
7007                         rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
7008                             ahcirf_d2h_register_fis);
7009                         satapkt->satapkt_cmd.satacmd_status_reg =
7010                             GET_RFIS_STATUS(rcvd_fisp);
7011                         ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
7012                 }
7013 
7014                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7015                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
7016                     "with SATA_PKT_COMPLETED", (void *)satapkt);
7017 
7018                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
7019                 CLEAR_BIT(finished_tags, finished_slot);
7020                 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7021 
7022                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7023         }
7024 out:
7025         AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
7026             "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
7027             ahci_portp->ahciport_pending_tags);
7028 
7029         ahci_flush_doneq(ahci_portp);
7030 
7031         mutex_exit(&ahci_portp->ahciport_mutex);
7032 
7033         return (AHCI_SUCCESS);
7034 }
7035 
7036 /*
7037  * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
7038  * with the 'I' bit set and has been copied into system memory. It will
7039  * be sent under the following situations:
7040  *
7041  * 1. NCQ command is completed
7042  *
7043  * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
7044  * via the Set Device Bits FIS. When such event is generated, the software
7045  * needs to read PxSACT register and compares the current value to the
7046  * list of commands previously issue by software. ahciport_pending_ncq_tags
7047  * keeps the tags of previously issued commands.
7048  *
7049  * 2. Asynchronous Notification
7050  *
7051  * Asynchronous Notification is a feature in SATA spec 2.6.
7052  *
7053  * 1) ATAPI device will send a signal to the host when media is inserted or
7054  * removed and avoids polling the device for media changes. The signal
7055  * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
7056  * set to '1'. At the moment, it's not supported yet.
7057  *
7058  * 2) Port multiplier will send a signal to the host when a hot plug event
7059  * has occured on a port multiplier port. It is used when command based
7060  * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
7061  */
7062 static int
7063 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
7064     ahci_port_t *ahci_portp, uint8_t port)
7065 {
7066         ahci_addr_t addr;
7067 
7068         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7069             "ahci_intr_set_device_bits enter: port %d", port);
7070 
7071         /* Initialize HBA port address */
7072         AHCI_ADDR_SET_PORT(&addr, port);
7073 
7074         /* NCQ plug handler */
7075         (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
7076 
7077         /* Check port multiplier's asynchronous notification events */
7078         if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
7079                 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
7080                     ahci_portp, port);
7081         }
7082 
7083         /* ATAPI events is not supported yet */
7084 
7085         return (AHCI_SUCCESS);
7086 }
7087 /*
7088  * NCQ interrupt handler. Called upon a NCQ command is completed.
7089  * Only be called from ahci_intr_set_device_bits().
7090  */
7091 static int
7092 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7093     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7094 {
7095         uint32_t port_sactive;
7096         uint32_t port_cmd_issue;
7097         uint32_t issued_tags;
7098         int issued_slot;
7099         uint32_t finished_tags;
7100         int finished_slot;
7101         uint8_t port = addrp->aa_port;
7102         sata_pkt_t *satapkt;
7103 
7104         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7105             "ahci_intr_set_device_bits enter: port %d", port);
7106 
7107         mutex_enter(&ahci_portp->ahciport_mutex);
7108         if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7109                 mutex_exit(&ahci_portp->ahciport_mutex);
7110                 return (AHCI_SUCCESS);
7111         }
7112 
7113         /*
7114          * First the handler got which commands are finished by checking
7115          * PxSACT register
7116          */
7117         port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7118             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7119 
7120         finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7121             ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7122 
7123         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7124             "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7125             "port_sactive = 0x%x", port,
7126             ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7127 
7128         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7129             "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7130 
7131         /*
7132          * For NCQ commands, the software can determine which command has
7133          * already been transmitted to the device by checking PxCI register.
7134          */
7135         port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7136             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7137 
7138         issued_tags = ahci_portp->ahciport_pending_tags &
7139             ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7140 
7141         /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7142         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7143             != DDI_FM_OK) {
7144                 mutex_exit(&ahci_portp->ahciport_mutex);
7145                 return (AHCI_FAILURE);
7146         }
7147 
7148         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7149             "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7150             "port_cmd_issue = 0x%x", port,
7151             ahci_portp->ahciport_pending_tags, port_cmd_issue);
7152 
7153         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7154             "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7155 
7156         /*
7157          * Clear ahciport_pending_tags bit when the corresponding command
7158          * is already sent down to the device.
7159          */
7160         while (issued_tags) {
7161                 issued_slot = ddi_ffs(issued_tags) - 1;
7162                 if (issued_slot == -1) {
7163                         goto next;
7164                 }
7165                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7166                 CLEAR_BIT(issued_tags, issued_slot);
7167         }
7168 
7169 next:
7170         while (finished_tags) {
7171                 finished_slot = ddi_ffs(finished_tags) - 1;
7172                 if (finished_slot == -1) {
7173                         goto out;
7174                 }
7175 
7176                 /* The command is certainly transmitted to the device */
7177                 ASSERT(!(ahci_portp->ahciport_pending_tags &
7178                     (0x1 << finished_slot)));
7179 
7180                 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7181                 ASSERT(satapkt != NULL);
7182 
7183                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7184                     "ahci_intr_set_device_bits: sending up pkt 0x%p "
7185                     "with SATA_PKT_COMPLETED", (void *)satapkt);
7186 
7187                 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7188                 CLEAR_BIT(finished_tags, finished_slot);
7189                 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7190 
7191                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7192         }
7193 out:
7194         AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7195             "ahci_intr_set_device_bits: port %d "
7196             "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7197             port, ahci_portp->ahciport_pending_ncq_tags,
7198             ahci_portp->ahciport_pending_tags);
7199 
7200         ahci_flush_doneq(ahci_portp);
7201 
7202         mutex_exit(&ahci_portp->ahciport_mutex);
7203 
7204         return (AHCI_SUCCESS);
7205 }
7206 
7207 /*
7208  * Port multiplier asynchronous notification event handler. Called upon a
7209  * device is hot plugged/pulled.
7210  *
7211  * The async-notification event will only be recorded by ahcipmi_snotif_tags
7212  * here and will be handled by ahci_probe_pmult().
7213  *
7214  * NOTE: called only from ahci_port_intr().
7215  */
7216 static int
7217 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7218     uint8_t port)
7219 {
7220         sata_device_t sdevice;
7221 
7222         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7223             "ahci_intr_pmult_sntf_events enter: port %d ", port);
7224 
7225         /* no hot-plug while attaching process */
7226         mutex_enter(&ahci_ctlp->ahcictl_mutex);
7227         if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7228                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7229                 return (AHCI_SUCCESS);
7230         }
7231         mutex_exit(&ahci_ctlp->ahcictl_mutex);
7232 
7233         mutex_enter(&ahci_portp->ahciport_mutex);
7234         if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7235                 mutex_exit(&ahci_portp->ahciport_mutex);
7236                 return (AHCI_SUCCESS);
7237         }
7238 
7239         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7240 
7241         ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7242             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7243             (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7244         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7245             (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7246             AHCI_SNOTIF_CLEAR_ALL);
7247 
7248         if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7249                 mutex_exit(&ahci_portp->ahciport_mutex);
7250                 return (AHCI_SUCCESS);
7251         }
7252 
7253         /* Port Multiplier sub-device hot-plug handler */
7254         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7255                 mutex_exit(&ahci_portp->ahciport_mutex);
7256                 return (AHCI_SUCCESS);
7257         }
7258 
7259         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7260                 /* Not allowed to re-enter. */
7261                 mutex_exit(&ahci_portp->ahciport_mutex);
7262                 return (AHCI_SUCCESS);
7263         }
7264 
7265         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7266 
7267         /*
7268          * NOTE:
7269          * Even if Asynchronous Notification is supported (and enabled) by
7270          * both controller and the port multiplier, the content of PxSNTF
7271          * register is always set to 0x8000 by async notification event. We
7272          * need to check GSCR[32] on the port multiplier to find out the
7273          * owner of this event.
7274          * This is not accord with SATA spec 2.6 and needs further
7275          * clarification.
7276          */
7277         /* hot-plug will not reported while reseting. */
7278         if (ahci_portp->ahciport_reset_in_progress == 1) {
7279                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7280                     "port %d snotif event ignored", port);
7281                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7282                 mutex_exit(&ahci_portp->ahciport_mutex);
7283                 return (AHCI_SUCCESS);
7284         }
7285 
7286         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7287             "PxSNTF is set to 0x%x by port multiplier",
7288             ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7289 
7290         /*
7291          * Now we need do some necessary operation and inform SATA framework
7292          * that link/device events has happened.
7293          */
7294         bzero((void *)&sdevice, sizeof (sata_device_t));
7295         sdevice.satadev_addr.cport = ahci_ctlp->
7296             ahcictl_port_to_cport[port];
7297         sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7298         sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7299         sdevice.satadev_state = SATA_PSTATE_PWRON;
7300 
7301         /* Just reject packets, do not stop that port. */
7302         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7303 
7304         mutex_exit(&ahci_portp->ahciport_mutex);
7305         sata_hba_event_notify(
7306             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7307             &sdevice,
7308             SATA_EVNT_PMULT_LINK_CHANGED);
7309         mutex_enter(&ahci_portp->ahciport_mutex);
7310 
7311         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7312         mutex_exit(&ahci_portp->ahciport_mutex);
7313 
7314         return (AHCI_SUCCESS);
7315 }
7316 
7317 /*
7318  * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7319  * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7320  * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7321  * indicates a COMINIT signal was received.
7322  *
7323  * Hot plug insertion is detected by reception of a COMINIT signal from the
7324  * device. On reception of unsolicited COMINIT, the HBA shall generate a
7325  * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7326  * begin the normal communication negotiation sequence as outlined in the
7327  * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7328  * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7329  * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7330  * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7331  * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7332  * to handle hot plug insertion. In this interrupt handler, just do nothing
7333  * but print some log message and clear the bit.
7334  */
7335 static int
7336 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7337     ahci_port_t *ahci_portp, uint8_t port)
7338 {
7339 #if AHCI_DEBUG
7340         uint32_t port_serror;
7341 #endif
7342 
7343         mutex_enter(&ahci_portp->ahciport_mutex);
7344 
7345 #if AHCI_DEBUG
7346         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7347             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7348 
7349         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7350             "ahci_intr_port_connect_change: port %d, "
7351             "port_serror = 0x%x", port, port_serror);
7352 #endif
7353 
7354         /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7355         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7356             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7357             SERROR_EXCHANGED_ERR);
7358 
7359         mutex_exit(&ahci_portp->ahciport_mutex);
7360 
7361         return (AHCI_SUCCESS);
7362 }
7363 
7364 /*
7365  * Hot Plug Operation for platforms that support Mechanical Presence
7366  * Switches.
7367  *
7368  * When set, it indicates that a mechanical presence switch attached to this
7369  * port has been opened or closed, which may lead to a change in the connection
7370  * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7371  * are set to '1'.
7372  *
7373  * At the moment, this interrupt is not needed and disabled and we just log
7374  * the debug message.
7375  */
7376 static int
7377 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7378     ahci_port_t *ahci_portp, uint8_t port)
7379 {
7380         uint32_t cap_status, port_cmd_status;
7381 
7382         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7383             "ahci_intr_device_mechanical_presence_status enter, "
7384             "port %d", port);
7385 
7386         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7387             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7388 
7389         mutex_enter(&ahci_portp->ahciport_mutex);
7390         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7391             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7392 
7393         if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7394             !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7395                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7396                     "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7397                     "the interrupt: cap_status = 0x%x, "
7398                     "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7399                 mutex_exit(&ahci_portp->ahciport_mutex);
7400 
7401                 return (AHCI_SUCCESS);
7402         }
7403 
7404 #if AHCI_DEBUG
7405         if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7406                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7407                     "The mechanical presence switch is open: "
7408                     "port %d, port_cmd_status = 0x%x",
7409                     port, port_cmd_status);
7410         } else {
7411                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7412                     "The mechanical presence switch is close: "
7413                     "port %d, port_cmd_status = 0x%x",
7414                     port, port_cmd_status);
7415         }
7416 #endif
7417 
7418         mutex_exit(&ahci_portp->ahciport_mutex);
7419 
7420         return (AHCI_SUCCESS);
7421 }
7422 
7423 /*
7424  * Native Hot Plug Support.
7425  *
7426  * When set, it indicates that the internal PHYRDY signal changed state.
7427  * This bit reflects the state of PxSERR.DIAG.N.
7428  *
7429  * There are three kinds of conditions to generate this interrupt event:
7430  * 1. a device is inserted
7431  * 2. a device is disconnected
7432  * 3. when the link enters/exits a Partial or Slumber interface power
7433  *    management state
7434  *
7435  * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7436  * bit may be set due to the link entering the Partial or Slumber power
7437  * management state, rather than due to a hot plug insertion or removal
7438  * event. So far, the interface power management is disabled, so the
7439  * driver can reliably get removal detection notification via the
7440  * PxSERR.DIAG.N bit.
7441  */
7442 static int
7443 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7444     ahci_port_t *ahci_portp, uint8_t port)
7445 {
7446         uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7447         sata_device_t sdevice;
7448         int dev_exists_now = 0;
7449         int dev_existed_previously = 0;
7450         ahci_addr_t port_addr;
7451 
7452         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7453             "ahci_intr_phyrdy_change enter, port %d", port);
7454 
7455         /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7456         mutex_enter(&ahci_portp->ahciport_mutex);
7457         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7458             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7459             SERROR_PHY_RDY_CHG);
7460         mutex_exit(&ahci_portp->ahciport_mutex);
7461 
7462         mutex_enter(&ahci_ctlp->ahcictl_mutex);
7463         if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7464             (ahci_portp == NULL)) {
7465                 /* The whole controller setup is not yet done. */
7466                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7467                 return (AHCI_SUCCESS);
7468         }
7469         mutex_exit(&ahci_ctlp->ahcictl_mutex);
7470 
7471         mutex_enter(&ahci_portp->ahciport_mutex);
7472 
7473         /* SStatus tells the presence of device. */
7474         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7475             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7476 
7477         if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7478                 dev_exists_now = 1;
7479         }
7480 
7481         if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7482                 dev_existed_previously = 1;
7483         }
7484 
7485         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7486                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7487                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7488                     "ahci_intr_phyrdy_change: port %d "
7489                     "AHCI_PORT_FLAG_NODEV is cleared", port);
7490                 if (dev_exists_now == 0)
7491                         dev_existed_previously = 1;
7492         }
7493 
7494         bzero((void *)&sdevice, sizeof (sata_device_t));
7495         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7496         sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7497         sdevice.satadev_addr.pmport = 0;
7498         sdevice.satadev_state = SATA_PSTATE_PWRON;
7499         ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7500 
7501         AHCI_ADDR_SET_PORT(&port_addr, port);
7502 
7503         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7504         if (dev_exists_now) {
7505                 if (dev_existed_previously) { /* 1 -> 1 */
7506                         /* Things are fine now. The loss was temporary. */
7507                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7508                             "ahci_intr_phyrdy_change  port %d "
7509                             "device link lost/established", port);
7510 
7511                         mutex_exit(&ahci_portp->ahciport_mutex);
7512                         sata_hba_event_notify(
7513                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7514                             &sdevice,
7515                             SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7516                         mutex_enter(&ahci_portp->ahciport_mutex);
7517 
7518                 } else { /* 0 -> 1 */
7519                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7520                             "ahci_intr_phyrdy_change: port %d "
7521                             "device link established", port);
7522 
7523                         /*
7524                          * A new device has been detected. The new device
7525                          * might be a port multiplier instead of a drive, so
7526                          * we cannot update the signature directly.
7527                          */
7528                         (void) ahci_initialize_port(ahci_ctlp,
7529                             ahci_portp, &port_addr);
7530 
7531                         /* Try to start the port */
7532                         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7533                             != AHCI_SUCCESS) {
7534                                 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7535                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7536                                     "ahci_intr_phyrdy_change: port %d failed "
7537                                     "at start port", port);
7538                         }
7539 
7540                         /* Clear the max queue depth for inserted device */
7541                         ahci_portp->ahciport_max_ncq_tags = 0;
7542 
7543                         mutex_exit(&ahci_portp->ahciport_mutex);
7544                         sata_hba_event_notify(
7545                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7546                             &sdevice,
7547                             SATA_EVNT_LINK_ESTABLISHED);
7548                         mutex_enter(&ahci_portp->ahciport_mutex);
7549 
7550                 }
7551         } else { /* No device exists now */
7552 
7553                 if (dev_existed_previously) { /* 1 -> 0 */
7554                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7555                             "ahci_intr_phyrdy_change: port %d "
7556                             "device link lost", port);
7557 
7558                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7559                         (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7560                             ahci_portp, port);
7561 
7562                         if (ahci_portp->ahciport_device_type ==
7563                             SATA_DTYPE_PMULT) {
7564                                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7565                         }
7566 
7567                         /* An existing device is lost. */
7568                         ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7569                         ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7570 
7571                         mutex_exit(&ahci_portp->ahciport_mutex);
7572                         sata_hba_event_notify(
7573                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7574                             &sdevice,
7575                             SATA_EVNT_LINK_LOST);
7576                         mutex_enter(&ahci_portp->ahciport_mutex);
7577                 }
7578         }
7579         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7580 
7581         mutex_exit(&ahci_portp->ahciport_mutex);
7582 
7583         return (AHCI_SUCCESS);
7584 }
7585 
7586 /*
7587  * PxIS.UFS - Unknown FIS Error
7588  *
7589  * This interrupt event means an unknown FIS was received and has been
7590  * copied into system memory. An unknown FIS is not considered an illegal
7591  * FIS, unless the length received is more than 64 bytes. If an unknown
7592  * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7593  * normal operation. If the unknown FIS is more than 64 bytes, then it
7594  * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7595  * a fatal error.
7596  *
7597  * PxIS.IPMS - Incorrect Port Multiplier Status
7598  *
7599  * IPMS Indicates that the HBA received a FIS from a device that did not
7600  * have a command outstanding. The IPMS bit may be set during enumeration
7601  * of devices on a Port Multiplier due to the normal Port Multiplier
7602  * enumeration process. It is recommended that IPMS only be used after
7603  * enumeration is complete on the Port Multiplier (copied from spec).
7604  *
7605  * PxIS.OFS - Overflow Error
7606  *
7607  * Command list overflow is defined as software building a command table
7608  * that has fewer total bytes than the transaction given to the device.
7609  * On device writes, the HBA will run out of data, and on reads, there
7610  * will be no room to put the data.
7611  *
7612  * For an overflow on data read, either PIO or DMA, the HBA will set
7613  * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7614  * non-fatal error when the HBA can continues. Sometimes, it will cause
7615  * a fatal error and need the software to do something.
7616  *
7617  * For an overflow on data write, setting PxIS.OFS is optional for both
7618  * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7619  * software to clean up from this serious error.
7620  *
7621  * PxIS.INFS - Interface Non-Fatal Error
7622  *
7623  * This interrupt event indicates that the HBA encountered an error on
7624  * the Serial ATA interface but was able to continue operation. The kind
7625  * of error usually occurred during a non-Data FIS, and under this condition
7626  * the FIS will be re-transmitted by HBA automatically.
7627  *
7628  * When the FMA is implemented, there should be a stat structure to
7629  * record how many every kind of error happens.
7630  */
7631 static int
7632 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7633     uint8_t port, uint32_t intr_status)
7634 {
7635         uint32_t port_serror;
7636 #if AHCI_DEBUG
7637         uint32_t port_cmd_status;
7638         uint32_t port_cmd_issue;
7639         uint32_t port_sactive;
7640         int current_slot;
7641         uint32_t current_tags;
7642         sata_pkt_t *satapkt;
7643         ahci_cmd_header_t *cmd_header;
7644         uint32_t cmd_dmacount;
7645 #endif
7646 
7647         mutex_enter(&ahci_portp->ahciport_mutex);
7648 
7649         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7650             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7651 
7652         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7653             "ahci_intr_non_fatal_error: port %d, "
7654             "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7655 
7656         ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7657 
7658         if (intr_status & AHCI_INTR_STATUS_UFS) {
7659                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7660                     "ahci port %d has unknown FIS error", port);
7661 
7662                 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7663                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7664                     (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7665                     SERROR_FIS_TYPE);
7666         }
7667 
7668 #if AHCI_DEBUG
7669         if (intr_status & AHCI_INTR_STATUS_IPMS) {
7670                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7671                     "has Incorrect Port Multiplier Status error", port);
7672         }
7673 
7674         if (intr_status & AHCI_INTR_STATUS_OFS) {
7675                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7676                     "ahci port %d has overflow error", port);
7677         }
7678 
7679         if (intr_status & AHCI_INTR_STATUS_INFS) {
7680                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7681                     "ahci port %d has interface non fatal error", port);
7682         }
7683 
7684         /*
7685          * Record the error occurred command's slot.
7686          */
7687         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7688             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7689                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7690                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7691 
7692                 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7693                     AHCI_CMD_STATUS_CCS_SHIFT;
7694 
7695                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7696                         satapkt = ahci_portp->ahciport_err_retri_pkt;
7697                         ASSERT(satapkt != NULL);
7698                         ASSERT(current_slot == 0);
7699                 } else {
7700                         satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7701                 }
7702 
7703                 if (satapkt != NULL) {
7704                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7705                             "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7706                             "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7707                             satapkt->satapkt_cmd.satacmd_cmd_reg);
7708 
7709                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7710                             "ahci_intr_non_fatal_error: port %d, "
7711                             "satapkt 0x%p is being processed when error occurs",
7712                             port, (void *)satapkt);
7713 
7714                         /*
7715                          * PRD Byte Count field of command header is not
7716                          * required to reflect the total number of bytes
7717                          * transferred when an overflow occurs, so here
7718                          * just log the value.
7719                          */
7720                         cmd_dmacount =
7721                             ahci_portp->ahciport_prd_bytecounts[current_slot];
7722                         if (cmd_dmacount) {
7723                                 cmd_header = &ahci_portp->
7724                                     ahciport_cmd_list[current_slot];
7725                                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7726                                     "ahci_intr_non_fatal_error: port %d, "
7727                                     "PRD Byte Count = 0x%x, "
7728                                     "ahciport_prd_bytecounts = 0x%x", port,
7729                                     cmd_header->ahcich_prd_byte_count,
7730                                     cmd_dmacount);
7731                         }
7732                 }
7733         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7734                 /*
7735                  * For queued command, list those command which have already
7736                  * been transmitted to the device and still not completed.
7737                  */
7738                 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7739                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7740 
7741                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7742                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7743 
7744                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7745                     "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7746                     "port_sactive = 0x%x port_cmd_issue = 0x%x",
7747                     ahci_portp->ahciport_pending_ncq_tags,
7748                     port_sactive, port_cmd_issue);
7749 
7750                 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7751                     port_sactive & ~port_cmd_issue &
7752                     AHCI_NCQ_SLOT_MASK(ahci_portp);
7753 
7754                 while (current_tags) {
7755                         current_slot = ddi_ffs(current_tags) - 1;
7756                         if (current_slot == -1) {
7757                                 goto out;
7758                         }
7759 
7760                         satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7761                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7762                             ahci_ctlp, "ahci_intr_non_fatal_error: "
7763                             "port %d, satapkt 0x%p is outstanding when "
7764                             "error occurs", port, (void *)satapkt);
7765 
7766                         CLEAR_BIT(current_tags, current_slot);
7767                 }
7768         }
7769 out:
7770 #endif
7771         mutex_exit(&ahci_portp->ahciport_mutex);
7772 
7773         return (AHCI_SUCCESS);
7774 }
7775 
7776 /*
7777  * According to the AHCI spec, the error types include system memory
7778  * errors, interface errors, port multiplier errors, device errors,
7779  * command list overflow, command list underflow, native command
7780  * queuing tag errors and pio data transfer errors.
7781  *
7782  * System memory errors such as target abort, master abort, and parity
7783  * may cause the host to stop, and they are serious errors and needed
7784  * to be recovered with software intervention. When system software
7785  * has given a pointer to the HBA that doesn't exist in physical memory,
7786  * a master/target abort error occurs, and PxIS.HBFS will be set. A
7787  * data error such as CRC or parity occurs, the HBA aborts the transfer
7788  * (if necessary) and PxIS.HBDS will be set.
7789  *
7790  * Interface errors are errors that occur due to electrical issues on
7791  * the interface, or protocol miscommunication between the device and
7792  * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7793  * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7794  * causes PxIS.IFS/PxIS.INFS to be set are
7795  *      1. in PxSERR.ERR, P bit is set to '1'
7796  *      2. in PxSERR.DIAG, C or H bit is set to '1'
7797  *      3. PhyRdy drop unexpectly, N bit is set to '1'
7798  * If the error occurred during a non-data FIS, the FIS must be
7799  * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7800  * the error occurred during a data FIS, the transfer will stop, so
7801  * the error is fatal and PxIS.IFS is set.
7802  *
7803  * When a FIS arrives that updates the taskfile, the HBA checks to see
7804  * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7805  * stops processing any more commands.
7806  *
7807  * Command list overflow is defined as software building a command table
7808  * that has fewer total bytes than the transaction given to the device.
7809  * On device writes, the HBA will run out of data, and on reads, there
7810  * will be no room to put the data. For an overflow on data read, either
7811  * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7812  * For an overflow on data write, setting PxIS.OFS is optional for both
7813  * DMA and PIO, and a COMRESET is required by software to clean up from
7814  * this serious error.
7815  *
7816  * Command list underflow is defined as software building a command
7817  * table that has more total bytes than the transaction given to the
7818  * device. For data writes, both PIO and DMA, the device will detect
7819  * an error and end the transfer. And these errors are most likely going
7820  * to be fatal errors that will cause the port to be restarted. For
7821  * data reads, the HBA updates its PRD byte count, and may be
7822  * able to continue normally, but is not required to. And The HBA is
7823  * not required to detect underflow conditions for native command
7824  * queuing command.
7825  *
7826  * The HBA does not actively check incoming DMA Setup FISes to ensure
7827  * that the PxSACT register bit for that slot is set. Existing error
7828  * mechanisms, such as host bus failure, or bad protocol, are used to
7829  * recover from this case.
7830  *
7831  * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7832  * DATA FIS must be an integral number of Dwords. If the HBA receives
7833  * a request which is not an integral number of Dwords, the HBA
7834  * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7835  * software restarts the port. And the HBA ensures that the size
7836  * of the DATA FIS received during a PIO command matches the size in
7837  * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7838  * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7839  * stop running until software restarts the port.
7840  */
7841 /*
7842  * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7843  *
7844  * PxIS.IFS indicates that the hba encountered an error on the serial ata
7845  * interface which caused the transfer to stop.
7846  *
7847  * PxIS.HBDS indicates that the hba encountered a data error
7848  * (uncorrectable ecc/parity) when reading from or writing to system memory.
7849  *
7850  * PxIS.HBFS indicates that the hba encountered a host bus error that it
7851  * cannot recover from, such as a bad software pointer.
7852  *
7853  * PxIS.TFES is set whenever the status register is updated by the device
7854  * and the error bit (bit 0) is set.
7855  */
7856 static int
7857 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7858     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7859 {
7860         uint32_t port_cmd_status;
7861         uint32_t port_serror;
7862         uint32_t task_file_status;
7863         int failed_slot;
7864         sata_pkt_t *spkt = NULL;
7865         uint8_t err_byte;
7866         ahci_event_arg_t *args;
7867         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7868         uint32_t failed_tags = 0;
7869         int task_fail_flag = 0, task_abort_flag = 0;
7870         uint32_t slot_status;
7871 
7872         mutex_enter(&ahci_portp->ahciport_mutex);
7873 
7874         /*
7875          * ahci_intr_phyrdy_change() may have rendered it to
7876          * SATA_DTYPE_NONE.
7877          */
7878         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7879                 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7880                     "ahci_intr_fatal_error: port %d no device attached, "
7881                     "and just return without doing anything", port);
7882                 goto out0;
7883         }
7884 
7885         if (intr_status & AHCI_INTR_STATUS_TFES) {
7886                 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7887                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7888                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7889                     "ahci_intr_fatal_error: port %d "
7890                     "task_file_status = 0x%x", port, task_file_status);
7891                 task_fail_flag = 1;
7892 
7893                 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7894                     >> AHCI_TFD_ERR_SHIFT;
7895                 if (err_byte == SATA_ERROR_ABORT)
7896                         task_abort_flag = 1;
7897         }
7898 
7899         /*
7900          * Here we just log the fatal error info in interrupt context.
7901          * Misc recovery processing will be handled in task queue.
7902          */
7903         if (task_fail_flag  == 1) {
7904                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7905                         /*
7906                          * Read PxCMD.CCS to determine the slot that the HBA
7907                          * was processing when the error occurred.
7908                          */
7909                         port_cmd_status = ddi_get32(
7910                             ahci_ctlp->ahcictl_ahci_acc_handle,
7911                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7912                         failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7913                             AHCI_CMD_STATUS_CCS_SHIFT;
7914                         failed_tags = 0x1 << failed_slot;
7915 
7916                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7917                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7918                             "ahci_intr_fatal_error: spkt 0x%p is being "
7919                             "processed when fatal error occurred for port %d",
7920                             spkt, port);
7921 
7922                         /*
7923                          * Won't emit the error message if it is an IDENTIFY
7924                          * DEVICE command sent to an ATAPI device.
7925                          */
7926                         if ((spkt != NULL) &&
7927                             (spkt->satapkt_cmd.satacmd_cmd_reg ==
7928                             SATAC_ID_DEVICE) &&
7929                             (task_abort_flag == 1))
7930                                 goto out1;
7931 
7932                         /*
7933                          * Won't emit the error message if it is an ATAPI PACKET
7934                          * command
7935                          */
7936                         if ((spkt != NULL) &&
7937                             (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7938                                 goto out1;
7939 
7940                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7941                         slot_status = ddi_get32(
7942                             ahci_ctlp->ahcictl_ahci_acc_handle,
7943                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7944                         failed_tags = slot_status &
7945                             AHCI_NCQ_SLOT_MASK(ahci_portp);
7946                 }
7947         }
7948 
7949         /* print the fatal error type */
7950         ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7951         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7952 
7953         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7954             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7955 
7956         /* print PxSERR related error message */
7957         ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7958 
7959         /* print task file register value */
7960         if (task_fail_flag == 1) {
7961                 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7962                     "= 0x%x", instance, port, task_file_status);
7963                 if (task_abort_flag == 1) {
7964                         cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7965                             "port %d are aborted", instance, port);
7966                         ahci_dump_commands(ahci_ctlp, port, failed_tags);
7967                 }
7968         }
7969 
7970 out1:
7971         /* Prepare the argument for the taskq */
7972         args = ahci_portp->ahciport_event_args;
7973         args->ahciea_ctlp = (void *)ahci_ctlp;
7974         args->ahciea_portp = (void *)ahci_portp;
7975         args->ahciea_event = intr_status;
7976         AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7977 
7978         /* Start the taskq to handle error recovery */
7979         if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7980             ahci_events_handler,
7981             (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7982                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7983                 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7984                     "port %d failed", instance, port);
7985         }
7986 out0:
7987         mutex_exit(&ahci_portp->ahciport_mutex);
7988 
7989         return (AHCI_SUCCESS);
7990 }
7991 
7992 /*
7993  * Hot Plug Operation for platforms that support Cold Presence Detect.
7994  *
7995  * When set, a device status has changed as detected by the cold presence
7996  * detect logic. This bit can either be set due to a non-connected port
7997  * receiving a device, or a connected port having its device removed.
7998  * This bit is only valid if the port supports cold presence detect as
7999  * indicated by PxCMD.CPD set to '1'.
8000  *
8001  * At the moment, this interrupt is not needed and disabled and we just
8002  * log the debug message.
8003  */
8004 static int
8005 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
8006     ahci_port_t *ahci_portp, uint8_t port)
8007 {
8008         uint32_t port_cmd_status;
8009         sata_device_t sdevice;
8010 
8011         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8012             "ahci_intr_cold_port_detect enter, port %d", port);
8013 
8014         mutex_enter(&ahci_portp->ahciport_mutex);
8015 
8016         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8017             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8018         if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
8019                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8020                     "port %d does not support cold presence detect, so "
8021                     "we just ignore this interrupt", port);
8022                 mutex_exit(&ahci_portp->ahciport_mutex);
8023                 return (AHCI_SUCCESS);
8024         }
8025 
8026         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8027             "port %d device status has changed", port);
8028 
8029         bzero((void *)&sdevice, sizeof (sata_device_t));
8030         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
8031         sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
8032         sdevice.satadev_addr.pmport = 0;
8033         sdevice.satadev_state = SATA_PSTATE_PWRON;
8034 
8035         if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
8036                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8037                     "port %d: a device is hot plugged", port);
8038                 mutex_exit(&ahci_portp->ahciport_mutex);
8039                 sata_hba_event_notify(
8040                     ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8041                     &sdevice,
8042                     SATA_EVNT_DEVICE_ATTACHED);
8043                 mutex_enter(&ahci_portp->ahciport_mutex);
8044 
8045         } else {
8046                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8047                     "port %d: a device is hot unplugged", port);
8048                 mutex_exit(&ahci_portp->ahciport_mutex);
8049                 sata_hba_event_notify(
8050                     ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8051                     &sdevice,
8052                     SATA_EVNT_DEVICE_DETACHED);
8053                 mutex_enter(&ahci_portp->ahciport_mutex);
8054         }
8055 
8056         mutex_exit(&ahci_portp->ahciport_mutex);
8057 
8058         return (AHCI_SUCCESS);
8059 }
8060 
8061 /*
8062  * Enable the interrupts for a particular port.
8063  */
8064 static void
8065 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8066 {
8067         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8068 
8069         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8070             "ahci_enable_port_intrs enter, port %d", port);
8071 
8072         /*
8073          * Clear port interrupt status before enabling interrupt
8074          */
8075         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8076             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
8077             AHCI_PORT_INTR_MASK);
8078 
8079         /*
8080          * Clear the pending bit from IS.IPS
8081          */
8082         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8083             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
8084 
8085         /*
8086          * Enable the following interrupts:
8087          *      Device to Host Register FIS Interrupt (DHRS)
8088          *      PIO Setup FIS Interrupt (PSS)
8089          *      Set Device Bits Interrupt (SDBS)
8090          *      Unknown FIS Interrupt (UFS)
8091          *      Port Connect Change Status (PCS)
8092          *      PhyRdy Change Status (PRCS)
8093          *      Overflow Status (OFS)
8094          *      Interface Non-fatal Error Status (INFS)
8095          *      Interface Fatal Error Status (IFS)
8096          *      Host Bus Data Error Status (HBDS)
8097          *      Host Bus Fatal Error Status (HBFS)
8098          *      Task File Error Status (TFES)
8099          */
8100         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8101             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8102             (AHCI_INTR_STATUS_DHRS |
8103             AHCI_INTR_STATUS_PSS |
8104             AHCI_INTR_STATUS_SDBS |
8105             AHCI_INTR_STATUS_UFS |
8106             AHCI_INTR_STATUS_DPS |
8107             AHCI_INTR_STATUS_PCS |
8108             AHCI_INTR_STATUS_PRCS |
8109             AHCI_INTR_STATUS_OFS |
8110             AHCI_INTR_STATUS_INFS |
8111             AHCI_INTR_STATUS_IFS |
8112             AHCI_INTR_STATUS_HBDS |
8113             AHCI_INTR_STATUS_HBFS |
8114             AHCI_INTR_STATUS_TFES));
8115 }
8116 
8117 /*
8118  * Enable interrupts for all the ports.
8119  */
8120 static void
8121 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8122 {
8123         uint32_t ghc_control;
8124 
8125         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8126 
8127         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8128 
8129         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8130             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8131 
8132         ghc_control |= AHCI_HBA_GHC_IE;
8133 
8134         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8135             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8136 }
8137 
8138 /*
8139  * Disable interrupts for a particular port.
8140  */
8141 static void
8142 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8143 {
8144         ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8145             MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8146 
8147         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8148             "ahci_disable_port_intrs enter, port %d", port);
8149 
8150         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8151             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8152 }
8153 
8154 /*
8155  * Disable interrupts for the whole HBA.
8156  *
8157  * The global bit is cleared, then all interrupt sources from all
8158  * ports are disabled.
8159  */
8160 static void
8161 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8162 {
8163         uint32_t ghc_control;
8164 
8165         ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8166             MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8167 
8168         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8169             NULL);
8170 
8171         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8172             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8173 
8174         ghc_control &= ~AHCI_HBA_GHC_IE;
8175 
8176         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8177             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8178 }
8179 
8180 /*
8181  * Handle FIXED or MSI interrupts.
8182  */
8183 /*
8184  * According to AHCI spec, the HBA may support several interrupt modes:
8185  *      * pin based interrupts (FIXED)
8186  *      * single MSI message interrupts
8187  *      * multiple MSI based message interrupts
8188  *
8189  * For pin based interrupts, the software interrupt handler need to check IS
8190  * register to find out which port has pending interrupts. And then check
8191  * PxIS register to find out which interrupt events happened on that port.
8192  *
8193  * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8194  * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8195  * in that software interrupt handler need to check IS register to determine
8196  * which port triggered the interrupts since it uses a single message for all
8197  * port interrupts.
8198  *
8199  * HBA may optionally support multiple MSI message for better performance. In
8200  * this mode, each port may have its own interrupt message, and thus generation
8201  * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8202  * represents a power-of-2 wrapper on the number of implemented ports, and
8203  * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8204  * maximum number of assigned interrupts. When the number of MSI messages
8205  * allocated is less than the number requested, then hardware may have two
8206  * implementation behaviors:
8207  *      * assign each ports its own interrupt and then force all additional
8208  *        ports to share the last interrupt message, and this condition is
8209  *        indicated by clearing GHC.MRSM to '0'
8210  *      * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8211  * When multiple-message MSI is enabled, hardware will still set IS register
8212  * as single message case. And this IS register may be used by software when
8213  * fewer than the requested number of messages is granted in order to determine
8214  * which port had the interrupt.
8215  *
8216  * Note: The current ahci driver only supports the first two interrupt modes:
8217  * pin based interrupts and single MSI message interrupts, and the reason
8218  * is indicated in below code.
8219  */
8220 static int
8221 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8222 {
8223         dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8224         int             count, avail, actual;
8225         int             i, rc;
8226 
8227         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8228             "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8229 
8230         /* get number of interrupts. */
8231         rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8232         if ((rc != DDI_SUCCESS) || (count == 0)) {
8233                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8234                     "ddi_intr_get_nintrs() failed, "
8235                     "rc %d count %d\n", rc, count);
8236                 return (DDI_FAILURE);
8237         }
8238 
8239         /* get number of available interrupts. */
8240         rc = ddi_intr_get_navail(dip, intr_type, &avail);
8241         if ((rc != DDI_SUCCESS) || (avail == 0)) {
8242                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8243                     "ddi_intr_get_navail() failed, "
8244                     "rc %d avail %d\n", rc, avail);
8245                 return (DDI_FAILURE);
8246         }
8247 
8248 #if AHCI_DEBUG
8249         if (avail < count) {
8250                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8251                     "ddi_intr_get_nintrs returned %d, navail() returned %d",
8252                     count, avail);
8253         }
8254 #endif
8255 
8256         /*
8257          * Note: So far Solaris restricts the maximum number of messages for
8258          * x86 to 2, that is avail is 2, so here we set the count with 1 to
8259          * force the driver to use single MSI message interrupt. In future if
8260          * Solaris remove the restriction, then we need to delete the below
8261          * code and try to use multiple interrupt routine to gain better
8262          * performance.
8263          */
8264         if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8265                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8266                     "force to use one interrupt routine though the "
8267                     "HBA supports %d interrupt", count);
8268                 count = 1;
8269         }
8270 
8271         /* Allocate an array of interrupt handles. */
8272         ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8273         ahci_ctlp->ahcictl_intr_htable =
8274             kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8275 
8276         /* call ddi_intr_alloc(). */
8277         rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8278             intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8279 
8280         if ((rc != DDI_SUCCESS) || (actual == 0)) {
8281                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8282                     "ddi_intr_alloc() failed, rc %d count %d actual %d "
8283                     "avail %d\n", rc, count, actual, avail);
8284                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8285                     ahci_ctlp->ahcictl_intr_size);
8286                 return (DDI_FAILURE);
8287         }
8288 
8289         /* use interrupt count returned */
8290 #if AHCI_DEBUG
8291         if (actual < count) {
8292                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8293                     "Requested: %d, Received: %d", count, actual);
8294         }
8295 #endif
8296 
8297         ahci_ctlp->ahcictl_intr_cnt = actual;
8298 
8299         /*
8300          * Get priority for first, assume remaining are all the same.
8301          */
8302         if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8303             &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8304                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8305                     "ddi_intr_get_pri() failed", NULL);
8306 
8307                 /* Free already allocated intr. */
8308                 for (i = 0; i < actual; i++) {
8309                         (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8310                 }
8311 
8312                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8313                     ahci_ctlp->ahcictl_intr_size);
8314                 return (DDI_FAILURE);
8315         }
8316 
8317         /* Test for high level interrupt. */
8318         if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8319                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8320                     "ahci_add_intrs: Hi level intr not supported", NULL);
8321 
8322                 /* Free already allocated intr. */
8323                 for (i = 0; i < actual; i++) {
8324                         (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8325                 }
8326 
8327                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8328                     sizeof (ddi_intr_handle_t));
8329 
8330                 return (DDI_FAILURE);
8331         }
8332 
8333         /* Call ddi_intr_add_handler(). */
8334         for (i = 0; i < actual; i++) {
8335                 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8336                     ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8337                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8338                             "ddi_intr_add_handler() failed", NULL);
8339 
8340                         /* Free already allocated intr. */
8341                         for (i = 0; i < actual; i++) {
8342                                 (void) ddi_intr_free(
8343                                     ahci_ctlp->ahcictl_intr_htable[i]);
8344                         }
8345 
8346                         kmem_free(ahci_ctlp->ahcictl_intr_htable,
8347                             ahci_ctlp->ahcictl_intr_size);
8348                         return (DDI_FAILURE);
8349                 }
8350         }
8351 
8352         if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8353             &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8354                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8355                     "ddi_intr_get_cap() failed", NULL);
8356 
8357                 /* Free already allocated intr. */
8358                 for (i = 0; i < actual; i++) {
8359                         (void) ddi_intr_free(
8360                             ahci_ctlp->ahcictl_intr_htable[i]);
8361                 }
8362 
8363                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8364                     ahci_ctlp->ahcictl_intr_size);
8365                 return (DDI_FAILURE);
8366         }
8367 
8368         if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8369                 /* Call ddi_intr_block_enable() for MSI. */
8370                 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8371                     ahci_ctlp->ahcictl_intr_cnt);
8372         } else {
8373                 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8374                 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8375                         (void) ddi_intr_enable(
8376                             ahci_ctlp->ahcictl_intr_htable[i]);
8377                 }
8378         }
8379 
8380         return (DDI_SUCCESS);
8381 }
8382 
8383 /*
8384  * Removes the registered interrupts irrespective of whether they
8385  * were legacy or MSI.
8386  *
8387  * NOTE: The controller interrupts must be disabled before calling
8388  * this routine.
8389  */
8390 static void
8391 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8392 {
8393         int x;
8394 
8395         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8396 
8397         /* Disable all interrupts. */
8398         if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8399             (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8400                 /* Call ddi_intr_block_disable(). */
8401                 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8402                     ahci_ctlp->ahcictl_intr_cnt);
8403         } else {
8404                 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8405                         (void) ddi_intr_disable(
8406                             ahci_ctlp->ahcictl_intr_htable[x]);
8407                 }
8408         }
8409 
8410         /* Call ddi_intr_remove_handler(). */
8411         for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8412                 (void) ddi_intr_remove_handler(
8413                     ahci_ctlp->ahcictl_intr_htable[x]);
8414                 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8415         }
8416 
8417         kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8418 }
8419 
8420 /*
8421  * This routine tries to put port into P:NotRunning state by clearing
8422  * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8423  * and PxCMD.CR to '0'.
8424  */
8425 static int
8426 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8427     ahci_port_t *ahci_portp, uint8_t port)
8428 {
8429         uint32_t port_cmd_status;
8430         int loop_count;
8431 
8432         ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8433             MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8434 
8435         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8436             "ahci_put_port_into_notrunning_state enter: port %d", port);
8437 
8438         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8439             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8440 
8441         port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8442         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8443             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8444 
8445         /* Wait until PxCMD.CR is cleared */
8446         loop_count = 0;
8447         do {
8448                 port_cmd_status =
8449                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8450                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8451 
8452                 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8453                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8454                             "clearing port %d CMD.CR timeout, "
8455                             "port_cmd_status = 0x%x", port,
8456                             port_cmd_status);
8457                         /*
8458                          * We are effectively timing out after 0.5 sec.
8459                          * This value is specified in AHCI spec.
8460                          */
8461                         break;
8462                 }
8463 
8464                 /* Wait for 10 millisec */
8465                 drv_usecwait(AHCI_10MS_USECS);
8466         } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8467 
8468         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8469 
8470         if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8471                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8472                     "ahci_put_port_into_notrunning_state: failed to clear "
8473                     "PxCMD.CR to '0' after loop count: %d, and "
8474                     "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8475                 return (AHCI_FAILURE);
8476         } else {
8477                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8478                     "ahci_put_port_into_notrunning_state: succeeded to clear "
8479                     "PxCMD.CR to '0' after loop count: %d, and "
8480                     "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8481                 return (AHCI_SUCCESS);
8482         }
8483 }
8484 
8485 /*
8486  * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8487  * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8488  * stable state, then set PxCMD.ST to '1' to start the port directly.
8489  * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8490  * COMRESET to the device to put it in an idle state.
8491  *
8492  * The fifth argument returns whether the port reset is involved during
8493  * the process.
8494  *
8495  * The routine will be called under following scenarios:
8496  *      + To reset the HBA
8497  *      + To abort the packet(s)
8498  *      + To reset the port
8499  *      + To activate the port
8500  *      + Fatal error recovery
8501  *      + To abort the timeout packet(s)
8502  *
8503  * NOTES!!! During this procedure, PxSERR register will be cleared, and
8504  * according to the spec, the clearance of three bits will also clear
8505  * three interrupt status bits.
8506  *      1. PxSERR.DIAG.F will clear PxIS.UFS
8507  *      2. PxSERR.DIAG.X will clear PxIS.PCS
8508  *      3. PxSERR.DIAG.N will clear PxIS.PRCS
8509  *
8510  * Among these three interrupt events, the driver needs to take care of
8511  * PxIS.PRCS, which is the hot plug event. When the driver found out
8512  * a device was unplugged, it will call the interrupt handler.
8513  */
8514 static int
8515 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8516     ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8517 {
8518         uint32_t port_sstatus;
8519         uint32_t task_file_status;
8520         sata_device_t sdevice;
8521         int rval;
8522         ahci_addr_t addr_port;
8523         ahci_pmult_info_t *pminfo = NULL;
8524         int dev_exists_begin = 0;
8525         int dev_exists_end = 0;
8526         uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8527         int npmport = 0;
8528         uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8529 
8530         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8531 
8532         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8533             "ahci_restart_port_wait_till_ready: port %d enter", port);
8534 
8535         AHCI_ADDR_SET_PORT(&addr_port, port);
8536 
8537         if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8538                 dev_exists_begin = 1;
8539 
8540         /* First clear PxCMD.ST */
8541         rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8542             port);
8543         if (rval != AHCI_SUCCESS)
8544                 /*
8545                  * If PxCMD.CR does not clear within a reasonable time, it
8546                  * may assume the interface is in a hung condition and may
8547                  * continue with issuing the port reset.
8548                  */
8549                 goto reset;
8550 
8551         /* Then clear PxSERR */
8552         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8553             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8554             AHCI_SERROR_CLEAR_ALL);
8555 
8556         /* Then get PxTFD */
8557         task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8558             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8559 
8560         /*
8561          * Check whether the device is in a stable status, if yes,
8562          * then start the port directly. However for ahci_tran_reset_dport,
8563          * we may have to perform a port reset.
8564          */
8565         if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8566             !(flag & AHCI_PORT_RESET))
8567                 goto out;
8568 
8569 reset:
8570         /*
8571          * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8572          * a COMRESET to the device
8573          */
8574         ahci_disable_port_intrs(ahci_ctlp, port);
8575         rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8576         ahci_enable_port_intrs(ahci_ctlp, port);
8577 
8578 #ifdef AHCI_DEBUG
8579         if (rval != AHCI_SUCCESS)
8580                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8581                     "ahci_restart_port_wait_till_ready: port %d failed",
8582                     port);
8583 #endif
8584 
8585         if (reset_flag != NULL)
8586                 *reset_flag = 1;
8587 
8588         /* Indicate to the framework that a reset has happened. */
8589         if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8590             (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8591             !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8592                 /* Set the reset in progress flag */
8593                 ahci_portp->ahciport_reset_in_progress = 1;
8594 
8595                 bzero((void *)&sdevice, sizeof (sata_device_t));
8596                 sdevice.satadev_addr.cport =
8597                     ahci_ctlp->ahcictl_port_to_cport[port];
8598                 sdevice.satadev_addr.pmport = 0;
8599                 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8600 
8601                 sdevice.satadev_state = SATA_DSTATE_RESET |
8602                     SATA_DSTATE_PWR_ACTIVE;
8603                 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8604                         mutex_exit(&ahci_portp->ahciport_mutex);
8605                         sata_hba_event_notify(
8606                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8607                             &sdevice,
8608                             SATA_EVNT_DEVICE_RESET);
8609                         mutex_enter(&ahci_portp->ahciport_mutex);
8610                 }
8611 
8612                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8613                     "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8614         } else {
8615                 ahci_portp->ahciport_reset_in_progress = 0;
8616         }
8617 
8618 out:
8619         (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8620 
8621         /* SStatus tells the presence of device. */
8622         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8623             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8624 
8625         if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8626                 dev_exists_end = 1;
8627         }
8628 
8629         if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8630                 return (rval);
8631 
8632         /* Check whether a hot plug event happened */
8633         if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8634                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8635                     "ahci_restart_port_wait_till_ready: port %d "
8636                     "device is removed", port);
8637                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8638                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8639                     "ahci_restart_port_wait_till_ready: port %d "
8640                     "AHCI_PORT_FLAG_NODEV flag is set", port);
8641                 mutex_exit(&ahci_portp->ahciport_mutex);
8642                 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8643                 mutex_enter(&ahci_portp->ahciport_mutex);
8644 
8645                 return (rval);
8646         }
8647 
8648 
8649         /* 0/1 -> 1 : device may change */
8650         /*
8651          * May be called by ahci_fatal_error_recovery_handler, so
8652          * don't issue software if the previous device is ATAPI.
8653          */
8654         if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8655                 return (rval);
8656 
8657         /*
8658          * The COMRESET will make port multiplier enter legacy mode.
8659          * Issue a software reset to make it work again.
8660          */
8661         ahci_disable_port_intrs(ahci_ctlp, port);
8662         ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8663         ahci_enable_port_intrs(ahci_ctlp, port);
8664 
8665         /*
8666          * Following codes are specific for the port multiplier
8667          */
8668         if (previous_dev_type != SATA_DTYPE_PMULT &&
8669             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8670                 /* in case previous_dev_type is corrupt */
8671                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8672                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8673                 return (rval);
8674         }
8675 
8676         /* Device change: PMult -> Non-PMult */
8677         if (previous_dev_type == SATA_DTYPE_PMULT &&
8678             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8679                 /*
8680                  * This might happen because
8681                  * 1. Software reset failed. Port multiplier is not correctly
8682                  *    enumerated.
8683                  * 2. Another non-port-multiplier device is attached. Perhaps
8684                  *    the port multiplier was replaced by another device by
8685                  *    whatever reason, but AHCI driver missed hot-plug event.
8686                  *
8687                  * Now that the port has been initialized, we just need to
8688                  * update the port structure according new device, then report
8689                  * and wait SATA framework to probe new device.
8690                  */
8691 
8692                 /* Force to release pmult resource */
8693                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8694                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8695 
8696                 bzero((void *)&sdevice, sizeof (sata_device_t));
8697                 sdevice.satadev_addr.cport =
8698                     ahci_ctlp->ahcictl_port_to_cport[port];
8699                 sdevice.satadev_addr.pmport = 0;
8700                 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8701 
8702                 sdevice.satadev_state = SATA_DSTATE_RESET |
8703                     SATA_DSTATE_PWR_ACTIVE;
8704 
8705                 mutex_exit(&ahci_portp->ahciport_mutex);
8706                 sata_hba_event_notify(
8707                     ahci_ctlp->ahcictl_dip,
8708                     &sdevice,
8709                     SATA_EVNT_DEVICE_RESET);
8710                 mutex_enter(&ahci_portp->ahciport_mutex);
8711 
8712                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8713                     "Port multiplier is [Gone] at port %d ", port);
8714                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8715                     "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8716 
8717                 return (AHCI_SUCCESS);
8718         }
8719 
8720         /* Device change: Non-PMult -> PMult */
8721         if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8722 
8723                 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8724                 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8725 
8726                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8727         }
8728         pminfo = ahci_portp->ahciport_pmult_info;
8729         ASSERT(pminfo != NULL);
8730 
8731         /* Device (may) change: PMult -> PMult */
8732         /*
8733          * First initialize port multiplier. Set state to READY and wait for
8734          * probe entry point to initialize it
8735          */
8736         ahci_portp->ahciport_port_state = SATA_STATE_READY;
8737 
8738         /*
8739          * It's a little complicated while target is a port multiplier. we
8740          * need to COMRESET all pmports behind that PMult otherwise those
8741          * sub-links between the PMult and the sub-devices will be in an
8742          * inactive state (indicated by PSCR0/PxSSTS) and the following access
8743          * to those sub-devices will be rejected by Link-Fatal-Error.
8744          */
8745         /*
8746          * The PxSNTF will be set soon after the pmult is plugged. While the
8747          * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8748          * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8749          * will initialized it.
8750          */
8751         for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8752                 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8753 
8754         /* Report reset event. */
8755         ahci_portp->ahciport_reset_in_progress = 1;
8756 
8757         bzero((void *)&sdevice, sizeof (sata_device_t));
8758         sdevice.satadev_addr.cport = cport;
8759         sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8760         sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8761         sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8762         sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8763             SATA_EVNT_DEVICE_RESET);
8764 
8765         return (rval);
8766 }
8767 
8768 /*
8769  * This routine may be called under four scenarios:
8770  *      a) do the recovery from fatal error
8771  *      b) or we need to timeout some commands
8772  *      c) or we need to abort some commands
8773  *      d) or we need reset device/port/controller
8774  *
8775  * In all these scenarios, we need to send any pending unfinished
8776  * commands up to sata framework.
8777  */
8778 static void
8779 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8780     ahci_port_t *ahci_portp,
8781     uint32_t slot_status,
8782     uint32_t failed_tags,
8783     uint32_t timeout_tags,
8784     uint32_t aborted_tags,
8785     uint32_t reset_tags)
8786 {
8787         uint32_t finished_tags = 0;
8788         uint32_t unfinished_tags = 0;
8789         int tmp_slot;
8790         sata_pkt_t *satapkt;
8791         int ncq_cmd_in_progress = 0;
8792         int err_retri_cmd_in_progress = 0;
8793         int rdwr_pmult_cmd_in_progress = 0;
8794 
8795         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8796 
8797         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8798             "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8799             ahci_portp->ahciport_port_num, slot_status);
8800 
8801         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8802             "ahci_mop_commands: failed_tags: 0x%x, "
8803             "timeout_tags: 0x%x aborted_tags: 0x%x, "
8804             "reset_tags: 0x%x", failed_tags,
8805             timeout_tags, aborted_tags, reset_tags);
8806 
8807 #ifdef AHCI_DEBUG
8808         if (ahci_debug_flags & AHCIDBG_ERRS) {
8809                 int i;
8810                 char msg_buf[200] = {0, };
8811                 for (i = 0x1f; i >= 0; i--) {
8812                         if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8813                                 msg_buf[i] = 'X';
8814                         else
8815                                 msg_buf[i] = '.';
8816                 }
8817                 msg_buf[0x20] = '\0';
8818                 cmn_err(CE_NOTE, "port[%d] slots: %s",
8819                     ahci_portp->ahciport_port_num, msg_buf);
8820                 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8821                     (void *)ahci_portp->ahciport_err_retri_pkt,
8822                     (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8823         }
8824 #endif
8825 
8826         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8827                 finished_tags = ahci_portp->ahciport_pending_tags &
8828                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8829 
8830                 unfinished_tags = slot_status &
8831                     AHCI_SLOT_MASK(ahci_ctlp) &
8832                     ~failed_tags &
8833                     ~aborted_tags &
8834                     ~reset_tags &
8835                     ~timeout_tags;
8836         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8837                 ncq_cmd_in_progress = 1;
8838                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8839                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8840 
8841                 unfinished_tags = slot_status &
8842                     AHCI_NCQ_SLOT_MASK(ahci_portp) &
8843                     ~failed_tags &
8844                     ~aborted_tags &
8845                     ~reset_tags &
8846                     ~timeout_tags;
8847         } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8848 
8849                 /*
8850                  * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8851                  * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8852                  * complete successfully due to one of the following three
8853                  * conditions:
8854                  *
8855                  *      1. Fatal error - failed_tags includes its slot
8856                  *      2. Timed out - timeout_tags includes its slot
8857                  *      3. Aborted when hot unplug - aborted_tags includes its
8858                  *         slot
8859                  *
8860                  * Please note that the command is always sent down in Slot 0
8861                  */
8862                 err_retri_cmd_in_progress = 1;
8863                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8864                     "ahci_mop_commands is called for port %d while "
8865                     "REQUEST SENSE or READ LOG EXT for error retrieval "
8866                     "is being executed slot_status = 0x%x",
8867                     ahci_portp->ahciport_port_num, slot_status);
8868                 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8869                 ASSERT(slot_status == 0x1);
8870         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8871                 rdwr_pmult_cmd_in_progress = 1;
8872                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8873                     "ahci_mop_commands is called for port %d while "
8874                     "READ/WRITE PORTMULT command is being executed",
8875                     ahci_portp->ahciport_port_num);
8876 
8877                 ASSERT(slot_status == 0x1);
8878         }
8879 
8880 #ifdef AHCI_DEBUG
8881         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8882             "ahci_mop_commands: finished_tags: 0x%x, "
8883             "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8884 #endif
8885 
8886         /* Send up finished packets with SATA_PKT_COMPLETED */
8887         while (finished_tags) {
8888                 tmp_slot = ddi_ffs(finished_tags) - 1;
8889                 if (tmp_slot == -1) {
8890                         break;
8891                 }
8892 
8893                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8894                 ASSERT(satapkt != NULL);
8895 
8896                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8897                     "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8898                     (void *)satapkt);
8899 
8900                 /*
8901                  * Cannot fetch the return register content since the port
8902                  * was restarted, so the corresponding tag will be set to
8903                  * aborted tags.
8904                  */
8905                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8906                         CLEAR_BIT(finished_tags, tmp_slot);
8907                         aborted_tags |= tmp_slot;
8908                         continue;
8909                 }
8910 
8911                 if (ncq_cmd_in_progress)
8912                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8913                             tmp_slot);
8914                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8915                 CLEAR_BIT(finished_tags, tmp_slot);
8916                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8917 
8918                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8919         }
8920 
8921         /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8922         while (failed_tags) {
8923                 if (err_retri_cmd_in_progress) {
8924                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8925                         ASSERT(satapkt != NULL);
8926                         ASSERT(failed_tags == 0x1);
8927 
8928                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8929                             "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8930                             (void *)satapkt);
8931                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8932                         break;
8933                 }
8934                 if (rdwr_pmult_cmd_in_progress) {
8935                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8936                         ASSERT(satapkt != NULL);
8937                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8938                             "ahci_mop_commands: sending up "
8939                             "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8940                             (void *)satapkt);
8941                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8942                         break;
8943                 }
8944 
8945                 tmp_slot = ddi_ffs(failed_tags) - 1;
8946                 if (tmp_slot == -1) {
8947                         break;
8948                 }
8949 
8950                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8951                 ASSERT(satapkt != NULL);
8952 
8953                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8954                     "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8955                     (void *)satapkt);
8956 
8957                 if (ncq_cmd_in_progress)
8958                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8959                             tmp_slot);
8960                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8961                 CLEAR_BIT(failed_tags, tmp_slot);
8962                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8963 
8964                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8965         }
8966 
8967         /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8968         while (timeout_tags) {
8969                 if (err_retri_cmd_in_progress) {
8970                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8971                         ASSERT(satapkt != NULL);
8972                         ASSERT(timeout_tags == 0x1);
8973 
8974                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8975                             "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8976                             (void *)satapkt);
8977                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8978                         break;
8979                 }
8980                 if (rdwr_pmult_cmd_in_progress) {
8981                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8982                         ASSERT(satapkt != NULL);
8983                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8984                             "ahci_mop_commands: sending up "
8985                             "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8986                             (void *)satapkt);
8987                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8988                         break;
8989                 }
8990 
8991                 tmp_slot = ddi_ffs(timeout_tags) - 1;
8992                 if (tmp_slot == -1) {
8993                         break;
8994                 }
8995 
8996                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8997                 ASSERT(satapkt != NULL);
8998 
8999                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9000                     "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
9001                     (void *)satapkt);
9002 
9003                 if (ncq_cmd_in_progress)
9004                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9005                             tmp_slot);
9006                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9007                 CLEAR_BIT(timeout_tags, tmp_slot);
9008                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9009 
9010                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
9011         }
9012 
9013         /* Send up aborted packets with SATA_PKT_ABORTED */
9014         while (aborted_tags) {
9015                 if (err_retri_cmd_in_progress) {
9016                         satapkt = ahci_portp->ahciport_err_retri_pkt;
9017                         ASSERT(satapkt != NULL);
9018                         ASSERT(aborted_tags == 0x1);
9019 
9020                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9021                             "sending up pkt 0x%p with SATA_PKT_ABORTED",
9022                             (void *)satapkt);
9023                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9024                         break;
9025                 }
9026                 if (rdwr_pmult_cmd_in_progress) {
9027                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9028                         ASSERT(satapkt != NULL);
9029                         ASSERT(aborted_tags == 0x1);
9030                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9031                             "ahci_mop_commands: sending up "
9032                             "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
9033                             (void *)satapkt);
9034                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9035                         break;
9036                 }
9037 
9038                 tmp_slot = ddi_ffs(aborted_tags) - 1;
9039                 if (tmp_slot == -1) {
9040                         break;
9041                 }
9042 
9043                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9044                 ASSERT(satapkt != NULL);
9045 
9046                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9047                     "sending up pkt 0x%p with SATA_PKT_ABORTED",
9048                     (void *)satapkt);
9049 
9050                 if (ncq_cmd_in_progress)
9051                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9052                             tmp_slot);
9053                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9054                 CLEAR_BIT(aborted_tags, tmp_slot);
9055                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9056 
9057                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9058         }
9059 
9060         /* Send up reset packets with SATA_PKT_RESET. */
9061         while (reset_tags) {
9062                 if (rdwr_pmult_cmd_in_progress) {
9063                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9064                         ASSERT(satapkt != NULL);
9065                         ASSERT(aborted_tags == 0x1);
9066                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9067                             "ahci_mop_commands: sending up "
9068                             "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
9069                             (void *)satapkt);
9070                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9071                         break;
9072                 }
9073 
9074                 tmp_slot = ddi_ffs(reset_tags) - 1;
9075                 if (tmp_slot == -1) {
9076                         break;
9077                 }
9078 
9079                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9080                 ASSERT(satapkt != NULL);
9081 
9082                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9083                     "sending up pkt 0x%p with SATA_PKT_RESET",
9084                     (void *)satapkt);
9085 
9086                 if (ncq_cmd_in_progress)
9087                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9088                             tmp_slot);
9089                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9090                 CLEAR_BIT(reset_tags, tmp_slot);
9091                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9092 
9093                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9094         }
9095 
9096         /* Send up unfinished packets with SATA_PKT_RESET */
9097         while (unfinished_tags) {
9098                 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9099                 if (tmp_slot == -1) {
9100                         break;
9101                 }
9102 
9103                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9104                 ASSERT(satapkt != NULL);
9105 
9106                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9107                     "sending up pkt 0x%p with SATA_PKT_RESET",
9108                     (void *)satapkt);
9109 
9110                 if (ncq_cmd_in_progress)
9111                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9112                             tmp_slot);
9113                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9114                 CLEAR_BIT(unfinished_tags, tmp_slot);
9115                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9116 
9117                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9118         }
9119 
9120         ahci_portp->ahciport_mop_in_progress--;
9121         ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9122 
9123         if (ahci_portp->ahciport_mop_in_progress == 0)
9124                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9125 
9126         ahci_flush_doneq(ahci_portp);
9127 }
9128 
9129 /*
9130  * This routine is going to first request a READ LOG EXT sata pkt from sata
9131  * module, and then deliver it to the HBA to get the ncq failure context.
9132  * The return value is the exactly failed tags.
9133  */
9134 static uint32_t
9135 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9136     uint8_t port)
9137 {
9138         sata_device_t   sdevice;
9139         sata_pkt_t      *rdlog_spkt, *spkt;
9140         ddi_dma_handle_t buf_dma_handle;
9141         ahci_addr_t     addr;
9142         int             loop_count;
9143         int             rval;
9144         int             failed_slot;
9145         uint32_t        failed_tags = 0;
9146         struct sata_ncq_error_recovery_page *ncq_err_page;
9147 
9148         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9149             "ahci_get_rdlogext_data enter: port %d", port);
9150 
9151         /* Prepare the sdevice data */
9152         bzero((void *)&sdevice, sizeof (sata_device_t));
9153         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9154 
9155         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9156         sdevice.satadev_addr.pmport = 0;
9157 
9158         /* Translate sata_device.satadev_addr -> ahci_addr */
9159         ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9160 
9161         /*
9162          * Call the sata hba interface to get a rdlog spkt
9163          */
9164         loop_count = 0;
9165 loop:
9166         rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9167             &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9168         if (rdlog_spkt == NULL) {
9169                 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9170                         /* Sleep for a while */
9171                         drv_usecwait(AHCI_10MS_USECS);
9172                         goto loop;
9173                 }
9174                 /* Timed out after 1s */
9175                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9176                     "failed to get rdlog spkt for port %d", port);
9177                 return (failed_tags);
9178         }
9179 
9180         ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9181 
9182         /*
9183          * This flag is used to handle the specific error recovery when the
9184          * READ LOG EXT command gets a failure (fatal error or time-out).
9185          */
9186         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9187 
9188         /*
9189          * This start is not supposed to fail because after port is restarted,
9190          * the whole command list is empty.
9191          */
9192         ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9193         (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9194         ahci_portp->ahciport_err_retri_pkt = NULL;
9195 
9196         /* Remove the flag after READ LOG EXT command is completed */
9197         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9198 
9199         if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9200                 /* Update the request log data */
9201                 buf_dma_handle = *(ddi_dma_handle_t *)
9202                     (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9203                 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9204                     DDI_DMA_SYNC_FORKERNEL);
9205                 if (rval == DDI_SUCCESS) {
9206                         ncq_err_page =
9207                             (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9208                             satapkt_cmd.satacmd_bp->b_un.b_addr;
9209 
9210                         /* Get the failed tag */
9211                         failed_slot = ncq_err_page->ncq_tag;
9212                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9213                             "ahci_get_rdlogext_data: port %d "
9214                             "failed slot %d", port, failed_slot);
9215                         if (failed_slot & NQ) {
9216                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9217                                     "the failed slot is not a valid tag", NULL);
9218                                 goto out;
9219                         }
9220 
9221                         failed_slot &= NCQ_TAG_MASK;
9222                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9223                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9224                             "ahci_get_rdlogext_data: failed spkt 0x%p",
9225                             (void *)spkt);
9226                         if (spkt == NULL) {
9227                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9228                                     "the failed slot spkt is NULL", NULL);
9229                                 goto out;
9230                         }
9231 
9232                         failed_tags = 0x1 << failed_slot;
9233 
9234                         /* Fill out the error context */
9235                         ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9236                             ncq_err_page);
9237                         ahci_update_sata_registers(ahci_ctlp, port,
9238                             &spkt->satapkt_device);
9239                 }
9240         }
9241 out:
9242         sata_free_error_retrieval_pkt(rdlog_spkt);
9243 
9244         return (failed_tags);
9245 }
9246 
9247 /*
9248  * This routine is going to first request a REQUEST SENSE sata pkt from sata
9249  * module, and then deliver it to the HBA to get the sense data and copy
9250  * the sense data back to the orignal failed sata pkt, and free the REQUEST
9251  * SENSE sata pkt later.
9252  */
9253 static void
9254 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9255     uint8_t port, sata_pkt_t *spkt)
9256 {
9257         sata_device_t   sdevice;
9258         sata_pkt_t      *rs_spkt;
9259         sata_cmd_t      *sata_cmd;
9260         ddi_dma_handle_t buf_dma_handle;
9261         ahci_addr_t     addr;
9262         int             loop_count;
9263 #if AHCI_DEBUG
9264         struct scsi_extended_sense *rqsense;
9265 #endif
9266 
9267         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9268             "ahci_get_rqsense_data enter: port %d", port);
9269 
9270         /* Prepare the sdevice data */
9271         bzero((void *)&sdevice, sizeof (sata_device_t));
9272         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9273 
9274         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9275         sdevice.satadev_addr.pmport = 0;
9276 
9277         /* Translate sata_device.satadev_addr -> ahci_addr */
9278         ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9279 
9280         sata_cmd = &spkt->satapkt_cmd;
9281 
9282         /*
9283          * Call the sata hba interface to get a rs spkt
9284          */
9285         loop_count = 0;
9286 loop:
9287         rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9288             &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9289         if (rs_spkt == NULL) {
9290                 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9291                         /* Sleep for a while */
9292                         drv_usecwait(AHCI_10MS_USECS);
9293                         goto loop;
9294 
9295                 }
9296                 /* Timed out after 1s */
9297                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9298                     "failed to get rs spkt for port %d", port);
9299                 return;
9300         }
9301 
9302         ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9303 
9304         /*
9305          * This flag is used to handle the specific error recovery when the
9306          * REQUEST SENSE command gets a faiure (fatal error or time-out).
9307          */
9308         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9309 
9310         /*
9311          * This start is not supposed to fail because after port is restarted,
9312          * the whole command list is empty.
9313          */
9314         ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9315         (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9316         ahci_portp->ahciport_err_retri_pkt = NULL;
9317 
9318         /* Remove the flag after REQUEST SENSE command is completed */
9319         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9320 
9321         if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9322                 /* Update the request sense data */
9323                 buf_dma_handle = *(ddi_dma_handle_t *)
9324                     (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9325                 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9326                     DDI_DMA_SYNC_FORKERNEL);
9327                 /* Copy the request sense data */
9328                 bcopy(rs_spkt->
9329                     satapkt_cmd.satacmd_bp->b_un.b_addr,
9330                     &sata_cmd->satacmd_rqsense,
9331                     SATA_ATAPI_MIN_RQSENSE_LEN);
9332 #if AHCI_DEBUG
9333                 rqsense = (struct scsi_extended_sense *)
9334                     sata_cmd->satacmd_rqsense;
9335 
9336                 /* Dump the sense data */
9337                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9338                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9339                     "Sense data for satapkt %p ATAPI cmd 0x%x",
9340                     spkt, sata_cmd->satacmd_acdb[0]);
9341                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9342                     "  es_code 0x%x es_class 0x%x "
9343                     "es_key 0x%x es_add_code 0x%x "
9344                     "es_qual_code 0x%x",
9345                     rqsense->es_code, rqsense->es_class,
9346                     rqsense->es_key, rqsense->es_add_code,
9347                     rqsense->es_qual_code);
9348 #endif
9349         }
9350 
9351         sata_free_error_retrieval_pkt(rs_spkt);
9352 }
9353 
9354 /*
9355  * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9356  * the port must be restarted. When the HBA detects thus error, it may try
9357  * to abort a transfer. And if the transfer was aborted, the device is
9358  * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9359  * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9360  * that the device is in a stable status and transfers may be restarted without
9361  * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9362  * then the software will send the COMRESET to do the port reset.
9363  *
9364  * Software should perform the appropriate error recovery actions based on
9365  * whether non-queued commands were being issued or natived command queuing
9366  * commands were being issued.
9367  *
9368  * And software will complete the command that had the error with error mark
9369  * to higher level software.
9370  *
9371  * Fatal errors include the following:
9372  *      PxIS.IFS - Interface Fatal Error Status
9373  *      PxIS.HBDS - Host Bus Data Error Status
9374  *      PxIS.HBFS - Host Bus Fatal Error Status
9375  *      PxIS.TFES - Task File Error Status
9376  */
9377 static void
9378 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9379     ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9380 {
9381         uint32_t        port_cmd_status;
9382         uint32_t        slot_status = 0;
9383         uint32_t        failed_tags = 0;
9384         int             failed_slot;
9385         int             reset_flag = 0, flag = 0;
9386         ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9387         sata_cmd_t      *sata_cmd = NULL;
9388         sata_pkt_t      *spkt = NULL;
9389 #if AHCI_DEBUG
9390         ahci_cmd_header_t *cmd_header;
9391 #endif
9392         uint8_t         port = addrp->aa_port;
9393         int             instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9394         int             rval;
9395 
9396         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9397 
9398         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9399             "ahci_fatal_error_recovery_handler enter: port %d", port);
9400 
9401         /* Port multiplier error */
9402         if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9403                 /* FBS code is neither completed nor tested. */
9404                 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9405                     port, intr_status);
9406 
9407                 /* Force a port reset */
9408                 flag = AHCI_PORT_RESET;
9409         }
9410 
9411         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9412             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9413 
9414                 /* Read PxCI to see which commands are still outstanding */
9415                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9416                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9417 
9418                 /*
9419                  * Read PxCMD.CCS to determine the slot that the HBA
9420                  * was processing when the error occurred.
9421                  */
9422                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9423                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9424                 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9425                     AHCI_CMD_STATUS_CCS_SHIFT;
9426 
9427                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9428                         spkt = ahci_portp->ahciport_err_retri_pkt;
9429                         ASSERT(spkt != NULL);
9430                 } else {
9431                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9432                         if (spkt == NULL) {
9433                                 /* May happen when interface errors occur? */
9434                                 goto next;
9435                         }
9436                 }
9437 
9438 #if AHCI_DEBUG
9439                 /*
9440                  * Debugging purpose...
9441                  */
9442                 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9443                         cmd_header =
9444                             &ahci_portp->ahciport_cmd_list[failed_slot];
9445                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9446                             "ahci_fatal_error_recovery_handler: port %d, "
9447                             "PRD Byte Count = 0x%x, "
9448                             "ahciport_prd_bytecounts = 0x%x", port,
9449                             cmd_header->ahcich_prd_byte_count,
9450                             ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9451                 }
9452 #endif
9453 
9454                 sata_cmd = &spkt->satapkt_cmd;
9455 
9456                 /* Fill out the status and error registers for PxIS.TFES */
9457                 if (intr_status & AHCI_INTR_STATUS_TFES) {
9458                         ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9459                             ahcirf_d2h_register_fis);
9460 
9461                         /* Copy the error context back to the sata_cmd */
9462                         ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9463                 }
9464 
9465                 /* The failed command must be one of the outstanding commands */
9466                 failed_tags = 0x1 << failed_slot;
9467                 ASSERT(failed_tags & slot_status);
9468 
9469                 /* Update the sata registers, especially PxSERR register */
9470                 ahci_update_sata_registers(ahci_ctlp, port,
9471                     &spkt->satapkt_device);
9472 
9473         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9474                 /* Read PxSACT to see which commands are still outstanding */
9475                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9476                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9477         }
9478 next:
9479 
9480 #if AHCI_DEBUG
9481         /*
9482          * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9483          * set, it means a fatal error happened after REQUEST SENSE command
9484          * or READ LOG EXT command is delivered to the HBA during the error
9485          * recovery process. At this time, the only outstanding command is
9486          * supposed to be REQUEST SENSE command or READ LOG EXT command.
9487          */
9488         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9489                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9490                     "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9491                     "command or READ LOG EXT command for error data retrieval "
9492                     "failed", port);
9493                 ASSERT(slot_status == 0x1);
9494                 ASSERT(failed_slot == 0);
9495                 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9496                     SCMD_REQUEST_SENSE ||
9497                     spkt->satapkt_cmd.satacmd_cmd_reg ==
9498                     SATAC_READ_LOG_EXT);
9499         }
9500 #endif
9501 
9502         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9503         ahci_portp->ahciport_mop_in_progress++;
9504 
9505         rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9506             port, flag, &reset_flag);
9507 
9508         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9509                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9510                 if (rval == AHCI_SUCCESS)
9511                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9512                             "succeed", instance, port);
9513                 else
9514                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9515                             "failed", instance, port);
9516         }
9517 
9518         /*
9519          * Won't retrieve error information:
9520          * 1. Port reset was involved to recover
9521          * 2. Device is gone
9522          * 3. IDENTIFY DEVICE command sent to ATAPI device
9523          * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9524          */
9525         if (reset_flag ||
9526             ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9527             spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9528             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9529                 goto out;
9530 
9531         /*
9532          * Deliver READ LOG EXT to gather information about the error when
9533          * a COMRESET has not been performed as part of the error recovery
9534          * during NCQ command processing.
9535          */
9536         if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9537                 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9538                     ahci_portp, port);
9539                 goto out;
9540         }
9541 
9542         /*
9543          * Deliver REQUEST SENSE for ATAPI command to gather information about
9544          * the error when a COMRESET has not been performed as part of the
9545          * error recovery.
9546          */
9547         if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9548                 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9549 out:
9550         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9551             "ahci_fatal_error_recovery_handler: port %d fatal error "
9552             "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9553             "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9554             port, slot_status, ahci_portp->ahciport_pending_tags,
9555             ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9556 
9557         ahci_mop_commands(ahci_ctlp,
9558             ahci_portp,
9559             slot_status,
9560             failed_tags, /* failed tags */
9561             0, /* timeout tags */
9562             0, /* aborted tags */
9563             0); /* reset tags */
9564 }
9565 
9566 /*
9567  * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9568  *      1. device specific.PxFBS.SDE=1
9569  *      2. Non Device specific.
9570  * Nothing will be done when Command-based switching is employed.
9571  *
9572  * Currently code is neither completed nor tested.
9573  */
9574 static void
9575 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9576     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9577 {
9578 #ifndef __lock_lint
9579         _NOTE(ARGUNUSED(intr_status))
9580 #endif
9581         uint32_t        port_fbs_ctrl;
9582         int loop_count = 0;
9583         ahci_addr_t     addr;
9584 
9585         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9586 
9587         /* Nothing will be done under Command-based switching. */
9588         if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9589                 return;
9590 
9591         port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9592             (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9593 
9594         if (!(port_fbs_ctrl & AHCI_FBS_EN))
9595                 /* FBS is not enabled. */
9596                 return;
9597 
9598         /* Problem's getting complicated now. */
9599         /*
9600          * If FIS-based switching is used, we need to check
9601          * the PxFBS to see the error type.
9602          */
9603         port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9604             (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9605 
9606         /* Refer to spec(v1.2) 9.3.6.1 */
9607         if (port_fbs_ctrl & AHCI_FBS_SDE) {
9608                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9609                     "A Device Sepcific Error: port %d", port);
9610                 /*
9611                  * Controller has paused commands for all other
9612                  * sub-devices until PxFBS.DEC is set.
9613                  */
9614                 ahci_reject_all_abort_pkts(ahci_ctlp,
9615                     ahci_portp, 0);
9616 
9617                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9618                     (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9619                     port_fbs_ctrl | AHCI_FBS_DEC);
9620 
9621                 /*
9622                  * Wait controller clear PxFBS.DEC,
9623                  * then we can continue.
9624                  */
9625                 loop_count = 0;
9626                 do {
9627                         port_fbs_ctrl = ddi_get32(ahci_ctlp->
9628                             ahcictl_ahci_acc_handle, (uint32_t *)
9629                             AHCI_PORT_PxFBS(ahci_ctlp, port));
9630 
9631                         if (loop_count++ > 1000)
9632                                 /*
9633                                  * Esclate the error. Follow
9634                                  * non-device specific error
9635                                  * procedure.
9636                                  */
9637                                 return;
9638 
9639                         drv_usecwait(AHCI_100US_USECS);
9640                 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9641 
9642                 /*
9643                  * Issue a software reset to ensure drive is in
9644                  * a known state.
9645                  */
9646                 (void) ahci_software_reset(ahci_ctlp,
9647                     ahci_portp, &addr);
9648 
9649         } else {
9650 
9651                 /* Process Non-Device Specific Error. */
9652                 /* This will be handled later on. */
9653                 cmn_err(CE_NOTE, "!FBS is not supported now.");
9654         }
9655 }
9656 /*
9657  * Handle events - fatal error recovery
9658  */
9659 static void
9660 ahci_events_handler(void *args)
9661 {
9662         ahci_event_arg_t *ahci_event_arg;
9663         ahci_ctl_t *ahci_ctlp;
9664         ahci_port_t *ahci_portp;
9665         ahci_addr_t *addrp;
9666         uint32_t event;
9667         int instance;
9668 
9669         ahci_event_arg = (ahci_event_arg_t *)args;
9670 
9671         ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9672         ahci_portp = ahci_event_arg->ahciea_portp;
9673         addrp = ahci_event_arg->ahciea_addrp;
9674         event = ahci_event_arg->ahciea_event;
9675         instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9676 
9677         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9678             "ahci_events_handler enter: port %d intr_status = 0x%x",
9679             ahci_portp->ahciport_port_num, event);
9680 
9681         mutex_enter(&ahci_portp->ahciport_mutex);
9682 
9683         /*
9684          * ahci_intr_phyrdy_change() may have rendered it to
9685          * SATA_DTYPE_NONE.
9686          */
9687         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9688                 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9689                     "ahci_events_handler: port %d no device attached, "
9690                     "and just return without doing anything",
9691                     ahci_portp->ahciport_port_num);
9692 
9693                 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9694                         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9695                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9696                             "succeed", instance, ahci_portp->ahciport_port_num);
9697                 }
9698 
9699                 goto out;
9700         }
9701 
9702         if (event & (AHCI_INTR_STATUS_IFS |
9703             AHCI_INTR_STATUS_HBDS |
9704             AHCI_INTR_STATUS_HBFS |
9705             AHCI_INTR_STATUS_TFES))
9706                 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9707                     addrp, event);
9708 
9709 out:
9710         mutex_exit(&ahci_portp->ahciport_mutex);
9711 }
9712 
9713 /*
9714  * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9715  * detect there are some commands which are timed out.
9716  */
9717 static void
9718 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9719     uint8_t port, uint32_t tmp_timeout_tags)
9720 {
9721         uint32_t slot_status = 0;
9722         uint32_t finished_tags = 0;
9723         uint32_t timeout_tags = 0;
9724 
9725         AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9726             "ahci_timeout_pkts enter: port %d", port);
9727 
9728         mutex_enter(&ahci_portp->ahciport_mutex);
9729 
9730         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9731             RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9732             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9733                 /* Read PxCI to see which commands are still outstanding */
9734                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9735                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9736         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9737                 /* Read PxSACT to see which commands are still outstanding */
9738                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9739                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9740         }
9741 
9742 #if AHCI_DEBUG
9743         /*
9744          * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9745          * set, it means a fatal error happened after REQUEST SENSE command
9746          * or READ LOG EXT command is delivered to the HBA during the error
9747          * recovery process. At this time, the only outstanding command is
9748          * supposed to be REQUEST SENSE command or READ LOG EXT command.
9749          */
9750         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9751                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9752                     "ahci_timeout_pkts called while REQUEST SENSE "
9753                     "command or READ LOG EXT command for error recovery "
9754                     "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9755                     "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9756                     tmp_timeout_tags, slot_status,
9757                     ahci_portp->ahciport_pending_tags,
9758                     ahci_portp->ahciport_pending_ncq_tags);
9759                 ASSERT(slot_status == 0x1);
9760         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9761                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9762                     "ahci_timeout_pkts called while executing R/W PMULT "
9763                     "command timeout_tags = 0x%x, slot_status = 0x%x",
9764                     tmp_timeout_tags, slot_status);
9765                 ASSERT(slot_status == 0x1);
9766         }
9767 #endif
9768 
9769         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9770         ahci_portp->ahciport_mop_in_progress++;
9771 
9772         (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9773             port, AHCI_PORT_RESET, NULL);
9774 
9775         /*
9776          * Re-identify timeout tags because some previously checked commands
9777          * could already complete.
9778          */
9779         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9780                 finished_tags = ahci_portp->ahciport_pending_tags &
9781                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9782                 timeout_tags = tmp_timeout_tags & ~finished_tags;
9783 
9784                 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9785                     "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9786                     "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9787                     "pending_tags = 0x%x ",
9788                     port, finished_tags, timeout_tags,
9789                     slot_status, ahci_portp->ahciport_pending_tags);
9790         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9791                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9792                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9793                 timeout_tags = tmp_timeout_tags & ~finished_tags;
9794 
9795                 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9796                     "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9797                     "timeout_tags = 0x%x, port_sactive = 0x%x, "
9798                     "pending_ncq_tags = 0x%x ",
9799                     port, finished_tags, timeout_tags,
9800                     slot_status, ahci_portp->ahciport_pending_ncq_tags);
9801         } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9802             RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9803                 timeout_tags = tmp_timeout_tags;
9804         }
9805 
9806         ahci_mop_commands(ahci_ctlp,
9807             ahci_portp,
9808             slot_status,
9809             0,                  /* failed tags */
9810             timeout_tags,       /* timeout tags */
9811             0,                  /* aborted tags */
9812             0);                 /* reset tags */
9813 
9814         mutex_exit(&ahci_portp->ahciport_mutex);
9815 }
9816 
9817 /*
9818  * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9819  * for long time.
9820  */
9821 static void
9822 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9823 {
9824         ahci_port_t *ahci_portp;
9825         sata_pkt_t *spkt;
9826         uint32_t pending_tags;
9827         uint32_t timeout_tags;
9828         uint32_t port_cmd_status;
9829         uint32_t port_sactive;
9830         uint8_t port;
9831         int tmp_slot;
9832         int current_slot;
9833         uint32_t current_tags;
9834         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9835 
9836         mutex_enter(&ahci_ctlp->ahcictl_mutex);
9837 
9838         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9839             "ahci_watchdog_handler entered", NULL);
9840 
9841         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9842                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9843                         continue;
9844                 }
9845 
9846                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9847 
9848                 mutex_enter(&ahci_portp->ahciport_mutex);
9849                 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9850                         mutex_exit(&ahci_portp->ahciport_mutex);
9851                         continue;
9852                 }
9853 
9854                 /* Skip the check for those ports in error recovery */
9855                 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9856                     !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9857                         mutex_exit(&ahci_portp->ahciport_mutex);
9858                         continue;
9859                 }
9860 
9861                 pending_tags = 0;
9862                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9863                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9864 
9865                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9866                     RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9867                         current_slot = 0;
9868                         pending_tags = 0x1;
9869                 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9870                         current_slot =
9871                             (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9872                             AHCI_CMD_STATUS_CCS_SHIFT;
9873                         pending_tags = ahci_portp->ahciport_pending_tags;
9874                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9875                         port_sactive = ddi_get32(
9876                             ahci_ctlp->ahcictl_ahci_acc_handle,
9877                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9878                         current_tags = port_sactive &
9879                             ~port_cmd_status &
9880                             AHCI_NCQ_SLOT_MASK(ahci_portp);
9881                         pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9882                 }
9883 
9884                 timeout_tags = 0;
9885                 while (pending_tags) {
9886                         tmp_slot = ddi_ffs(pending_tags) - 1;
9887                         if (tmp_slot == -1) {
9888                                 break;
9889                         }
9890 
9891                         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9892                                 spkt = ahci_portp->ahciport_err_retri_pkt;
9893                         else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9894                                 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9895                         else
9896                                 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9897 
9898                         if ((spkt != NULL) && spkt->satapkt_time &&
9899                             !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9900                                 /*
9901                                  * If a packet has survived for more than it's
9902                                  * max life cycles, it is a candidate for time
9903                                  * out.
9904                                  */
9905                                 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9906                                     ahci_watchdog_timeout;
9907 
9908                                 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9909                                     > 0)
9910                                         goto next;
9911 
9912 #if AHCI_DEBUG
9913                                 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9914                                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9915                                             ahci_ctlp, "watchdog: the current "
9916                                             "tags is 0x%x", current_tags);
9917                                 } else {
9918                                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9919                                             ahci_ctlp, "watchdog: the current "
9920                                             "slot is %d", current_slot);
9921                                 }
9922 #endif
9923 
9924                                 /*
9925                                  * We need to check whether the HBA has
9926                                  * begun to execute the command, if not,
9927                                  * then re-set the timer of the command.
9928                                  */
9929                                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9930                                     (tmp_slot != current_slot) ||
9931                                     NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9932                                     ((0x1 << tmp_slot) & current_tags)) {
9933                                         ahci_portp->ahciport_slot_timeout \
9934                                             [tmp_slot] = spkt->satapkt_time;
9935                                 } else {
9936                                         timeout_tags |= (0x1 << tmp_slot);
9937                                         cmn_err(CE_WARN, "!ahci%d: watchdog "
9938                                             "port %d satapkt 0x%p timed out\n",
9939                                             instance, port, (void *)spkt);
9940                                 }
9941                         }
9942 next:
9943                         CLEAR_BIT(pending_tags, tmp_slot);
9944                 }
9945 
9946                 if (timeout_tags) {
9947                         mutex_exit(&ahci_portp->ahciport_mutex);
9948                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
9949                         ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9950                             port, timeout_tags);
9951                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
9952                         mutex_enter(&ahci_portp->ahciport_mutex);
9953                 }
9954 
9955                 mutex_exit(&ahci_portp->ahciport_mutex);
9956         }
9957 
9958         /* Re-install the watchdog timeout handler */
9959         if (ahci_ctlp->ahcictl_timeout_id != 0) {
9960                 ahci_ctlp->ahcictl_timeout_id =
9961                     timeout((void (*)(void *))ahci_watchdog_handler,
9962                     (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9963         }
9964 
9965         mutex_exit(&ahci_ctlp->ahcictl_mutex);
9966 }
9967 
9968 /*
9969  * Fill the error context into sata_cmd for non-queued command error.
9970  */
9971 static void
9972 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9973 {
9974         scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9975         scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9976         scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9977         scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9978         scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9979         scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9980         scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9981 
9982         if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9983                 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9984                 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9985                 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9986                 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9987         }
9988 }
9989 
9990 /*
9991  * Fill the ncq error page into sata_cmd for queued command error.
9992  */
9993 static void
9994 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9995     struct sata_ncq_error_recovery_page *ncq_err_page)
9996 {
9997         scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9998         scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9999         scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
10000         scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
10001         scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
10002         scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
10003         scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
10004         scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
10005         scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
10006         scmd->satacmd_status_reg = ncq_err_page->ncq_status;
10007         scmd->satacmd_error_reg = ncq_err_page->ncq_error;
10008 }
10009 
10010 /*
10011  * Put the respective register value to sata_cmd_t for satacmd_flags.
10012  */
10013 static void
10014 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
10015 {
10016         if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
10017                 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
10018         if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
10019                 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
10020         if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
10021                 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
10022         if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
10023                 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
10024         if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
10025                 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
10026         if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
10027                 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
10028         if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
10029                 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
10030         if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
10031                 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
10032         if (scmd->satacmd_flags.sata_copy_out_device_reg)
10033                 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
10034         if (scmd->satacmd_flags.sata_copy_out_error_reg)
10035                 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
10036 }
10037 
10038 static void
10039 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10040     uint32_t intr_status)
10041 {
10042         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
10043 
10044         if (intr_status & AHCI_INTR_STATUS_IFS)
10045                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
10046                     "error", instance, port);
10047 
10048         if (intr_status & AHCI_INTR_STATUS_HBDS)
10049                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
10050                     instance, port);
10051 
10052         if (intr_status & AHCI_INTR_STATUS_HBFS)
10053                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
10054                     instance, port);
10055 
10056         if (intr_status & AHCI_INTR_STATUS_TFES)
10057                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
10058                     instance, port);
10059 
10060         cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
10061             "recovery", instance, port);
10062 }
10063 
10064 static void
10065 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
10066     uint32_t slot_tags)
10067 {
10068         ahci_port_t *ahci_portp;
10069         int tmp_slot;
10070         sata_pkt_t *spkt;
10071         sata_cmd_t cmd;
10072 
10073         ahci_portp = ahci_ctlp->ahcictl_ports[port];
10074         ASSERT(ahci_portp != NULL);
10075 
10076         while (slot_tags) {
10077                 tmp_slot = ddi_ffs(slot_tags) - 1;
10078                 if (tmp_slot == -1) {
10079                         break;
10080                 }
10081 
10082                 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
10083                 if (spkt != NULL) {
10084                         cmd = spkt->satapkt_cmd;
10085 
10086                         cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
10087                             "features_reg = 0x%x sec_count_msb = 0x%x "
10088                             "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
10089                             "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
10090                             "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
10091                             "lba_high_lsb = 0x%x device_reg = 0x%x "
10092                             "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10093                             cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10094                             cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10095                             cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10096                             cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10097                             cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10098                             cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10099                             *((uint32_t *)&(cmd.satacmd_flags)));
10100                 }
10101 
10102                 CLEAR_BIT(slot_tags, tmp_slot);
10103         }
10104 }
10105 
10106 /*
10107  * Dump the serror message to the log.
10108  */
10109 static void
10110 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10111     uint32_t port_serror, int debug_only)
10112 {
10113         static char err_buf[512];
10114         static char err_msg_header[16];
10115         char *err_msg = err_buf;
10116 
10117         *err_buf = '\0';
10118         *err_msg_header = '\0';
10119 
10120         if (port_serror & SERROR_DATA_ERR_FIXED) {
10121                 err_msg = strcat(err_msg,
10122                     "\tRecovered Data Integrity Error (I)\n");
10123         }
10124 
10125         if (port_serror & SERROR_COMM_ERR_FIXED) {
10126                 err_msg = strcat(err_msg,
10127                     "\tRecovered Communication Error (M)\n");
10128         }
10129 
10130         if (port_serror & SERROR_DATA_ERR) {
10131                 err_msg = strcat(err_msg,
10132                     "\tTransient Data Integrity Error (T)\n");
10133         }
10134 
10135         if (port_serror & SERROR_PERSISTENT_ERR) {
10136                 err_msg = strcat(err_msg,
10137                     "\tPersistent Communication or Data Integrity Error (C)\n");
10138         }
10139 
10140         if (port_serror & SERROR_PROTOCOL_ERR) {
10141                 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10142         }
10143 
10144         if (port_serror & SERROR_INT_ERR) {
10145                 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10146         }
10147 
10148         if (port_serror & SERROR_PHY_RDY_CHG) {
10149                 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10150         }
10151 
10152         if (port_serror & SERROR_PHY_INT_ERR) {
10153                 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10154         }
10155 
10156         if (port_serror & SERROR_COMM_WAKE) {
10157                 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10158         }
10159 
10160         if (port_serror & SERROR_10B_TO_8B_ERR) {
10161                 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10162         }
10163 
10164         if (port_serror & SERROR_DISPARITY_ERR) {
10165                 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10166         }
10167 
10168         if (port_serror & SERROR_CRC_ERR) {
10169                 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10170         }
10171 
10172         if (port_serror & SERROR_HANDSHAKE_ERR) {
10173                 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10174         }
10175 
10176         if (port_serror & SERROR_LINK_SEQ_ERR) {
10177                 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10178         }
10179 
10180         if (port_serror & SERROR_TRANS_ERR) {
10181                 err_msg = strcat(err_msg,
10182                     "\tTransport state transition error (T)\n");
10183         }
10184 
10185         if (port_serror & SERROR_FIS_TYPE) {
10186                 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10187         }
10188 
10189         if (port_serror & SERROR_EXCHANGED_ERR) {
10190                 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10191         }
10192 
10193         if (*err_msg == '\0')
10194                 return;
10195 
10196         if (debug_only) {
10197                 (void) sprintf(err_msg_header, "port %d", port);
10198                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10199                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10200         } else if (ahci_ctlp) {
10201                 cmn_err(CE_WARN, "!ahci%d: %s %s",
10202                     ddi_get_instance(ahci_ctlp->ahcictl_dip),
10203                     err_msg_header, err_msg);
10204 
10205                 /* sata trace debug */
10206                 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10207                     "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10208                     err_msg_header, err_msg);
10209         } else {
10210                 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10211 
10212                 /* sata trace debug */
10213                 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10214         }
10215 }
10216 
10217 /*
10218  * Translate the sata_address_t type into the ahci_addr_t type.
10219  * sata_device.satadev_addr structure is used as source.
10220  */
10221 static void
10222 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10223     ahci_addr_t *ahci_addrp)
10224 {
10225         sata_address_t *sata_addrp = &sd->satadev_addr;
10226         ahci_addrp->aa_port =
10227             ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10228         ahci_addrp->aa_pmport = sata_addrp->pmport;
10229 
10230         switch (sata_addrp->qual) {
10231         case SATA_ADDR_DCPORT:
10232         case SATA_ADDR_CPORT:
10233                 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10234                 break;
10235         case SATA_ADDR_PMULT:
10236         case SATA_ADDR_PMULT_SPEC:
10237                 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10238                 break;
10239         case SATA_ADDR_DPMPORT:
10240         case SATA_ADDR_PMPORT:
10241                 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10242                 break;
10243         case SATA_ADDR_NULL:
10244         default:
10245                 /* something went wrong */
10246                 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10247                 break;
10248         }
10249 }
10250 
10251 /*
10252  * This routine is to calculate the total number of ports implemented
10253  * by the HBA.
10254  */
10255 static int
10256 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10257 {
10258         uint8_t i;
10259         int num = 0;
10260 
10261         for (i = 0; i < AHCI_MAX_PORTS; i++) {
10262                 if (((uint32_t)0x1 << i) & ports_implemented)
10263                         num++;
10264         }
10265 
10266         return (num);
10267 }
10268 
10269 #if AHCI_DEBUG
10270 static void
10271 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10272 {
10273         static char name[16];
10274         va_list ap;
10275 
10276         mutex_enter(&ahci_log_mutex);
10277 
10278         va_start(ap, fmt);
10279         if (ahci_ctlp) {
10280                 (void) sprintf(name, "ahci%d: ",
10281                     ddi_get_instance(ahci_ctlp->ahcictl_dip));
10282         } else {
10283                 (void) sprintf(name, "ahci: ");
10284         }
10285 
10286         (void) vsprintf(ahci_log_buf, fmt, ap);
10287         va_end(ap);
10288 
10289         cmn_err(level, "%s%s", name, ahci_log_buf);
10290 
10291         mutex_exit(&ahci_log_mutex);
10292 }
10293 #endif
10294 
10295 /*
10296  * quiesce(9E) entry point.
10297  *
10298  * This function is called when the system is single-threaded at high
10299  * PIL with preemption disabled. Therefore, this function must not be
10300  * blocked. Because no taskqs are running, there is no need for us to
10301  * take any action for enclosure services which are running in the
10302  * taskq context, especially as no interrupts are generated by it nor
10303  * are any messages expected to come in.
10304  *
10305  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10306  * DDI_FAILURE indicates an error condition and should almost never happen.
10307  */
10308 static int
10309 ahci_quiesce(dev_info_t *dip)
10310 {
10311         ahci_ctl_t *ahci_ctlp;
10312         ahci_port_t *ahci_portp;
10313         int instance, port;
10314 
10315         instance = ddi_get_instance(dip);
10316         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10317 
10318         if (ahci_ctlp == NULL)
10319                 return (DDI_FAILURE);
10320 
10321 #if AHCI_DEBUG
10322         ahci_debug_flags = 0;
10323 #endif
10324 
10325         ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10326 
10327         /* disable all the interrupts. */
10328         ahci_disable_all_intrs(ahci_ctlp);
10329 
10330         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10331                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10332                         continue;
10333                 }
10334 
10335                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10336 
10337                 /*
10338                  * Stop the port by clearing PxCMD.ST
10339                  *
10340                  * Here we must disable the port interrupt because
10341                  * ahci_disable_all_intrs only clear GHC.IE, and IS
10342                  * register will be still set if PxIE is enabled.
10343                  * When ahci shares one IRQ with other drivers, the
10344                  * intr handler may claim the intr mistakenly.
10345                  */
10346                 ahci_disable_port_intrs(ahci_ctlp, port);
10347                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10348                     ahci_portp, port);
10349         }
10350 
10351         ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10352 
10353         return (DDI_SUCCESS);
10354 }
10355 
10356 /*
10357  * The function will add a sata packet to the done queue.
10358  */
10359 static void
10360 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10361 {
10362         ASSERT(satapkt != NULL);
10363         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10364 
10365         /* set the reason for all packets */
10366         satapkt->satapkt_reason = reason;
10367         satapkt->satapkt_hba_driver_private = NULL;
10368 
10369         if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10370             satapkt->satapkt_comp) {
10371                 /*
10372                  * only add to queue when mode is not synch and there is
10373                  * completion callback
10374                  */
10375                 *ahci_portp->ahciport_doneqtail = satapkt;
10376                 ahci_portp->ahciport_doneqtail =
10377                     (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10378                 ahci_portp->ahciport_doneq_len++;
10379 
10380         } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10381             ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10382                 /*
10383                  * for sync/non-poll mode, just call cv_broadcast
10384                  */
10385                 cv_broadcast(&ahci_portp->ahciport_cv);
10386 }
10387 
10388 /*
10389  * The function will call completion callback of sata packet on the
10390  * completed queue
10391  */
10392 static void
10393 ahci_flush_doneq(ahci_port_t *ahci_portp)
10394 {
10395         sata_pkt_t *satapkt, *next;
10396 
10397         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10398 
10399         if (ahci_portp->ahciport_doneq) {
10400                 satapkt = ahci_portp->ahciport_doneq;
10401 
10402                 ahci_portp->ahciport_doneq = NULL;
10403                 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10404                 ahci_portp->ahciport_doneq_len = 0;
10405 
10406                 mutex_exit(&ahci_portp->ahciport_mutex);
10407 
10408                 while (satapkt != NULL) {
10409                         next = satapkt->satapkt_hba_driver_private;
10410                         satapkt->satapkt_hba_driver_private = NULL;
10411 
10412                         /* Call the callback */
10413                         (*satapkt->satapkt_comp)(satapkt);
10414 
10415                         satapkt = next;
10416                 }
10417 
10418                 mutex_enter(&ahci_portp->ahciport_mutex);
10419         }
10420 }
10421 
10422 /*
10423  * Sets the state for the specified port on the controller to desired state.
10424  * This must be run in the context of the enclosure taskq which ensures that
10425  * only one event is outstanding at any time.
10426  */
10427 static boolean_t
10428 ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire)
10429 {
10430         ahci_em_led_msg_t msg;
10431         ahci_em_msg_hdr_t hdr;
10432         uint32_t msgval, hdrval;
10433         uint_t i, max_delay = ahci_em_tx_delay_count;
10434 
10435         msg.alm_hba = port;
10436         msg.alm_pminfo = 0;
10437         msg.alm_value = 0;
10438 
10439         if (desire & AHCI_EM_LED_IDENT_ENABLE) {
10440                 msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF;
10441         }
10442 
10443         if (desire & AHCI_EM_LED_FAULT_ENABLE) {
10444                 msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF;
10445         }
10446 
10447         if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 &&
10448             (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) {
10449                 msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF;
10450         }
10451 
10452         hdr.aemh_rsvd = 0;
10453         hdr.aemh_mlen = sizeof (ahci_em_led_msg_t);
10454         hdr.aemh_dlen = 0;
10455         hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED;
10456 
10457         bcopy(&msg, &msgval, sizeof (msgval));
10458         bcopy(&hdr, &hdrval, sizeof (hdrval));
10459 
10460         /*
10461          * First, make sure we can transmit. We should not have been placed in a
10462          * situation where an outstanding transmission is going on.
10463          */
10464         for (i = 0; i < max_delay; i++) {
10465                 uint32_t val;
10466 
10467                 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10468                     (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10469                 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10470                         break;
10471 
10472                 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10473         }
10474 
10475         if (i == max_delay)
10476                 return (B_FALSE);
10477 
10478         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10479             (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval);
10480         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10481             (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval);
10482         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10483             (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM);
10484 
10485         for (i = 0; i < max_delay; i++) {
10486                 uint32_t val;
10487 
10488                 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10489                     (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10490                 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10491                         break;
10492 
10493                 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10494         }
10495 
10496         if (i == max_delay)
10497                 return (B_FALSE);
10498 
10499         return (B_TRUE);
10500 }
10501 
10502 typedef struct ahci_em_led_task_arg {
10503         ahci_ctl_t              *aelta_ctl;
10504         uint8_t                 aelta_port;
10505         uint_t                  aelta_op;
10506         ahci_em_led_state_t     aelta_state;
10507         uint_t                  aelta_ret;
10508         kcondvar_t              aelta_cv;
10509         uint_t                  aelta_ref;
10510 } ahci_em_led_task_arg_t;
10511 
10512 static void
10513 ahci_em_led_task_free(ahci_em_led_task_arg_t *task)
10514 {
10515         ASSERT3U(task->aelta_ref, ==, 0);
10516         cv_destroy(&task->aelta_cv);
10517         kmem_free(task, sizeof (*task));
10518 }
10519 
10520 static void
10521 ahci_em_led_task(void *arg)
10522 {
10523         boolean_t ret, cleanup = B_FALSE;
10524         ahci_em_led_task_arg_t *led = arg;
10525         ahci_em_led_state_t state;
10526 
10527         mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10528         if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) {
10529                 led->aelta_ret = EIO;
10530                 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10531                 return;
10532         }
10533 
10534         state = led->aelta_ctl->ahcictl_em_state[led->aelta_port];
10535         mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10536 
10537         switch (led->aelta_op) {
10538         case AHCI_EM_IOC_SET_OP_ADD:
10539                 state |= led->aelta_state;
10540                 break;
10541         case AHCI_EM_IOC_SET_OP_REM:
10542                 state &= ~led->aelta_state;
10543                 break;
10544         case AHCI_EM_IOC_SET_OP_SET:
10545                 state = led->aelta_state;
10546                 break;
10547         default:
10548                 led->aelta_ret = ENOTSUP;
10549                 return;
10550         }
10551 
10552         ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state);
10553 
10554         mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10555         if (ret) {
10556                 led->aelta_ctl->ahcictl_em_state[led->aelta_port] =
10557                     led->aelta_state;
10558                 led->aelta_ret = 0;
10559         } else {
10560                 led->aelta_ret = EIO;
10561                 led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10562         }
10563         led->aelta_ref--;
10564         if (led->aelta_ref > 0) {
10565                 cv_signal(&led->aelta_cv);
10566         } else {
10567                 cleanup = B_TRUE;
10568         }
10569         mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10570 
10571         if (cleanup) {
10572                 ahci_em_led_task_free(led);
10573         }
10574 }
10575 
10576 static void
10577 ahci_em_reset(void *arg)
10578 {
10579         uint_t i, max_delay = ahci_em_reset_delay_count;
10580         ahci_ctl_t *ahci_ctlp = arg;
10581 
10582         /*
10583          * We've been asked to reset the device. The caller should have set the
10584          * resetting flag. Make sure that we don't have a request to quiesce.
10585          */
10586         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10587         ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING);
10588         if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) {
10589                 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10590                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10591                 return;
10592         }
10593         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10594 
10595         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10596             (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST);
10597         for (i = 0; i < max_delay; i++) {
10598                 uint32_t val;
10599 
10600                 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10601                     (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10602                 if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0)
10603                         break;
10604 
10605                 delay(drv_usectohz(ahci_em_reset_delay_ms * 1000));
10606         }
10607 
10608         if (i == max_delay) {
10609                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10610                 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10611                 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10612                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10613                 cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting",
10614                     ddi_get_instance(ahci_ctlp->ahcictl_dip));
10615                 return;
10616         }
10617 
10618         for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10619 
10620                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i))
10621                         continue;
10622 
10623                 /*
10624                  * Try to flush all the LEDs as part of reset. If it fails,
10625                  * drive on.
10626                  */
10627                 if (!ahci_em_set_led(ahci_ctlp, i,
10628                     ahci_ctlp->ahcictl_em_state[i])) {
10629                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10630                         ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10631                         ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10632                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10633                         cmn_err(CE_WARN, "!ahci%d: enclosure timed out "
10634                             "setting port %u",
10635                             ddi_get_instance(ahci_ctlp->ahcictl_dip), i);
10636                         return;
10637                 }
10638         }
10639 
10640         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10641         ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10642         ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY;
10643         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10644 }
10645 
10646 static boolean_t
10647 ahci_em_init(ahci_ctl_t *ahci_ctlp)
10648 {
10649         char name[128];
10650 
10651         /*
10652          * First make sure we actually have enclosure services and if so, that
10653          * we have the hardware support that we care about for this.
10654          */
10655         if (ahci_ctlp->ahcictl_em_loc == 0 ||
10656             (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0)
10657                 return (B_TRUE);
10658 
10659         /*
10660          * Next, make sure that the buffer is large enough for us. We need two
10661          * dwords or 8 bytes. The location register is stored in dwords.
10662          */
10663         if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) <
10664             AHCI_EM_BUFFER_MIN) {
10665                 return (B_TRUE);
10666         }
10667 
10668         ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT;
10669 
10670         ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc &
10671             AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4;
10672         ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr;
10673 
10674         bzero(ahci_ctlp->ahcictl_em_state,
10675             sizeof (ahci_ctlp->ahcictl_em_state));
10676 
10677         (void) snprintf(name, sizeof (name), "ahcti_em_taskq%d",
10678             ddi_get_instance(ahci_ctlp->ahcictl_dip));
10679         if ((ahci_ctlp->ahcictl_em_taskq =
10680             ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1,
10681             TASKQ_DEFAULTPRI, 0)) == NULL) {
10682                 cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em "
10683                     "services", ddi_get_instance(ahci_ctlp->ahcictl_dip));
10684                 return (B_FALSE);
10685         }
10686 
10687         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10688         ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
10689         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10690         (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10691             ahci_ctlp, DDI_SLEEP);
10692 
10693         return (B_TRUE);
10694 }
10695 
10696 static int
10697 ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10698 {
10699         int i;
10700         ahci_ioc_em_get_t get;
10701 
10702         bzero(&get, sizeof (get));
10703         get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented;
10704         if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) {
10705                 get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY;
10706         }
10707 
10708         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10709         for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10710                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) {
10711                         continue;
10712                 }
10713                 get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i];
10714         }
10715         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10716 
10717         if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0)
10718                 return (EFAULT);
10719 
10720         return (0);
10721 }
10722 
10723 static int
10724 ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10725 {
10726         int ret;
10727         ahci_ioc_em_set_t set;
10728         ahci_em_led_task_arg_t *task;
10729         boolean_t signal, cleanup;
10730 
10731         if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0)
10732                 return (EFAULT);
10733 
10734         if (set.aiems_port > ahci_ctlp->ahcictl_num_ports)
10735                 return (EINVAL);
10736 
10737         if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) {
10738                 return (EINVAL);
10739         }
10740 
10741         if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE |
10742             AHCI_EM_LED_FAULT_ENABLE |
10743             AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) {
10744                 return (EINVAL);
10745         }
10746 
10747         switch (set.aiems_op) {
10748         case AHCI_EM_IOC_SET_OP_ADD:
10749         case AHCI_EM_IOC_SET_OP_REM:
10750         case AHCI_EM_IOC_SET_OP_SET:
10751                 break;
10752         default:
10753                 return (EINVAL);
10754         }
10755 
10756         if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 &&
10757             ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) {
10758                 return (ENOTSUP);
10759         }
10760 
10761         task = kmem_alloc(sizeof (*task), KM_NOSLEEP | KM_NORMALPRI);
10762         if (task == NULL) {
10763                 return (ENOMEM);
10764         }
10765 
10766         task->aelta_ctl = ahci_ctlp;
10767         task->aelta_port = (uint8_t)set.aiems_port;
10768         task->aelta_op = set.aiems_op;
10769         task->aelta_state = set.aiems_leds;
10770 
10771         cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL);
10772 
10773         /*
10774          * Initialize the reference count to two. One for us and one for the
10775          * taskq. This will be used in case we get canceled.
10776          */
10777         task->aelta_ref = 2;
10778 
10779         /*
10780          * Once dispatched, the task state is protected by our global mutex.
10781          */
10782         (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq,
10783             ahci_em_led_task, task, DDI_SLEEP);
10784 
10785         signal = B_FALSE;
10786         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10787         while (task->aelta_ref > 1) {
10788                 if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) ==
10789                     0) {
10790                         signal = B_TRUE;
10791                         break;
10792                 }
10793         }
10794 
10795         /*
10796          * Remove our reference count. If we were woken up because of a signal
10797          * then the taskq may still be dispatched. In which case we shouldn't
10798          * free this memory until it is done. In that case, the taskq will take
10799          * care of it.
10800          */
10801         task->aelta_ref--;
10802         cleanup = (task->aelta_ref == 0);
10803         if (signal) {
10804                 ret = EINTR;
10805         } else {
10806                 ret = task->aelta_ret;
10807         }
10808         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10809 
10810         if (cleanup) {
10811                 ahci_em_led_task_free(task);
10812         }
10813 
10814         return (ret);
10815 }
10816 
10817 static int
10818 ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg)
10819 {
10820         int inst;
10821         ahci_ctl_t *ahci_ctlp;
10822 
10823         inst = ddi_get_instance(dip);
10824         if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) {
10825                 return (ENXIO);
10826         }
10827 
10828         switch (cmd) {
10829         case AHCI_EM_IOC_GET:
10830                 return (ahci_em_ioctl_get(ahci_ctlp, arg));
10831         case AHCI_EM_IOC_SET:
10832                 return (ahci_em_ioctl_set(ahci_ctlp, arg));
10833         default:
10834                 return (ENOTTY);
10835         }
10836 
10837 }
10838 
10839 static void
10840 ahci_em_quiesce(ahci_ctl_t *ahci_ctlp)
10841 {
10842         ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT);
10843 
10844         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10845         ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE;
10846         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10847 
10848         ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq);
10849 }
10850 
10851 static void
10852 ahci_em_suspend(ahci_ctl_t *ahci_ctlp)
10853 {
10854         ahci_em_quiesce(ahci_ctlp);
10855 
10856         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10857         ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY;
10858         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10859 }
10860 
10861 static void
10862 ahci_em_resume(ahci_ctl_t *ahci_ctlp)
10863 {
10864         mutex_enter(&ahci_ctlp->ahcictl_mutex);
10865         ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
10866         mutex_exit(&ahci_ctlp->ahcictl_mutex);
10867 
10868         (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10869             ahci_ctlp, DDI_SLEEP);
10870 }
10871 
10872 static void
10873 ahci_em_fini(ahci_ctl_t *ahci_ctlp)
10874 {
10875         if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10876                 return;
10877         }
10878 
10879         ahci_em_quiesce(ahci_ctlp);
10880         ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq);
10881         ahci_ctlp->ahcictl_em_taskq = NULL;
10882 }