1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 */
26
27 /*
28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
29 *
30 * Power Management Support
31 * ------------------------
32 *
33 * At the moment, the ahci driver only implements suspend/resume to
34 * support Suspend to RAM on X86 feature. Device power management isn't
35 * implemented, link power management is disabled, and hot plug isn't
36 * allowed during the period from suspend to resume.
37 *
38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
39 * and DDI_RESUME entries, and don't need to take care of new requests
40 * sent down after suspend because the target driver (sd) has already
41 * handled these conditions, and blocked these requests. For the detailed
42 * information, please check with sdopen, sdclose and sdioctl routines.
43 *
44 */
45
46 #include <sys/note.h>
47 #include <sys/scsi/scsi.h>
48 #include <sys/pci.h>
49 #include <sys/disp.h>
50 #include <sys/sata/sata_hba.h>
51 #include <sys/sata/adapters/ahci/ahcireg.h>
52 #include <sys/sata/adapters/ahci/ahcivar.h>
53
54 /*
55 * FMA header files
56 */
57 #include <sys/ddifm.h>
58 #include <sys/fm/protocol.h>
59 #include <sys/fm/util.h>
60 #include <sys/fm/io/ddi.h>
61
62 /*
63 * This is the string displayed by modinfo, etc.
64 */
65 static char ahci_ident[] = "ahci driver";
66
67 /*
68 * Function prototypes for driver entry points
69 */
70 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
71 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
72 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
73 static int ahci_quiesce(dev_info_t *);
74
75 /*
76 * Function prototypes for SATA Framework interfaces
77 */
78 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
79 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
80
81 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
82 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
83 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
84 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
85 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
86 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
87 #if defined(__lock_lint)
88 static int ahci_selftest(dev_info_t *, sata_device_t *);
89 #endif
90
91 /*
92 * FMA Prototypes
93 */
94 static void ahci_fm_init(ahci_ctl_t *);
95 static void ahci_fm_fini(ahci_ctl_t *);
96 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
97 int ahci_check_acc_handle(ddi_acc_handle_t);
98 int ahci_check_dma_handle(ddi_dma_handle_t);
99 void ahci_fm_ereport(ahci_ctl_t *, char *);
100 static int ahci_check_all_handle(ahci_ctl_t *);
101 static int ahci_check_ctl_handle(ahci_ctl_t *);
102 static int ahci_check_port_handle(ahci_ctl_t *, int);
103 static int ahci_check_slot_handle(ahci_port_t *, int);
104
105 /*
106 * Local function prototypes
107 */
108 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
109 static int ahci_alloc_ports_state(ahci_ctl_t *);
110 static void ahci_dealloc_ports_state(ahci_ctl_t *);
111 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
112 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
113 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
114 static void ahci_dealloc_rcvd_fis(ahci_port_t *);
115 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
116 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
117 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
118 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
119 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
120 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
121
122 static int ahci_initialize_controller(ahci_ctl_t *);
123 static void ahci_uninitialize_controller(ahci_ctl_t *);
124 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
125 static int ahci_config_space_init(ahci_ctl_t *);
126 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
127
128 static void ahci_drain_ports_taskq(ahci_ctl_t *);
129 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
130 uint8_t);
131 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
132 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
133 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
134 sata_device_t *);
135 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
136 sata_pmult_gscr_t *);
137 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
138 sata_device_t *);
139 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
140 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
141 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
142 sata_device_t *);
143
144 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
145 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
146 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
147 ahci_addr_t *);
148 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
149 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
150 ahci_addr_t *, sata_pkt_t *);
151 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
152 ahci_addr_t *, sata_pkt_t *);
153 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
154 ahci_addr_t *, int);
155 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
156 static void ahci_copy_ncq_err_page(sata_cmd_t *,
157 struct sata_ncq_error_recovery_page *);
158 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
159 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
160 static void ahci_flush_doneq(ahci_port_t *);
161
162 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
163 static int ahci_hba_reset(ahci_ctl_t *);
164 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
165 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
166 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
167 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
168 ahci_addr_t *);
169 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
170 ahci_addr_t *);
171 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
172 ahci_addr_t *);
173 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
174 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
175 uint8_t);
176 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
177 uint8_t, int, int *);
178 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
179 uint32_t, uint32_t, uint32_t, uint32_t);
180 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
181 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
182 uint8_t, sata_pkt_t *);
183 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
184 ahci_addr_t *, uint32_t);
185 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
186 uint8_t, uint32_t);
187 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
188 uint8_t, uint32_t);
189 static void ahci_events_handler(void *);
190 static void ahci_watchdog_handler(ahci_ctl_t *);
191
192 static uint_t ahci_intr(caddr_t, caddr_t);
193 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
194 static int ahci_add_intrs(ahci_ctl_t *, int);
195 static void ahci_rem_intrs(ahci_ctl_t *);
196 static void ahci_enable_all_intrs(ahci_ctl_t *);
197 static void ahci_disable_all_intrs(ahci_ctl_t *);
198 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
199 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
200
201 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
202 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
203 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
204 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
205 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
206 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
207 ahci_port_t *, uint8_t);
208 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
209 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
210 uint8_t, uint32_t);
211 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
212 uint8_t, uint32_t);
213 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
214
215 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
216 static int ahci_get_num_implemented_ports(uint32_t);
217 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
218 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
219 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
220 #if AHCI_DEBUG
221 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
222 #endif
223
224
225 /*
226 * DMA attributes for the data buffer
227 *
228 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
229 * does not support 64-bit addressing
230 */
231 static ddi_dma_attr_t buffer_dma_attr = {
232 DMA_ATTR_V0, /* dma_attr_version */
233 0x0ull, /* dma_attr_addr_lo: lowest bus address */
234 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
235 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */
236 0x2ull, /* dma_attr_align: word aligned */
237 1, /* dma_attr_burstsizes */
238 1, /* dma_attr_minxfer */
239 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
240 0xffffffffull, /* dma_attr_seg */
241 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */
242 512, /* dma_attr_granular */
243 0, /* dma_attr_flags */
244 };
245
246 /*
247 * DMA attributes for the rcvd FIS
248 *
249 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
250 * does not support 64-bit addressing
251 */
252 static ddi_dma_attr_t rcvd_fis_dma_attr = {
253 DMA_ATTR_V0, /* dma_attr_version */
254 0x0ull, /* dma_attr_addr_lo: lowest bus address */
255 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
256 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
257 0x100ull, /* dma_attr_align: 256-byte aligned */
258 1, /* dma_attr_burstsizes */
259 1, /* dma_attr_minxfer */
260 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
261 0xffffffffull, /* dma_attr_seg */
262 1, /* dma_attr_sgllen */
263 1, /* dma_attr_granular */
264 0, /* dma_attr_flags */
265 };
266
267 /*
268 * DMA attributes for the command list
269 *
270 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
271 * does not support 64-bit addressing
272 */
273 static ddi_dma_attr_t cmd_list_dma_attr = {
274 DMA_ATTR_V0, /* dma_attr_version */
275 0x0ull, /* dma_attr_addr_lo: lowest bus address */
276 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
277 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
278 0x400ull, /* dma_attr_align: 1K-byte aligned */
279 1, /* dma_attr_burstsizes */
280 1, /* dma_attr_minxfer */
281 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
282 0xffffffffull, /* dma_attr_seg */
283 1, /* dma_attr_sgllen */
284 1, /* dma_attr_granular */
285 0, /* dma_attr_flags */
286 };
287
288 /*
289 * DMA attributes for cmd tables
290 *
291 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
292 * does not support 64-bit addressing
293 */
294 static ddi_dma_attr_t cmd_table_dma_attr = {
295 DMA_ATTR_V0, /* dma_attr_version */
296 0x0ull, /* dma_attr_addr_lo: lowest bus address */
297 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
298 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
299 0x80ull, /* dma_attr_align: 128-byte aligned */
300 1, /* dma_attr_burstsizes */
301 1, /* dma_attr_minxfer */
302 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
303 0xffffffffull, /* dma_attr_seg */
304 1, /* dma_attr_sgllen */
305 1, /* dma_attr_granular */
306 0, /* dma_attr_flags */
307 };
308
309
310 /* Device access attributes */
311 static ddi_device_acc_attr_t accattr = {
312 DDI_DEVICE_ATTR_V1,
313 DDI_STRUCTURE_LE_ACC,
314 DDI_STRICTORDER_ACC,
315 DDI_DEFAULT_ACC
316 };
317
318
319 static struct dev_ops ahcictl_dev_ops = {
320 DEVO_REV, /* devo_rev */
321 0, /* refcnt */
322 ahci_getinfo, /* info */
323 nulldev, /* identify */
324 nulldev, /* probe */
325 ahci_attach, /* attach */
326 ahci_detach, /* detach */
327 nodev, /* no reset */
328 (struct cb_ops *)0, /* driver operations */
329 NULL, /* bus operations */
330 NULL, /* power */
331 ahci_quiesce, /* quiesce */
332 };
333
334 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
335 SATA_TRAN_HOTPLUG_OPS_REV_1,
336 ahci_tran_hotplug_port_activate,
337 ahci_tran_hotplug_port_deactivate
338 };
339
340 extern struct mod_ops mod_driverops;
341
342 static struct modldrv modldrv = {
343 &mod_driverops, /* driverops */
344 ahci_ident, /* short description */
345 &ahcictl_dev_ops, /* driver ops */
346 };
347
348 static struct modlinkage modlinkage = {
349 MODREV_1,
350 { &modldrv, NULL }
351 };
352
353 /* The following variables are watchdog handler related */
354 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
355 static clock_t ahci_watchdog_tick;
356
357 /*
358 * This static variable indicates the size of command table,
359 * and it's changeable with prdt number, which ahci_dma_prdt_number
360 * indicates.
361 */
362 static size_t ahci_cmd_table_size;
363
364 /*
365 * The below global variables are tunable via /etc/system
366 *
367 * ahci_dma_prdt_number
368 * ahci_msi_enabled
369 * ahci_buf_64bit_dma
370 * ahci_commu_64bit_dma
371 */
372
373 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
374 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
375
376 /* AHCI MSI is tunable */
377 boolean_t ahci_msi_enabled = B_TRUE;
378
379 /*
380 * 64-bit dma addressing for data buffer is tunable
381 *
382 * The variable controls only the below value:
383 * DBAU (upper 32-bits physical address of data block)
384 */
385 boolean_t ahci_buf_64bit_dma = B_TRUE;
386
387 /*
388 * 64-bit dma addressing for communication system descriptors is tunable
389 *
390 * The variable controls the below three values:
391 *
392 * PxCLBU (upper 32-bits for the command list base physical address)
393 * PxFBU (upper 32-bits for the received FIS base physical address)
394 * CTBAU (upper 32-bits of command table base)
395 */
396 boolean_t ahci_commu_64bit_dma = B_TRUE;
397
398 /*
399 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
400 * chipset. If the users want to have a try with 64-bit dma, please change
401 * the below variable value to enable it.
402 */
403 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
404
405 /*
406 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
407 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
408 * please change the below value to enable it.
409 */
410 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
411
412
413 /*
414 * End of global tunable variable definition
415 */
416
417 #if AHCI_DEBUG
418 uint32_t ahci_debug_flags = 0;
419 #else
420 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
421 #endif
422
423
424 #if AHCI_DEBUG
425 /* The following is needed for ahci_log() */
426 static kmutex_t ahci_log_mutex;
427 static char ahci_log_buf[512];
428 #endif
429
430 /* Opaque state pointer initialized by ddi_soft_state_init() */
431 static void *ahci_statep = NULL;
432
433 /*
434 * ahci module initialization.
435 */
436 int
437 _init(void)
438 {
439 int ret;
440
441 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
442 if (ret != 0) {
443 goto err_out;
444 }
445
446 #if AHCI_DEBUG
447 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
448 #endif
449
450 if ((ret = sata_hba_init(&modlinkage)) != 0) {
451 #if AHCI_DEBUG
452 mutex_destroy(&ahci_log_mutex);
453 #endif
454 ddi_soft_state_fini(&ahci_statep);
455 goto err_out;
456 }
457
458 /* watchdog tick */
459 ahci_watchdog_tick = drv_usectohz(
460 (clock_t)ahci_watchdog_timeout * 1000000);
461
462 ret = mod_install(&modlinkage);
463 if (ret != 0) {
464 sata_hba_fini(&modlinkage);
465 #if AHCI_DEBUG
466 mutex_destroy(&ahci_log_mutex);
467 #endif
468 ddi_soft_state_fini(&ahci_statep);
469 goto err_out;
470 }
471
472 return (ret);
473
474 err_out:
475 cmn_err(CE_WARN, "!ahci: Module init failed");
476 return (ret);
477 }
478
479 /*
480 * ahci module uninitialize.
481 */
482 int
483 _fini(void)
484 {
485 int ret;
486
487 ret = mod_remove(&modlinkage);
488 if (ret != 0) {
489 return (ret);
490 }
491
492 /* Remove the resources allocated in _init(). */
493 sata_hba_fini(&modlinkage);
494 #if AHCI_DEBUG
495 mutex_destroy(&ahci_log_mutex);
496 #endif
497 ddi_soft_state_fini(&ahci_statep);
498
499 return (ret);
500 }
501
502 /*
503 * _info entry point
504 */
505 int
506 _info(struct modinfo *modinfop)
507 {
508 return (mod_info(&modlinkage, modinfop));
509 }
510
511 /*
512 * The attach entry point for dev_ops.
513 */
514 static int
515 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
516 {
517 ahci_ctl_t *ahci_ctlp = NULL;
518 int instance = ddi_get_instance(dip);
519 int status;
520 int attach_state;
521 uint32_t cap_status, ahci_version;
522 uint32_t ghc_control;
523 int intr_types;
524 int i;
525 pci_regspec_t *regs;
526 int regs_length;
527 int rnumber;
528 #if AHCI_DEBUG
529 int speed;
530 #endif
531
532 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
533 NULL);
534
535 switch (cmd) {
536 case DDI_ATTACH:
537 break;
538
539 case DDI_RESUME:
540
541 /*
542 * During DDI_RESUME, the hardware state of the device
543 * (power may have been removed from the device) must be
544 * restored, allow pending requests to continue, and
545 * service new requests.
546 */
547 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
548 mutex_enter(&ahci_ctlp->ahcictl_mutex);
549
550 /*
551 * GHC.AE must be set to 1 before any other AHCI register
552 * is accessed
553 */
554 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
555 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
556 ghc_control |= AHCI_HBA_GHC_AE;
557 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
558 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
559
560 /* Restart watch thread */
561 if (ahci_ctlp->ahcictl_timeout_id == 0)
562 ahci_ctlp->ahcictl_timeout_id = timeout(
563 (void (*)(void *))ahci_watchdog_handler,
564 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
565
566 mutex_exit(&ahci_ctlp->ahcictl_mutex);
567
568 /*
569 * Re-initialize the controller and enable the interrupts and
570 * restart all the ports.
571 *
572 * Note that so far we don't support hot-plug during
573 * suspend/resume.
574 */
575 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
576 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
577 "Failed to initialize the controller "
578 "during DDI_RESUME", NULL);
579 return (DDI_FAILURE);
580 }
581
582 mutex_enter(&ahci_ctlp->ahcictl_mutex);
583 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
584 mutex_exit(&ahci_ctlp->ahcictl_mutex);
585
586 return (DDI_SUCCESS);
587
588 default:
589 return (DDI_FAILURE);
590 }
591
592 attach_state = AHCI_ATTACH_STATE_NONE;
593
594 /* Allocate soft state */
595 status = ddi_soft_state_zalloc(ahci_statep, instance);
596 if (status != DDI_SUCCESS) {
597 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
598 instance);
599 goto err_out;
600 }
601
602 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
603 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
604 ahci_ctlp->ahcictl_dip = dip;
605
606 /* Initialize the cport/port mapping */
607 for (i = 0; i < AHCI_MAX_PORTS; i++) {
608 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
609 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
610 }
611
612 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
613
614 /* Initialize FMA properties */
615 ahci_fm_init(ahci_ctlp);
616
617 attach_state |= AHCI_ATTACH_STATE_FMA;
618
619 /*
620 * Now map the AHCI base address; which includes global
621 * registers and port control registers
622 *
623 * According to the spec, the AHCI Base Address is BAR5,
624 * but BAR0-BAR4 are optional, so we need to check which
625 * rnumber is used for BAR5.
626 */
627
628 /*
629 * search through DDI "reg" property for the AHCI register set
630 */
631 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
632 DDI_PROP_DONTPASS, "reg", (int **)®s,
633 (uint_t *)®s_length) != DDI_PROP_SUCCESS) {
634 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
635 instance);
636 goto err_out;
637 }
638
639 /* AHCI Base Address is located at 0x24 offset */
640 for (rnumber = 0; rnumber < regs_length; ++rnumber) {
641 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
642 == AHCI_PCI_RNUM)
643 break;
644 }
645
646 ddi_prop_free(regs);
647
648 if (rnumber == regs_length) {
649 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
650 instance);
651 goto err_out;
652 }
653
654 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
655
656 status = ddi_regs_map_setup(dip,
657 rnumber,
658 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
659 0,
660 0,
661 &accattr,
662 &ahci_ctlp->ahcictl_ahci_acc_handle);
663 if (status != DDI_SUCCESS) {
664 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
665 instance);
666 goto err_out;
667 }
668
669 attach_state |= AHCI_ATTACH_STATE_REG_MAP;
670
671 /*
672 * GHC.AE must be set to 1 before any other AHCI register
673 * is accessed
674 */
675 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
676 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
677 ghc_control |= AHCI_HBA_GHC_AE;
678 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
679 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
680
681 /* Get the AHCI version information */
682 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
683 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
684
685 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
686 (ahci_version & 0xffff0000) >> 16,
687 ((ahci_version & 0x0000ff00) >> 4 |
688 (ahci_version & 0x000000ff)));
689
690 /* We don't support controllers whose versions are lower than 1.0 */
691 if (!(ahci_version & 0xffff0000)) {
692 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
693 "than version 1.0", instance);
694 goto err_out;
695 }
696
697 /* Get the HBA capabilities information */
698 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
699 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
700
701 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
702 cap_status);
703
704 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
705 if (ahci_version >= 0x00010200) {
706 uint32_t cap2_status;
707
708 /* Get the HBA capabilities extended information */
709 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
710 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
711
712 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
713 "hba capabilities extended = 0x%x", cap2_status);
714 }
715
716 #if AHCI_DEBUG
717 /* Get the interface speed supported by the HBA */
718 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
719 if (speed == 0x01) {
720 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
721 "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
722 } else if (speed == 0x10) {
723 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
724 "hba interface speed support: Gen 2 (3 Gbps)", NULL);
725 } else if (speed == 0x11) {
726 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
727 "hba interface speed support: Gen 3 (6 Gbps)", NULL);
728 }
729 #endif
730
731 /* Get the number of command slots supported by the HBA */
732 ahci_ctlp->ahcictl_num_cmd_slots =
733 ((cap_status & AHCI_HBA_CAP_NCS) >>
734 AHCI_HBA_CAP_NCS_SHIFT) + 1;
735
736 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
737 ahci_ctlp->ahcictl_num_cmd_slots);
738
739 /* Get the bit map which indicates ports implemented by the HBA */
740 ahci_ctlp->ahcictl_ports_implemented =
741 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
742 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
743
744 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
745 ahci_ctlp->ahcictl_ports_implemented);
746
747 /* Max port number implemented */
748 ahci_ctlp->ahcictl_num_ports =
749 ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
750
751 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
752 (cap_status & AHCI_HBA_CAP_NP) + 1);
753
754 /* Get the number of implemented ports by the HBA */
755 ahci_ctlp->ahcictl_num_implemented_ports =
756 ahci_get_num_implemented_ports(
757 ahci_ctlp->ahcictl_ports_implemented);
758
759 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
760 "hba number of implemented ports: %d",
761 ahci_ctlp->ahcictl_num_implemented_ports);
762
763 /* Check whether HBA supports 64bit DMA addressing */
764 if (!(cap_status & AHCI_HBA_CAP_S64A)) {
765 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
766 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
767 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
768 "hba does not support 64-bit addressing", NULL);
769 }
770
771 /* Checking for the support of Port Multiplier */
772 if (cap_status & AHCI_HBA_CAP_SPM) {
773 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
774 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
775 "hba supports port multiplier (CBSS)", NULL);
776
777 /* Support FIS-based switching ? */
778 if (cap_status & AHCI_HBA_CAP_FBSS) {
779 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
780 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
781 "hba supports FIS-based switching (FBSS)", NULL);
782 }
783 }
784
785 /* Checking for Support Command List Override */
786 if (cap_status & AHCI_HBA_CAP_SCLO) {
787 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
788 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
789 "hba supports command list override.", NULL);
790 }
791
792 /* Checking for Asynchronous Notification */
793 if (cap_status & AHCI_HBA_CAP_SSNTF) {
794 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
795 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
796 "hba supports asynchronous notification.", NULL);
797 }
798
799 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
800 != DDI_SUCCESS) {
801 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
802 instance);
803 goto err_out;
804 }
805
806 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
807
808 /*
809 * Check the pci configuration space, and set caps. We also
810 * handle the hardware defect in this function.
811 *
812 * For example, force ATI SB600 to use 32-bit dma addressing
813 * since it doesn't support 64-bit dma though its CAP register
814 * declares it support.
815 */
816 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
817 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
818 instance);
819 goto err_out;
820 }
821
822 /*
823 * Disable the whole controller interrupts before adding
824 * interrupt handlers(s).
825 */
826 ahci_disable_all_intrs(ahci_ctlp);
827
828 /* Get supported interrupt types */
829 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
830 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
831 instance);
832 goto err_out;
833 }
834
835 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
836 "ddi_intr_get_supported_types() returned: 0x%x",
837 intr_types);
838
839 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
840 /*
841 * Try MSI first, but fall back to FIXED if failed
842 */
843 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
844 DDI_SUCCESS) {
845 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
846 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
847 "Using MSI interrupt type", NULL);
848 goto intr_done;
849 }
850
851 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
852 "MSI registration failed, "
853 "trying FIXED interrupts", NULL);
854 }
855
856 if (intr_types & DDI_INTR_TYPE_FIXED) {
857 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
858 DDI_SUCCESS) {
859 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
860 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
861 "Using FIXED interrupt type", NULL);
862 goto intr_done;
863 }
864
865 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
866 "FIXED interrupt registration failed", NULL);
867 }
868
869 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
870
871 goto err_out;
872
873 intr_done:
874
875 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
876
877 /* Initialize the controller mutex */
878 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
879 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
880
881 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
882
883 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
884 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
885 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
886 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
887 }
888
889 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
890 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
891 sizeof (ahci_prdt_item_t));
892
893 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
894 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
895 " ahci_cmd_table_size is 0x%x",
896 ahci_dma_prdt_number, ahci_cmd_table_size);
897
898 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
899 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
900 ahci_dma_prdt_number;
901
902 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
903 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
904 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
905 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
906
907 /*
908 * enable 64bit dma for data buffer for SB600 if
909 * sb600_buf_64bit_dma_disable is B_FALSE
910 */
911 if ((ahci_buf_64bit_dma == B_FALSE) ||
912 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
913 !(sb600_buf_64bit_dma_disable == B_FALSE &&
914 ahci_ctlp->ahcictl_venid == 0x1002 &&
915 ahci_ctlp->ahcictl_devid == 0x4380))) {
916 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
917 0xffffffffull;
918 }
919
920 /*
921 * enable 64bit dma for command buffer for SB600/700/710/800
922 * if sbxxx_commu_64bit_dma_disable is B_FALSE
923 */
924 if ((ahci_commu_64bit_dma == B_FALSE) ||
925 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
926 !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
927 ahci_ctlp->ahcictl_venid == 0x1002 &&
928 (ahci_ctlp->ahcictl_devid == 0x4380 ||
929 ahci_ctlp->ahcictl_devid == 0x4391)))) {
930 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
931 0xffffffffull;
932 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
933 0xffffffffull;
934 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
935 0xffffffffull;
936 }
937
938 /* Allocate the ports structure */
939 status = ahci_alloc_ports_state(ahci_ctlp);
940 if (status != AHCI_SUCCESS) {
941 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
942 instance);
943 goto err_out;
944 }
945
946 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
947
948 /*
949 * Initialize the controller and ports.
950 */
951 status = ahci_initialize_controller(ahci_ctlp);
952 if (status != AHCI_SUCCESS) {
953 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
954 instance);
955 goto err_out;
956 }
957
958 attach_state |= AHCI_ATTACH_STATE_HW_INIT;
959
960 /* Start one thread to check packet timeouts */
961 ahci_ctlp->ahcictl_timeout_id = timeout(
962 (void (*)(void *))ahci_watchdog_handler,
963 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
964
965 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
966
967 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
968 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
969 instance);
970 goto err_out;
971 }
972
973 /* Check all handles at the end of the attach operation. */
974 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
975 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
976 instance);
977 goto err_out;
978 }
979
980 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
981
982 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
983
984 return (DDI_SUCCESS);
985
986 err_out:
987 /* FMA message */
988 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
989 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
990
991 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
992 mutex_enter(&ahci_ctlp->ahcictl_mutex);
993 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
994 ahci_ctlp->ahcictl_timeout_id = 0;
995 mutex_exit(&ahci_ctlp->ahcictl_mutex);
996 }
997
998 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
999 ahci_uninitialize_controller(ahci_ctlp);
1000 }
1001
1002 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1003 ahci_dealloc_ports_state(ahci_ctlp);
1004 }
1005
1006 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1007 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1008 }
1009
1010 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1011 ahci_rem_intrs(ahci_ctlp);
1012 }
1013
1014 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1015 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1016 }
1017
1018 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1019 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1020 }
1021
1022 if (attach_state & AHCI_ATTACH_STATE_FMA) {
1023 ahci_fm_fini(ahci_ctlp);
1024 }
1025
1026 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1027 ddi_soft_state_free(ahci_statep, instance);
1028 }
1029
1030 return (DDI_FAILURE);
1031 }
1032
1033 /*
1034 * The detach entry point for dev_ops.
1035 */
1036 static int
1037 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1038 {
1039 ahci_ctl_t *ahci_ctlp;
1040 int instance;
1041 int ret;
1042
1043 instance = ddi_get_instance(dip);
1044 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1045
1046 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1047
1048 switch (cmd) {
1049 case DDI_DETACH:
1050
1051 /* disable the interrupts for an uninterrupted detach */
1052 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1053 ahci_disable_all_intrs(ahci_ctlp);
1054 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1055
1056 /* unregister from the sata framework. */
1057 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1058 if (ret != AHCI_SUCCESS) {
1059 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1060 ahci_enable_all_intrs(ahci_ctlp);
1061 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1062 return (DDI_FAILURE);
1063 }
1064
1065 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1066
1067 /* stop the watchdog handler */
1068 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1069 ahci_ctlp->ahcictl_timeout_id = 0;
1070
1071 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1072
1073 /* uninitialize the controller */
1074 ahci_uninitialize_controller(ahci_ctlp);
1075
1076 /* remove the interrupts */
1077 ahci_rem_intrs(ahci_ctlp);
1078
1079 /* deallocate the ports structures */
1080 ahci_dealloc_ports_state(ahci_ctlp);
1081
1082 /* destroy mutex */
1083 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1084
1085 /* teardown the pci config */
1086 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1087
1088 /* remove the reg maps. */
1089 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1090
1091 /* release fma resource */
1092 ahci_fm_fini(ahci_ctlp);
1093
1094 /* free the soft state. */
1095 ddi_soft_state_free(ahci_statep, instance);
1096
1097 return (DDI_SUCCESS);
1098
1099 case DDI_SUSPEND:
1100
1101 /*
1102 * The steps associated with suspension must include putting
1103 * the underlying device into a quiescent state so that it
1104 * will not generate interrupts or modify or access memory.
1105 */
1106 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1107 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1108 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1109 return (DDI_SUCCESS);
1110 }
1111
1112 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1113
1114 /* stop the watchdog handler */
1115 if (ahci_ctlp->ahcictl_timeout_id) {
1116 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1117 ahci_ctlp->ahcictl_timeout_id = 0;
1118 }
1119
1120 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1121
1122 /*
1123 * drain the taskq
1124 */
1125 ahci_drain_ports_taskq(ahci_ctlp);
1126
1127 /*
1128 * Disable the interrupts and stop all the ports.
1129 */
1130 ahci_uninitialize_controller(ahci_ctlp);
1131
1132 return (DDI_SUCCESS);
1133
1134 default:
1135 return (DDI_FAILURE);
1136 }
1137 }
1138
1139 /*
1140 * The info entry point for dev_ops.
1141 *
1142 */
1143 static int
1144 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1145 void *arg, void **result)
1146 {
1147 #ifndef __lock_lint
1148 _NOTE(ARGUNUSED(dip))
1149 #endif /* __lock_lint */
1150
1151 ahci_ctl_t *ahci_ctlp;
1152 int instance;
1153 dev_t dev;
1154
1155 dev = (dev_t)arg;
1156 instance = getminor(dev);
1157
1158 switch (infocmd) {
1159 case DDI_INFO_DEVT2DEVINFO:
1160 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1161 if (ahci_ctlp != NULL) {
1162 *result = ahci_ctlp->ahcictl_dip;
1163 return (DDI_SUCCESS);
1164 } else {
1165 *result = NULL;
1166 return (DDI_FAILURE);
1167 }
1168 case DDI_INFO_DEVT2INSTANCE:
1169 *(int *)result = instance;
1170 break;
1171 default:
1172 break;
1173 }
1174
1175 return (DDI_SUCCESS);
1176 }
1177
1178 /*
1179 * Registers the ahci with sata framework.
1180 */
1181 static int
1182 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1183 {
1184 struct sata_hba_tran *sata_hba_tran;
1185
1186 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1187 "ahci_register_sata_hba_tran enter", NULL);
1188
1189 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1190
1191 /* Allocate memory for the sata_hba_tran */
1192 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1193
1194 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1195 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1196 sata_hba_tran->sata_tran_hba_dma_attr =
1197 &ahci_ctlp->ahcictl_buffer_dma_attr;
1198
1199 /* Report the number of implemented ports */
1200 sata_hba_tran->sata_tran_hba_num_cports =
1201 ahci_ctlp->ahcictl_num_implemented_ports;
1202
1203 /* Support ATAPI device */
1204 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1205
1206 /* Get the data transfer capability for PIO command by the HBA */
1207 if (cap_status & AHCI_HBA_CAP_PMD) {
1208 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1209 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1210 "DRQ block data transfer for PIO command protocol", NULL);
1211 }
1212
1213 /*
1214 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1215 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1216 * QUEUED (EXT), and SERVICE commands). Queued operations are
1217 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1218 * QUEUED commands when the HBA and device support native command
1219 * queuing(NCQ).
1220 *
1221 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1222 * CAP register of the HBA indicates NCQ is supported.
1223 *
1224 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1225 * set because the previous register content of PxCI can be re-written
1226 * in the register write.
1227 */
1228 if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1229 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1230 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1231 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1232 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1233 "Command Queuing", NULL);
1234 }
1235
1236 /* Support port multiplier? */
1237 if (cap_status & AHCI_HBA_CAP_SPM) {
1238 sata_hba_tran->sata_tran_hba_features_support |=
1239 SATA_CTLF_PORT_MULTIPLIER;
1240
1241 /* Support FIS-based switching for port multiplier? */
1242 if (cap_status & AHCI_HBA_CAP_FBSS) {
1243 sata_hba_tran->sata_tran_hba_features_support |=
1244 SATA_CTLF_PMULT_FBS;
1245 }
1246 }
1247
1248 /* Report the number of command slots */
1249 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1250
1251 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1252 sata_hba_tran->sata_tran_start = ahci_tran_start;
1253 sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1254 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1255 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1256 #ifdef __lock_lint
1257 sata_hba_tran->sata_tran_selftest = ahci_selftest;
1258 #endif
1259 /*
1260 * When SATA framework adds support for pwrmgt the
1261 * pwrmgt_ops needs to be updated
1262 */
1263 sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1264 sata_hba_tran->sata_tran_ioctl = NULL;
1265
1266 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1267
1268 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1269
1270 /* Attach it to SATA framework */
1271 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1272 != DDI_SUCCESS) {
1273 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1274 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1275 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1276 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1277 return (AHCI_FAILURE);
1278 }
1279
1280 return (AHCI_SUCCESS);
1281 }
1282
1283 /*
1284 * Unregisters the ahci with sata framework.
1285 */
1286 static int
1287 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1288 {
1289 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1290 "ahci_unregister_sata_hba_tran enter", NULL);
1291
1292 /* Detach from the SATA framework. */
1293 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1294 DDI_SUCCESS) {
1295 return (AHCI_FAILURE);
1296 }
1297
1298 /* Deallocate sata_hba_tran. */
1299 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1300 sizeof (sata_hba_tran_t));
1301
1302 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1303 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1304 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1305
1306 return (AHCI_SUCCESS);
1307 }
1308
1309 #define SET_PORTSTR(str, addrp) \
1310 if (AHCI_ADDR_IS_PORT(addrp)) \
1311 (void) sprintf((str), "%d", (addrp)->aa_port); \
1312 else if (AHCI_ADDR_IS_PMULT(addrp)) \
1313 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \
1314 else \
1315 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \
1316 (addrp)->aa_pmport);
1317
1318 /*
1319 * ahci_tran_probe_port is called by SATA framework. It returns port state,
1320 * port status registers and an attached device type via sata_device
1321 * structure.
1322 *
1323 * We return the cached information from a previous hardware probe. The
1324 * actual hardware probing itself was done either from within
1325 * ahci_initialize_controller() during the driver attach or from a phy
1326 * ready change interrupt handler.
1327 */
1328 static int
1329 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1330 {
1331 ahci_ctl_t *ahci_ctlp;
1332 ahci_port_t *ahci_portp;
1333 ahci_addr_t addr, pmult_addr;
1334 uint8_t cport = sd->satadev_addr.cport;
1335 char portstr[10];
1336 uint8_t device_type;
1337 uint32_t port_state;
1338 uint8_t port;
1339 int rval = SATA_SUCCESS, rval_init;
1340
1341 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1342 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1343
1344 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1345
1346 mutex_enter(&ahci_portp->ahciport_mutex);
1347
1348 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1349 ASSERT(AHCI_ADDR_IS_VALID(&addr));
1350 SET_PORTSTR(portstr, &addr);
1351
1352 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1353 "ahci_tran_probe_port enter: port %s", portstr);
1354
1355 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1356 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1357 ahci_portp->ahciport_pmult_info == NULL)) {
1358 /* port mutliplier is removed. */
1359 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1360 "ahci_tran_probe_port: "
1361 "pmult is removed from port %s", portstr);
1362 mutex_exit(&ahci_portp->ahciport_mutex);
1363 return (SATA_FAILURE);
1364 }
1365
1366 /*
1367 * The sata_device may refer to
1368 * 1. A controller port.
1369 * A controller port should be ready here.
1370 * 2. A port multiplier.
1371 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1372 * it and register the port multiplier to the framework.
1373 * SATA_ADDR_PMULT - check the status of all its device ports.
1374 * 3. A port multiplier port.
1375 * If it has not been initialized, initialized it.
1376 *
1377 * A port multiplier or a port multiplier port may require some
1378 * initialization because we cannot do these time-consuming jobs in an
1379 * interrupt context.
1380 */
1381 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1382 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1383 /* Initialize registers on a port multiplier */
1384 rval_init = ahci_initialize_pmult(ahci_ctlp,
1385 ahci_portp, &pmult_addr, sd);
1386 if (rval_init != AHCI_SUCCESS) {
1387 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1388 "ahci_tran_probe_port: "
1389 "pmult initialization failed.", NULL);
1390 mutex_exit(&ahci_portp->ahciport_mutex);
1391 return (SATA_FAILURE);
1392 }
1393 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1394 /* Check pmports hotplug events */
1395 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1396 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1397 SATA_ADDR_DPMPORT)) {
1398 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1399 &addr, sd) != AHCI_SUCCESS) {
1400 rval = SATA_FAILURE;
1401 goto out;
1402 }
1403 }
1404
1405 /* Update port state and device type */
1406 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1407
1408 switch (port_state) {
1409
1410 case SATA_PSTATE_FAILED:
1411 sd->satadev_state = SATA_PSTATE_FAILED;
1412 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1413 "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1414 goto out;
1415
1416 case SATA_PSTATE_SHUTDOWN:
1417 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1418 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1419 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1420 goto out;
1421
1422 case SATA_PSTATE_PWROFF:
1423 sd->satadev_state = SATA_PSTATE_PWROFF;
1424 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1425 "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1426 goto out;
1427
1428 case SATA_PSTATE_PWRON:
1429 sd->satadev_state = SATA_PSTATE_PWRON;
1430 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1431 "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1432 break;
1433
1434 default:
1435 sd->satadev_state = port_state;
1436 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1437 "ahci_tran_probe_port: port %s PORT NORMAL %x",
1438 portstr, port_state);
1439 break;
1440 }
1441
1442 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1443
1444 switch (device_type) {
1445
1446 case SATA_DTYPE_ATADISK:
1447 sd->satadev_type = SATA_DTYPE_ATADISK;
1448 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1449 "ahci_tran_probe_port: port %s DISK found", portstr);
1450 break;
1451
1452 case SATA_DTYPE_ATAPI:
1453 /*
1454 * HBA driver only knows it's an ATAPI device, and don't know
1455 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1456 * type need to be determined by checking IDENTIFY PACKET
1457 * DEVICE data
1458 */
1459 sd->satadev_type = SATA_DTYPE_ATAPI;
1460 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1461 "ahci_tran_probe_port: port %s ATAPI found", portstr);
1462 break;
1463
1464 case SATA_DTYPE_PMULT:
1465 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1466 sd->satadev_type = SATA_DTYPE_PMULT;
1467
1468 /* Update the number of pmports. */
1469 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1470 sd->satadev_add_info = ahci_portp->
1471 ahciport_pmult_info->ahcipmi_num_dev_ports;
1472
1473 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1474 "ahci_tran_probe_port: port %s Port Multiplier found",
1475 portstr);
1476 break;
1477
1478 case SATA_DTYPE_UNKNOWN:
1479 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1480 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1481 "ahci_tran_probe_port: port %s Unknown device found",
1482 portstr);
1483 break;
1484
1485 default:
1486 /* we don't support any other device types */
1487 sd->satadev_type = SATA_DTYPE_NONE;
1488 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1489 "ahci_tran_probe_port: port %s No device found", portstr);
1490 break;
1491 }
1492
1493 out:
1494 /* Register update only fails while probing a pmult/pmport */
1495 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1496 ahci_update_sata_registers(ahci_ctlp, port, sd);
1497 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1498 if (port_state & SATA_STATE_READY)
1499 if (ahci_update_pmult_pscr(ahci_ctlp,
1500 &addr, sd) != AHCI_SUCCESS)
1501 rval = SATA_FAILURE;
1502 }
1503
1504 /* Check handles for the sata registers access */
1505 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1506 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1507 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1508 DDI_SERVICE_UNAFFECTED);
1509 rval = SATA_FAILURE;
1510 }
1511
1512 mutex_exit(&ahci_portp->ahciport_mutex);
1513 return (rval);
1514 }
1515
1516 /*
1517 * There are four operation modes in sata framework:
1518 * SATA_OPMODE_INTERRUPTS
1519 * SATA_OPMODE_POLLING
1520 * SATA_OPMODE_ASYNCH
1521 * SATA_OPMODE_SYNCH
1522 *
1523 * Their combined meanings as following:
1524 *
1525 * SATA_OPMODE_SYNCH
1526 * The command has to be completed before sata_tran_start functions returns.
1527 * Either interrupts or polling could be used - it's up to the driver.
1528 * Mode used currently for internal, sata-module initiated operations.
1529 *
1530 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1531 * It is the same as the one above.
1532 *
1533 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1534 * The command has to be completed before sata_tran_start function returns.
1535 * No interrupt used, polling only. This should be the mode used for scsi
1536 * packets with FLAG_NOINTR.
1537 *
1538 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1539 * The command may be queued (callback function specified). Interrupts could
1540 * be used. It's normal operation mode.
1541 */
1542 /*
1543 * Called by sata framework to transport a sata packet down stream.
1544 */
1545 static int
1546 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1547 {
1548 ahci_ctl_t *ahci_ctlp;
1549 ahci_port_t *ahci_portp;
1550 ahci_addr_t addr;
1551 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1552 uint8_t port;
1553 char portstr[10];
1554
1555 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1556 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1557
1558 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1559 "ahci_tran_start enter: cport %d satapkt 0x%p",
1560 cport, (void *)spkt);
1561
1562 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1563
1564 mutex_enter(&ahci_portp->ahciport_mutex);
1565 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1566 SET_PORTSTR(portstr, &addr);
1567
1568 /* Sanity check */
1569 if (AHCI_ADDR_IS_PMPORT(&addr)) {
1570 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1571 ahci_portp->ahciport_pmult_info == NULL) {
1572
1573 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1574 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1575 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1576 ahci_update_sata_registers(ahci_ctlp, port,
1577 &spkt->satapkt_device);
1578 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1579 "ahci_tran_start returning PORT_ERROR while "
1580 "pmult removed: port: %s", portstr);
1581 mutex_exit(&ahci_portp->ahciport_mutex);
1582 return (SATA_TRAN_PORT_ERROR);
1583 }
1584
1585 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1586 SATA_STATE_READY)) {
1587 if (!ddi_in_panic() ||
1588 ahci_initialize_pmport(ahci_ctlp,
1589 ahci_portp, &addr) != AHCI_SUCCESS) {
1590 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1591 spkt->satapkt_device.satadev_type =
1592 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1593 spkt->satapkt_device.satadev_state =
1594 AHCIPORT_GET_STATE(ahci_portp, &addr);
1595 ahci_update_sata_registers(ahci_ctlp, port,
1596 &spkt->satapkt_device);
1597 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1598 "ahci_tran_start returning PORT_ERROR "
1599 "while sub-link is not initialized "
1600 "at port: %s", portstr);
1601 mutex_exit(&ahci_portp->ahciport_mutex);
1602 return (SATA_TRAN_PORT_ERROR);
1603 }
1604 }
1605 }
1606
1607 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1608 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1609 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1610 /*
1611 * In case the target driver would send the packet before
1612 * sata framework can have the opportunity to process those
1613 * event reports.
1614 */
1615 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1616 spkt->satapkt_device.satadev_state =
1617 ahci_portp->ahciport_port_state;
1618 ahci_update_sata_registers(ahci_ctlp, port,
1619 &spkt->satapkt_device);
1620 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1621 "ahci_tran_start returning PORT_ERROR while "
1622 "port in FAILED/SHUTDOWN/PWROFF state: "
1623 "port: %s", portstr);
1624 mutex_exit(&ahci_portp->ahciport_mutex);
1625 return (SATA_TRAN_PORT_ERROR);
1626 }
1627
1628 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1629 /*
1630 * ahci_intr_phyrdy_change() may have rendered it to
1631 * SATA_DTYPE_NONE.
1632 */
1633 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1634 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1635 spkt->satapkt_device.satadev_state =
1636 ahci_portp->ahciport_port_state;
1637 ahci_update_sata_registers(ahci_ctlp, port,
1638 &spkt->satapkt_device);
1639 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1640 "ahci_tran_start returning PORT_ERROR while "
1641 "no device attached: port: %s", portstr);
1642 mutex_exit(&ahci_portp->ahciport_mutex);
1643 return (SATA_TRAN_PORT_ERROR);
1644 }
1645
1646 /* R/W PMULT command will occupy the whole HBA port */
1647 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1648 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1649 "ahci_tran_start returning BUSY while "
1650 "executing READ/WRITE PORT-MULT command: "
1651 "port: %s", portstr);
1652 spkt->satapkt_reason = SATA_PKT_BUSY;
1653 mutex_exit(&ahci_portp->ahciport_mutex);
1654 return (SATA_TRAN_BUSY);
1655 }
1656
1657 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1658 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1659 "ahci_tran_start returning BUSY while "
1660 "hot-plug in progress: port: %s", portstr);
1661 spkt->satapkt_reason = SATA_PKT_BUSY;
1662 mutex_exit(&ahci_portp->ahciport_mutex);
1663 return (SATA_TRAN_BUSY);
1664 }
1665
1666 /*
1667 * SATA HBA driver should remember that a device was reset and it
1668 * is supposed to reject any packets which do not specify either
1669 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1670 *
1671 * This is to prevent a race condition when a device was arbitrarily
1672 * reset by the HBA driver (and lost it's setting) and a target
1673 * driver sending some commands to a device before the sata framework
1674 * has a chance to restore the device setting (such as cache enable/
1675 * disable or other resettable stuff).
1676 */
1677 /*
1678 * It is unnecessary to use specific flags to indicate
1679 * reset_in_progress for a pmport. While mopping, all command will be
1680 * mopped so that the entire HBA port is being dealt as a single
1681 * object.
1682 */
1683 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1684 ahci_portp->ahciport_reset_in_progress = 0;
1685 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1686 "ahci_tran_start [CLEAR] the "
1687 "reset_in_progress for port: %d", port);
1688 }
1689
1690 if (ahci_portp->ahciport_reset_in_progress &&
1691 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1692 ! ddi_in_panic()) {
1693 spkt->satapkt_reason = SATA_PKT_BUSY;
1694 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1695 "ahci_tran_start returning BUSY while "
1696 "reset in progress: port: %d", port);
1697 mutex_exit(&ahci_portp->ahciport_mutex);
1698 return (SATA_TRAN_BUSY);
1699 }
1700
1701 #ifdef AHCI_DEBUG
1702 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1703 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1704 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1705 spkt, port);
1706 }
1707 #endif
1708
1709 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1710 spkt->satapkt_reason = SATA_PKT_BUSY;
1711 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1712 "ahci_tran_start returning BUSY while "
1713 "mopping in progress: port: %d", port);
1714 mutex_exit(&ahci_portp->ahciport_mutex);
1715 return (SATA_TRAN_BUSY);
1716 }
1717
1718 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1719 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1720 DDI_SERVICE_UNAFFECTED);
1721 mutex_exit(&ahci_portp->ahciport_mutex);
1722 return (SATA_TRAN_BUSY);
1723 }
1724
1725 if (spkt->satapkt_op_mode &
1726 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1727 /*
1728 * If a SYNC command to be executed in interrupt context,
1729 * bounce it back to sata module.
1730 */
1731 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1732 servicing_interrupt()) {
1733 spkt->satapkt_reason = SATA_PKT_BUSY;
1734 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1735 "ahci_tran_start returning BUSY while "
1736 "sending SYNC mode under interrupt context: "
1737 "port : %d", port);
1738 mutex_exit(&ahci_portp->ahciport_mutex);
1739 return (SATA_TRAN_BUSY);
1740 }
1741
1742 /* We need to do the sync start now */
1743 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1744 spkt) == AHCI_FAILURE) {
1745 goto fail_out;
1746 }
1747 } else {
1748 /* Async start, using interrupt */
1749 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1750 == AHCI_FAILURE) {
1751 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1752 goto fail_out;
1753 }
1754 }
1755
1756 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1757 "sata tran accepted: port %s", portstr);
1758
1759 mutex_exit(&ahci_portp->ahciport_mutex);
1760 return (SATA_TRAN_ACCEPTED);
1761
1762 fail_out:
1763 /*
1764 * Failed to deliver packet to the controller.
1765 * Check if it's caused by invalid handles.
1766 */
1767 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1768 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1769 spkt->satapkt_device.satadev_type =
1770 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1771 spkt->satapkt_device.satadev_state =
1772 AHCIPORT_GET_STATE(ahci_portp, &addr);
1773 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1774 mutex_exit(&ahci_portp->ahciport_mutex);
1775 return (SATA_TRAN_PORT_ERROR);
1776 }
1777
1778 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1779 "return QUEUE_FULL: port %d", port);
1780 mutex_exit(&ahci_portp->ahciport_mutex);
1781 return (SATA_TRAN_QUEUE_FULL);
1782 }
1783
1784 /*
1785 * SATA_OPMODE_SYNCH flag is set
1786 *
1787 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1788 * without interrupt, otherwise we can still use the interrupt.
1789 */
1790 static int
1791 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1792 ahci_addr_t *addrp, sata_pkt_t *spkt)
1793 {
1794 int pkt_timeout_ticks;
1795 uint32_t timeout_tags;
1796 int rval;
1797 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1798 uint8_t port = addrp->aa_port;
1799
1800 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1801
1802 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1803 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1804
1805 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1806 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1807 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1808 addrp, spkt)) == AHCI_FAILURE) {
1809 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1810 return (rval);
1811 }
1812
1813 pkt_timeout_ticks =
1814 drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1815
1816 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1817 mutex_exit(&ahci_portp->ahciport_mutex);
1818
1819 /* Simulate the interrupt */
1820 ahci_port_intr(ahci_ctlp, ahci_portp, port);
1821
1822 drv_usecwait(AHCI_10MS_USECS);
1823
1824 mutex_enter(&ahci_portp->ahciport_mutex);
1825 pkt_timeout_ticks -= AHCI_10MS_TICKS;
1826 if (pkt_timeout_ticks < 0) {
1827 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1828 "port %d satapkt 0x%p timed out\n",
1829 instance, port, (void *)spkt);
1830 timeout_tags = (0x1 << rval);
1831 mutex_exit(&ahci_portp->ahciport_mutex);
1832 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1833 port, timeout_tags);
1834 mutex_enter(&ahci_portp->ahciport_mutex);
1835 }
1836 }
1837 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1838 return (AHCI_SUCCESS);
1839
1840 } else {
1841 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1842 addrp, spkt)) == AHCI_FAILURE)
1843 return (rval);
1844
1845 #if AHCI_DEBUG
1846 /*
1847 * Note that the driver always uses the slot 0 to deliver
1848 * REQUEST SENSE or READ LOG EXT command
1849 */
1850 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1851 ASSERT(rval == 0);
1852 #endif
1853
1854 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1855 cv_wait(&ahci_portp->ahciport_cv,
1856 &ahci_portp->ahciport_mutex);
1857
1858 return (AHCI_SUCCESS);
1859 }
1860 }
1861
1862 /*
1863 * Searches for and claims a free command slot.
1864 *
1865 * Returns value:
1866 *
1867 * AHCI_FAILURE returned only if
1868 * 1. No empty slot left
1869 * 2. Non-queued command requested while queued command(s) is outstanding
1870 * 3. Queued command requested while non-queued command(s) is outstanding
1871 * 4. HBA doesn't support multiple-use of command list while already a
1872 * non-queued command is oustanding
1873 * 5. Queued command requested while some queued command(s) has been
1874 * outstanding on a different port multiplier port. (AHCI spec 1.2,
1875 * 9.1.2)
1876 *
1877 * claimed slot number returned if succeeded
1878 *
1879 * NOTE: it will always return slot 0 for following commands to simplify the
1880 * algorithm.
1881 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process
1882 * 2. READ/WRITE PORTMULT command
1883 */
1884 static int
1885 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1886 ahci_addr_t *addrp, int command_type)
1887 {
1888 uint32_t port_cmd_issue;
1889 uint32_t free_slots;
1890 int slot;
1891
1892 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1893
1894 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1895 "ahciport_pending_tags = 0x%x "
1896 "ahciport_pending_ncq_tags = 0x%x",
1897 ahci_portp->ahciport_pending_tags,
1898 ahci_portp->ahciport_pending_ncq_tags);
1899
1900 /*
1901 * According to the AHCI spec, system software is responsible to
1902 * ensure that queued and non-queued commands are not mixed in
1903 * the command list.
1904 */
1905 if (command_type == AHCI_NON_NCQ_CMD) {
1906 /* Non-NCQ command request */
1907 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1908 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1909 "ahci_claim_free_slot: there is still pending "
1910 "queued command(s) in the command list, "
1911 "so no available slot for the non-queued "
1912 "command", NULL);
1913 return (AHCI_FAILURE);
1914 }
1915 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1916 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1917 "ahci_claim_free_slot: there is still pending "
1918 "read/write port-mult command(s) in command list, "
1919 "so no available slot for the non-queued command",
1920 NULL);
1921 return (AHCI_FAILURE);
1922 }
1923 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
1924 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1925 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1926 "ahci_claim_free_slot: HBA cannot support multiple-"
1927 "use of the command list for non-queued commands",
1928 NULL);
1929 return (AHCI_FAILURE);
1930 }
1931 free_slots = (~ahci_portp->ahciport_pending_tags) &
1932 AHCI_SLOT_MASK(ahci_ctlp);
1933 } else if (command_type == AHCI_NCQ_CMD) {
1934 /* NCQ command request */
1935 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1936 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1937 "ahci_claim_free_slot: there is still pending "
1938 "non-queued command(s) in the command list, "
1939 "so no available slot for the queued command",
1940 NULL);
1941 return (AHCI_FAILURE);
1942 }
1943
1944 /*
1945 * NCQ commands cannot be sent to different port multiplier
1946 * ports in Command-Based Switching mode
1947 */
1948 /*
1949 * NOTE: In Command-Based Switching mode, AHCI controller
1950 * usually reports a 'Handshake Error' when multiple NCQ
1951 * commands are outstanding simultaneously.
1952 */
1953 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
1954 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1955 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
1956 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
1957 AHCIPORT_NCQ_PMPORT(ahci_portp) !=
1958 addrp->aa_pmport) {
1959 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1960 "ahci_claim_free_slot: there is still "
1961 "pending queued command(s) in the "
1962 "command list for another Port Multiplier "
1963 "port, so no available slot.", NULL);
1964 return (AHCI_FAILURE);
1965 }
1966 }
1967
1968 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
1969 AHCI_NCQ_SLOT_MASK(ahci_portp);
1970 } else if (command_type == AHCI_ERR_RETRI_CMD) {
1971 /* Error retrieval command request */
1972 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1973 "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
1974 "SENSE or READ LOG EXT command", NULL);
1975 slot = 0;
1976 goto out;
1977 } else if (command_type == AHCI_RDWR_PMULT_CMD) {
1978 /*
1979 * An extra check on PxCI. Sometimes PxCI bits may not be
1980 * cleared during hot-plug or error recovery process.
1981 */
1982 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1983 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
1984
1985 if (port_cmd_issue != 0) {
1986 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1987 "ahci_claim_free_slot: there is still pending "
1988 "command(s) in command list (0x%x/0x%x, PxCI %x),"
1989 "so no available slot for R/W PMULT command.",
1990 NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
1991 NCQ_CMD_IN_PROGRESS(ahci_portp),
1992 port_cmd_issue);
1993 return (AHCI_FAILURE);
1994 }
1995
1996 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1997 "ahci_claim_free_slot: slot 0 is allocated for "
1998 "READ/WRITE PORTMULT command", NULL);
1999 slot = 0;
2000 goto out;
2001 }
2002
2003 slot = ddi_ffs(free_slots) - 1;
2004 if (slot == -1) {
2005 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2006 "ahci_claim_free_slot: no empty slots", NULL);
2007 return (AHCI_FAILURE);
2008 }
2009
2010 /*
2011 * According to the AHCI spec, to allow a simple mechanism for the
2012 * HBA to map command list slots to queue entries, software must
2013 * match the tag number it uses to the slot it is placing the command
2014 * in. For example, if a queued command is placed in slot 5, the tag
2015 * for that command must be 5.
2016 */
2017 if (command_type == AHCI_NCQ_CMD) {
2018 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2019 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2020 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2021 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2022 }
2023 }
2024
2025 ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2026
2027 out:
2028 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2029 "ahci_claim_free_slot: found slot: 0x%x", slot);
2030
2031 return (slot);
2032 }
2033
2034 /*
2035 * Builds the Command Table for the sata packet and delivers it to controller.
2036 *
2037 * Returns:
2038 * slot number if we can obtain a slot successfully
2039 * otherwise, return AHCI_FAILURE
2040 */
2041 static int
2042 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2043 ahci_addr_t *addrp, sata_pkt_t *spkt)
2044 {
2045 int cmd_slot;
2046 sata_cmd_t *scmd;
2047 ahci_fis_h2d_register_t *h2d_register_fisp;
2048 ahci_cmd_table_t *cmd_table;
2049 ahci_cmd_header_t *cmd_header;
2050 int ncookies;
2051 int i;
2052 int command_type = AHCI_NON_NCQ_CMD;
2053 int ncq_qdepth;
2054 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2055 uint8_t port, pmport;
2056 #if AHCI_DEBUG
2057 uint32_t *ptr;
2058 uint8_t *ptr2;
2059 #endif
2060
2061 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2062
2063 port = addrp->aa_port;
2064 pmport = addrp->aa_pmport;
2065
2066 spkt->satapkt_reason = SATA_PKT_BUSY;
2067
2068 scmd = &spkt->satapkt_cmd;
2069
2070 /* Check if the command is a NCQ command */
2071 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2072 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2073 command_type = AHCI_NCQ_CMD;
2074
2075 /*
2076 * When NCQ is support, system software must determine the
2077 * maximum tag allowed by the device and the HBA, and it
2078 * must use a value not beyond of the lower bound of the two.
2079 *
2080 * Sata module is going to calculate the qdepth and send
2081 * down to HBA driver via sata_cmd.
2082 */
2083 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2084
2085 /*
2086 * At the moment, the driver doesn't support the dynamic
2087 * setting of the maximum ncq depth, and the value can be
2088 * set either during the attach or after hot-plug insertion.
2089 */
2090 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2091 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2092 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2093 "ahci_deliver_satapkt: port %d the max tags for "
2094 "NCQ command is %d", port, ncq_qdepth);
2095 } else {
2096 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2097 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2098 " port %d the max tag for NCQ command is "
2099 "requested to change from %d to %d, at the"
2100 " moment the driver doesn't support the "
2101 "dynamic change so it's going to "
2102 "still use the previous tag value",
2103 instance, port,
2104 ahci_portp->ahciport_max_ncq_tags,
2105 ncq_qdepth);
2106 }
2107 }
2108 }
2109
2110 /* Check if the command is an error retrieval command */
2111 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2112 command_type = AHCI_ERR_RETRI_CMD;
2113
2114 /* Check if the command is an read/write pmult command */
2115 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2116 command_type = AHCI_RDWR_PMULT_CMD;
2117
2118 /* Check if there is an empty command slot */
2119 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2120 addrp, command_type);
2121 if (cmd_slot == AHCI_FAILURE) {
2122 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2123 return (AHCI_FAILURE);
2124 }
2125
2126 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2127 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2128 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2129 cmd_slot, port, (void *)spkt);
2130
2131 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2132 bzero((void *)cmd_table, ahci_cmd_table_size);
2133
2134 /* For data transfer operations, it is the H2D Register FIS */
2135 h2d_register_fisp =
2136 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2137
2138 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2139
2140 /*
2141 * PMP field only make sense when target is a port multiplier or a
2142 * device behind a port multiplier. Otherwise should set it to 0.
2143 */
2144 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2145 SET_FIS_PMP(h2d_register_fisp, pmport);
2146
2147 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2148 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2149 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2150 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2151
2152 switch (scmd->satacmd_addr_type) {
2153
2154 case 0:
2155 /*
2156 * satacmd_addr_type will be 0 for the commands below:
2157 * ATAPI command
2158 * SATAC_IDLE_IM
2159 * SATAC_STANDBY_IM
2160 * SATAC_DOWNLOAD_MICROCODE
2161 * SATAC_FLUSH_CACHE
2162 * SATAC_SET_FEATURES
2163 * SATAC_SMART
2164 * SATAC_ID_PACKET_DEVICE
2165 * SATAC_ID_DEVICE
2166 * SATAC_READ_PORTMULT
2167 * SATAC_WRITE_PORTMULT
2168 */
2169 /* FALLTHRU */
2170
2171 case ATA_ADDR_LBA:
2172 /* FALLTHRU */
2173
2174 case ATA_ADDR_LBA28:
2175 /* LBA[7:0] */
2176 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2177
2178 /* LBA[15:8] */
2179 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2180
2181 /* LBA[23:16] */
2182 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2183
2184 /* LBA [27:24] (also called dev_head) */
2185 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2186
2187 break;
2188
2189 case ATA_ADDR_LBA48:
2190 /* LBA[7:0] */
2191 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2192
2193 /* LBA[15:8] */
2194 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2195
2196 /* LBA[23:16] */
2197 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2198
2199 /* LBA [31:24] */
2200 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2201 scmd->satacmd_lba_low_msb);
2202
2203 /* LBA [39:32] */
2204 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2205 scmd->satacmd_lba_mid_msb);
2206
2207 /* LBA [47:40] */
2208 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2209 scmd->satacmd_lba_high_msb);
2210
2211 /* Set dev_head */
2212 SET_FIS_DEV_HEAD(h2d_register_fisp,
2213 scmd->satacmd_device_reg);
2214
2215 /* Set the extended sector count and features */
2216 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2217 scmd->satacmd_sec_count_msb);
2218 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2219 scmd->satacmd_features_reg_ext);
2220 break;
2221 }
2222
2223 /*
2224 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2225 * filled into features field, and sector count 8:15 is filled into
2226 * features (exp) field. The hba driver doesn't need to anything
2227 * special with regard to this, since sata framework has already
2228 * done so.
2229 *
2230 * However the driver needs to make sure TAG is filled into sector
2231 * field.
2232 */
2233 if (command_type == AHCI_NCQ_CMD) {
2234 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2235 (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2236 }
2237
2238 ncookies = scmd->satacmd_num_dma_cookies;
2239 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2240 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2241 ncookies, ahci_dma_prdt_number);
2242
2243 ASSERT(ncookies <= ahci_dma_prdt_number);
2244 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2245
2246 /* *** now fill the scatter gather list ******* */
2247 for (i = 0; i < ncookies; i++) {
2248 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2249 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2250 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2251 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2252 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2253 scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2254 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2255 scmd->satacmd_dma_cookie_list[i].dmac_size;
2256 }
2257
2258 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2259 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2260 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2261
2262 /* The ACMD field is filled in for ATAPI command */
2263 if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2264 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2265 SATA_ATAPI_MAX_CDB_LEN);
2266 }
2267
2268 /* Set Command Header in Command List */
2269 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2270 BZERO_DESCR_INFO(cmd_header);
2271 BZERO_PRD_BYTE_COUNT(cmd_header);
2272
2273 /* Set the number of entries in the PRD table */
2274 SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2275
2276 /* Set the length of the command in the CFIS area */
2277 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2278
2279 /*
2280 * PMP field only make sense when target is a port multiplier or a
2281 * device behind a port multiplier. Otherwise should set it to 0.
2282 */
2283 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2284 SET_PORT_MULTI_PORT(cmd_header, pmport);
2285
2286 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2287 "sata_data_direction = 0x%x",
2288 scmd->satacmd_flags.sata_data_direction);
2289
2290 /* Set A bit if it is an ATAPI command */
2291 if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2292 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2293
2294 /* Set W bit if data is going to the device */
2295 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2296 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2297
2298 /*
2299 * Set the prefetchable bit - this bit is only valid if the PRDTL
2300 * field is non-zero or the ATAPI 'A' bit is set in the command
2301 * header. This bit cannot be set when using native command
2302 * queuing commands or when using FIS-based switching with a Port
2303 * multiplier.
2304 */
2305 if (command_type != AHCI_NCQ_CMD)
2306 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2307
2308 /*
2309 * Now remember the sata packet in ahciport_slot_pkts[].
2310 * Error retrieval command and r/w port multiplier command will
2311 * be stored specifically for each port.
2312 */
2313 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2314 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2315 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2316
2317 /*
2318 * Keep the timeout value
2319 */
2320 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2321
2322 /*
2323 * If the intial timout is less than 1 tick, then make it longer by
2324 * 1 tick to avoid immediate timeout
2325 */
2326 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2327 ahci_watchdog_timeout)
2328 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2329 ahci_watchdog_timeout;
2330
2331 #if AHCI_DEBUG
2332 if (ahci_debug_flags & AHCIDBG_ATACMD &&
2333 scmd->satacmd_cmd_reg != SATAC_PACKET ||
2334 ahci_debug_flags & AHCIDBG_ATAPICMD &&
2335 scmd->satacmd_cmd_reg == SATAC_PACKET) {
2336
2337 /* Dump the command header and table */
2338 ahci_log(ahci_ctlp, CE_WARN, "\n");
2339 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2340 "0x%p cmd_reg 0x%x port %d", spkt,
2341 scmd->satacmd_cmd_reg, port);
2342 ptr = (uint32_t *)cmd_header;
2343 ahci_log(ahci_ctlp, CE_WARN,
2344 " Command Header:%8x %8x %8x %8x",
2345 ptr[0], ptr[1], ptr[2], ptr[3]);
2346
2347 /* Dump the H2D register FIS */
2348 ptr = (uint32_t *)h2d_register_fisp;
2349 ahci_log(ahci_ctlp, CE_WARN,
2350 " Command FIS: %8x %8x %8x %8x",
2351 ptr[0], ptr[1], ptr[2], ptr[3]);
2352
2353 /* Dump the ACMD register FIS */
2354 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2355 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2356 if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2357 ahci_log(ahci_ctlp, CE_WARN,
2358 " ATAPI command: %2x %2x %2x %2x "
2359 "%2x %2x %2x %2x",
2360 ptr2[8 * i], ptr2[8 * i + 1],
2361 ptr2[8 * i + 2], ptr2[8 * i + 3],
2362 ptr2[8 * i + 4], ptr2[8 * i + 5],
2363 ptr2[8 * i + 6], ptr2[8 * i + 7]);
2364
2365 /* Dump the PRDT */
2366 for (i = 0; i < ncookies; i++) {
2367 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2368 ahci_log(ahci_ctlp, CE_WARN,
2369 " Cookie %d: %8x %8x %8x %8x",
2370 i, ptr[0], ptr[1], ptr[2], ptr[3]);
2371 }
2372 }
2373 #endif
2374
2375 (void) ddi_dma_sync(
2376 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2377 0,
2378 ahci_cmd_table_size,
2379 DDI_DMA_SYNC_FORDEV);
2380
2381 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2382 cmd_slot * sizeof (ahci_cmd_header_t),
2383 sizeof (ahci_cmd_header_t),
2384 DDI_DMA_SYNC_FORDEV);
2385
2386 if ((ahci_check_dma_handle(ahci_portp->
2387 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2388 ahci_check_dma_handle(ahci_portp->
2389 ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2390 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2391 DDI_SERVICE_UNAFFECTED);
2392 return (AHCI_FAILURE);
2393 }
2394
2395 /* Set the corresponding bit in the PxSACT.DS for queued command */
2396 if (command_type == AHCI_NCQ_CMD) {
2397 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2398 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2399 (0x1 << cmd_slot));
2400 }
2401
2402 /* Indicate to the HBA that a command is active. */
2403 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2404 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2405 (0x1 << cmd_slot));
2406
2407 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2408 "exit: port %d", port);
2409
2410 /* Make sure the command is started by the PxSACT/PxCI */
2411 if (ahci_check_acc_handle(ahci_ctlp->
2412 ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2413 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2414 DDI_SERVICE_UNAFFECTED);
2415 return (AHCI_FAILURE);
2416 }
2417
2418 return (cmd_slot);
2419 }
2420
2421 /*
2422 * Called by the sata framework to abort the previously sent packet(s).
2423 *
2424 * Reset device to abort commands.
2425 */
2426 static int
2427 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2428 {
2429 ahci_ctl_t *ahci_ctlp;
2430 ahci_port_t *ahci_portp;
2431 uint32_t slot_status = 0;
2432 uint32_t aborted_tags = 0;
2433 uint32_t finished_tags = 0;
2434 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2435 uint8_t port;
2436 int tmp_slot;
2437 int instance = ddi_get_instance(dip);
2438
2439 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2440 port = ahci_ctlp->ahcictl_cport_to_port[cport];
2441
2442 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2443 "ahci_tran_abort enter: port %d", port);
2444
2445 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2446 mutex_enter(&ahci_portp->ahciport_mutex);
2447
2448 /*
2449 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2450 * commands are being mopped, therefore there is nothing else to do
2451 */
2452 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2453 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2454 "ahci_tran_abort: port %d is in "
2455 "mopping process, so just return directly ", port);
2456 mutex_exit(&ahci_portp->ahciport_mutex);
2457 return (SATA_SUCCESS);
2458 }
2459
2460 /*
2461 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2462 * command is being executed so no other commands is outstanding,
2463 * nothing to do.
2464 */
2465 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2466 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2467 "ahci_tran_abort: port %d is reading/writing "
2468 "port multiplier, so just return directly ", port);
2469 mutex_exit(&ahci_portp->ahciport_mutex);
2470 return (SATA_SUCCESS);
2471 }
2472
2473 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2474 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2475 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2476 /*
2477 * In case the targer driver would send the request before
2478 * sata framework can have the opportunity to process those
2479 * event reports.
2480 */
2481 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2482 spkt->satapkt_device.satadev_state =
2483 ahci_portp->ahciport_port_state;
2484 ahci_update_sata_registers(ahci_ctlp, port,
2485 &spkt->satapkt_device);
2486 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2487 "ahci_tran_abort returning SATA_FAILURE while "
2488 "port in FAILED/SHUTDOWN/PWROFF state: "
2489 "port: %d", port);
2490 mutex_exit(&ahci_portp->ahciport_mutex);
2491 return (SATA_FAILURE);
2492 }
2493
2494 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2495 /*
2496 * ahci_intr_phyrdy_change() may have rendered it to
2497 * AHCI_PORT_TYPE_NODEV.
2498 */
2499 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2500 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2501 spkt->satapkt_device.satadev_state =
2502 ahci_portp->ahciport_port_state;
2503 ahci_update_sata_registers(ahci_ctlp, port,
2504 &spkt->satapkt_device);
2505 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2506 "ahci_tran_abort returning SATA_FAILURE while "
2507 "no device attached: port: %d", port);
2508 mutex_exit(&ahci_portp->ahciport_mutex);
2509 return (SATA_FAILURE);
2510 }
2511
2512 if (flag == SATA_ABORT_ALL_PACKETS) {
2513 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2514 aborted_tags = ahci_portp->ahciport_pending_tags;
2515 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2516 aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2517
2518 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2519 instance, port);
2520 } else {
2521 aborted_tags = 0xffffffff;
2522 /*
2523 * Aborting one specific packet, first search the
2524 * ahciport_slot_pkts[] list for matching spkt.
2525 */
2526 for (tmp_slot = 0;
2527 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2528 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2529 aborted_tags = (0x1 << tmp_slot);
2530 break;
2531 }
2532 }
2533
2534 if (aborted_tags == 0xffffffff) {
2535 /* request packet is not on the pending list */
2536 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2537 "Cannot find the aborting pkt 0x%p on the "
2538 "pending list", (void *)spkt);
2539 ahci_update_sata_registers(ahci_ctlp, port,
2540 &spkt->satapkt_device);
2541 mutex_exit(&ahci_portp->ahciport_mutex);
2542 return (SATA_FAILURE);
2543 }
2544 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2545 instance, port, (void *)spkt);
2546 }
2547
2548 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2549 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2550 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2551 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2552 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2553 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2554
2555 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2556 ahci_portp->ahciport_mop_in_progress++;
2557
2558 /*
2559 * To abort the packet(s), first we are trying to clear PxCMD.ST
2560 * to stop the port, and if the port can be stopped
2561 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2562 * then we just send back the aborted packet(s) with ABORTED flag
2563 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2564 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2565 * perform a COMRESET.
2566 */
2567 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2568 ahci_portp, port, NULL, NULL);
2569
2570 /*
2571 * Compute which have finished and which need to be retried.
2572 *
2573 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2574 * minus the slot_status. The aborted_tags has to be deducted by
2575 * finished_tags since we can't possibly abort a tag which had finished
2576 * already.
2577 */
2578 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2579 finished_tags = ahci_portp->ahciport_pending_tags &
2580 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2581 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2582 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2583 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2584
2585 aborted_tags &= ~finished_tags;
2586
2587 ahci_mop_commands(ahci_ctlp,
2588 ahci_portp,
2589 slot_status,
2590 0, /* failed tags */
2591 0, /* timeout tags */
2592 aborted_tags,
2593 0); /* reset tags */
2594
2595 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2596 mutex_exit(&ahci_portp->ahciport_mutex);
2597
2598 return (SATA_SUCCESS);
2599 }
2600
2601 /*
2602 * Used to do device reset and reject all the pending packets on a device
2603 * during the reset operation.
2604 *
2605 * NOTE: ONLY called by ahci_tran_reset_dport
2606 */
2607 static int
2608 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2609 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2610 {
2611 uint32_t slot_status = 0;
2612 uint32_t reset_tags = 0;
2613 uint32_t finished_tags = 0;
2614 uint8_t port = addrp->aa_port;
2615 sata_device_t sdevice;
2616 int ret;
2617
2618 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2619
2620 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2621 "ahci_reset_device_reject_pkts on port: %d", port);
2622
2623 /*
2624 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2625 * commands are being mopped, therefore there is nothing else to do
2626 */
2627 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2628 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2629 "ahci_reset_device_reject_pkts: port %d is in "
2630 "mopping process, so return directly ", port);
2631 return (SATA_SUCCESS);
2632 }
2633
2634 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2635 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2636 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2637 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2638 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2639 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2640 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2641 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2642 }
2643
2644 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2645 != AHCI_SUCCESS) {
2646 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2647 "Try to do a port reset after software "
2648 "reset failed", port);
2649 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2650 if (ret != AHCI_SUCCESS) {
2651 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2652 "ahci_reset_device_reject_pkts: port %d "
2653 "failed", port);
2654 return (SATA_FAILURE);
2655 }
2656 }
2657 /* Set the reset in progress flag */
2658 ahci_portp->ahciport_reset_in_progress = 1;
2659
2660 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2661 ahci_portp->ahciport_mop_in_progress++;
2662
2663 /* Indicate to the framework that a reset has happened */
2664 bzero((void *)&sdevice, sizeof (sata_device_t));
2665 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2666 sdevice.satadev_addr.pmport = 0;
2667 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2668 sdevice.satadev_state = SATA_DSTATE_RESET |
2669 SATA_DSTATE_PWR_ACTIVE;
2670 mutex_exit(&ahci_portp->ahciport_mutex);
2671 sata_hba_event_notify(
2672 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2673 &sdevice,
2674 SATA_EVNT_DEVICE_RESET);
2675 mutex_enter(&ahci_portp->ahciport_mutex);
2676
2677 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2678 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2679
2680 /* Next try to mop the pending commands */
2681 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2682 finished_tags = ahci_portp->ahciport_pending_tags &
2683 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2684 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2685 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2686 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2687
2688 reset_tags &= ~finished_tags;
2689
2690 ahci_mop_commands(ahci_ctlp,
2691 ahci_portp,
2692 slot_status,
2693 0, /* failed tags */
2694 0, /* timeout tags */
2695 0, /* aborted tags */
2696 reset_tags); /* reset tags */
2697
2698 return (SATA_SUCCESS);
2699 }
2700
2701 /*
2702 * Used to do device reset and reject all the pending packets on a device
2703 * during the reset operation.
2704 *
2705 * NOTE: ONLY called by ahci_tran_reset_dport
2706 */
2707 static int
2708 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2709 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2710 {
2711 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2712 uint8_t port = addrp->aa_port;
2713 uint8_t pmport = addrp->aa_pmport;
2714 sata_device_t sdevice;
2715
2716 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2717
2718 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2719 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2720
2721 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2722 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2723 "ahci_reset_pmdevice_reject_pkts: port %d is in "
2724 "mopping process, so return directly ", port);
2725 return (SATA_SUCCESS);
2726 }
2727
2728 /* Checking for outstanding commands */
2729 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2730 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2731 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2732 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2733 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2734 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2735 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2736 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2737 }
2738
2739 /* Issue SOFTWARE reset command. */
2740 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2741 != AHCI_SUCCESS) {
2742 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2743 "Try to do a port reset after software "
2744 "reset failed", port);
2745 return (SATA_FAILURE);
2746 }
2747
2748 /* Set the reset in progress flag */
2749 ahci_portp->ahciport_reset_in_progress = 1;
2750
2751 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2752 ahci_portp->ahciport_mop_in_progress++;
2753
2754 /* Indicate to the framework that a reset has happened */
2755 bzero((void *)&sdevice, sizeof (sata_device_t));
2756 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2757 sdevice.satadev_addr.pmport = pmport;
2758 if (AHCI_ADDR_IS_PMULT(addrp))
2759 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2760 else
2761 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2762 sdevice.satadev_state = SATA_DSTATE_RESET |
2763 SATA_DSTATE_PWR_ACTIVE;
2764 mutex_exit(&ahci_portp->ahciport_mutex);
2765 sata_hba_event_notify(
2766 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2767 &sdevice,
2768 SATA_EVNT_DEVICE_RESET);
2769 mutex_enter(&ahci_portp->ahciport_mutex);
2770
2771 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2772 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2773 port, pmport);
2774
2775 /* Next try to mop the pending commands */
2776 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2777 finished_tags = ahci_portp->ahciport_pending_tags &
2778 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2779 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2780 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2781 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2782 reset_tags &= ~finished_tags;
2783
2784 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2785 "reset_tags = %x, finished_tags = %x, slot_status = %x",
2786 reset_tags, finished_tags, slot_status);
2787
2788 /*
2789 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2790 * try to reset a single device behind a port multiplier will
2791 * terminate all the commands on that HBA port. We need mop these
2792 * commands as well.
2793 */
2794 ahci_mop_commands(ahci_ctlp,
2795 ahci_portp,
2796 slot_status,
2797 0, /* failed tags */
2798 0, /* timeout tags */
2799 0, /* aborted tags */
2800 reset_tags); /* reset tags */
2801
2802 return (SATA_SUCCESS);
2803 }
2804
2805 /*
2806 * Used to do port reset and reject all the pending packets on a port during
2807 * the reset operation.
2808 */
2809 static int
2810 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2811 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2812 {
2813 uint32_t slot_status = 0;
2814 uint32_t reset_tags = 0;
2815 uint32_t finished_tags = 0;
2816 uint8_t port = addrp->aa_port;
2817
2818 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2819
2820 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2821 "ahci_reset_port_reject_pkts at port: %d", port);
2822
2823 /*
2824 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2825 * commands are being mopped, therefore there is nothing else to do
2826 */
2827 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2828 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2829 "ahci_reset_port_reject_pkts: port %d is in "
2830 "mopping process, so return directly ", port);
2831 return (SATA_SUCCESS);
2832 }
2833
2834 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2835 ahci_portp->ahciport_mop_in_progress++;
2836
2837 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2838 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2839 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2840 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2841 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2842 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2843 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2844 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2845 }
2846
2847 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2848 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2849 NULL) != AHCI_SUCCESS) {
2850
2851 /* Clear mop flag */
2852 ahci_portp->ahciport_mop_in_progress--;
2853 if (ahci_portp->ahciport_mop_in_progress == 0)
2854 ahci_portp->ahciport_flags &=
2855 ~AHCI_PORT_FLAG_MOPPING;
2856 return (SATA_FAILURE);
2857 }
2858
2859 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2860 finished_tags = ahci_portp->ahciport_pending_tags &
2861 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2862 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2863 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2864 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2865
2866 reset_tags &= ~finished_tags;
2867
2868 ahci_mop_commands(ahci_ctlp,
2869 ahci_portp,
2870 slot_status,
2871 0, /* failed tags */
2872 0, /* timeout tags */
2873 0, /* aborted tags */
2874 reset_tags); /* reset tags */
2875
2876 return (SATA_SUCCESS);
2877 }
2878
2879 /*
2880 * Used to do hba reset and reject all the pending packets on all ports
2881 * during the reset operation.
2882 */
2883 static int
2884 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2885 {
2886 ahci_port_t *ahci_portp;
2887 uint32_t slot_status[AHCI_MAX_PORTS];
2888 uint32_t reset_tags[AHCI_MAX_PORTS];
2889 uint32_t finished_tags[AHCI_MAX_PORTS];
2890 int port;
2891 int ret = SATA_SUCCESS;
2892
2893 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2894 "ahci_reset_hba_reject_pkts enter", NULL);
2895
2896 bzero(slot_status, sizeof (slot_status));
2897 bzero(reset_tags, sizeof (reset_tags));
2898 bzero(finished_tags, sizeof (finished_tags));
2899
2900 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2901 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2902 continue;
2903 }
2904
2905 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2906
2907 mutex_enter(&ahci_portp->ahciport_mutex);
2908 ahci_portp->ahciport_reset_in_progress = 1;
2909 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2910 slot_status[port] = ddi_get32(
2911 ahci_ctlp->ahcictl_ahci_acc_handle,
2912 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2913 reset_tags[port] = slot_status[port] &
2914 AHCI_SLOT_MASK(ahci_ctlp);
2915 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2916 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2917 port, reset_tags[port],
2918 ahci_portp->ahciport_pending_tags);
2919 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2920 slot_status[port] = ddi_get32(
2921 ahci_ctlp->ahcictl_ahci_acc_handle,
2922 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2923 reset_tags[port] = slot_status[port] &
2924 AHCI_NCQ_SLOT_MASK(ahci_portp);
2925 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2926 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2927 port, reset_tags[port],
2928 ahci_portp->ahciport_pending_tags);
2929 }
2930 mutex_exit(&ahci_portp->ahciport_mutex);
2931 }
2932
2933 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
2934 ret = SATA_FAILURE;
2935 }
2936
2937 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2938 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2939 continue;
2940 }
2941
2942 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2943
2944 mutex_enter(&ahci_portp->ahciport_mutex);
2945 /*
2946 * To prevent recursive enter to ahci_mop_commands, we need
2947 * check AHCI_PORT_FLAG_MOPPING flag.
2948 */
2949 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2950 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2951 "ahci_reset_hba_reject_pkts: port %d is in "
2952 "mopping process, so return directly ", port);
2953 mutex_exit(&ahci_portp->ahciport_mutex);
2954 continue;
2955 }
2956
2957 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2958 ahci_portp->ahciport_mop_in_progress++;
2959
2960 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2961 finished_tags[port] =
2962 ahci_portp->ahciport_pending_tags &
2963 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
2964 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2965 finished_tags[port] =
2966 ahci_portp->ahciport_pending_ncq_tags &
2967 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
2968
2969 reset_tags[port] &= ~finished_tags[port];
2970
2971 ahci_mop_commands(ahci_ctlp,
2972 ahci_portp,
2973 slot_status[port],
2974 0, /* failed tags */
2975 0, /* timeout tags */
2976 0, /* aborted tags */
2977 reset_tags[port]); /* reset tags */
2978 mutex_exit(&ahci_portp->ahciport_mutex);
2979 }
2980 out:
2981 return (ret);
2982 }
2983
2984 /*
2985 * Called by sata framework to reset a port(s) or device.
2986 */
2987 static int
2988 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
2989 {
2990 ahci_ctl_t *ahci_ctlp;
2991 ahci_port_t *ahci_portp;
2992 ahci_addr_t addr;
2993 uint8_t cport = sd->satadev_addr.cport;
2994 uint8_t pmport = sd->satadev_addr.pmport;
2995 uint8_t port;
2996 int ret = SATA_SUCCESS;
2997 int instance = ddi_get_instance(dip);
2998
2999 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3000 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3001 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3002
3003 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3004
3005 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3006 "ahci_tran_reset_dport enter: cport %d", cport);
3007
3008 switch (sd->satadev_addr.qual) {
3009 case SATA_ADDR_PMPORT:
3010 /*
3011 * If we want to issue a COMRESET on a pmport, we need to
3012 * reject the outstanding commands on that pmport. According
3013 * to AHCI spec, PxCI register could only be cleared by
3014 * clearing PxCMD.ST, which will halt the controller port - as
3015 * well as other pmports.
3016 *
3017 * Therefore we directly reset the controller port for
3018 * simplicity. ahci_tran_probe_port() will handle reset stuff
3019 * like initializing the given pmport.
3020 */
3021 /* FALLTHRU */
3022 case SATA_ADDR_CPORT:
3023 /* Port reset */
3024 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3025 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3026 "port %d reset port", instance, port);
3027
3028 mutex_enter(&ahci_portp->ahciport_mutex);
3029 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3030 mutex_exit(&ahci_portp->ahciport_mutex);
3031
3032 break;
3033
3034 case SATA_ADDR_DPMPORT:
3035 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3036 "port %d:%d reset device", instance, port, pmport);
3037 /* FALLTHRU */
3038 case SATA_ADDR_DCPORT:
3039 /* Device reset */
3040 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3041 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3042 "port %d reset device", instance, port);
3043
3044 mutex_enter(&ahci_portp->ahciport_mutex);
3045 /*
3046 * software reset request must be sent to SATA_PMULT_HOSTPORT
3047 * if target is a port multiplier:
3048 */
3049 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3050 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3051 AHCI_ADDR_SET_PMULT(&addr, port);
3052
3053 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3054 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3055 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3056 /*
3057 * In case the targer driver would send the request
3058 * before sata framework can have the opportunity to
3059 * process those event reports.
3060 */
3061 sd->satadev_state = ahci_portp->ahciport_port_state;
3062 ahci_update_sata_registers(ahci_ctlp, port, sd);
3063 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3064 "ahci_tran_reset_dport returning SATA_FAILURE "
3065 "while port in FAILED/SHUTDOWN/PWROFF state: "
3066 "port: %d", port);
3067 mutex_exit(&ahci_portp->ahciport_mutex);
3068 ret = SATA_FAILURE;
3069 break;
3070 }
3071
3072 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3073 SATA_DTYPE_NONE) {
3074 /*
3075 * ahci_intr_phyrdy_change() may have rendered it to
3076 * AHCI_PORT_TYPE_NODEV.
3077 */
3078 sd->satadev_type = SATA_DTYPE_NONE;
3079 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3080 &addr);
3081 ahci_update_sata_registers(ahci_ctlp, port, sd);
3082 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3083 "ahci_tran_reset_dport returning SATA_FAILURE "
3084 "while no device attached: port: %d", port);
3085 mutex_exit(&ahci_portp->ahciport_mutex);
3086 ret = SATA_FAILURE;
3087 break;
3088 }
3089
3090 if (AHCI_ADDR_IS_PORT(&addr)) {
3091 ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3092 ahci_portp, &addr);
3093 } else {
3094 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3095 ahci_portp, &addr);
3096 }
3097
3098 mutex_exit(&ahci_portp->ahciport_mutex);
3099 break;
3100
3101 case SATA_ADDR_CNTRL:
3102 /* Reset the whole controller */
3103 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3104 "reset the whole hba", instance);
3105 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3106 break;
3107
3108 default:
3109 ret = SATA_FAILURE;
3110 }
3111
3112 return (ret);
3113 }
3114
3115 /*
3116 * Called by sata framework to activate a port as part of hotplug.
3117 * (cfgadm -c connect satax/y)
3118 * Support port multiplier.
3119 */
3120 static int
3121 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3122 {
3123 ahci_ctl_t *ahci_ctlp;
3124 ahci_port_t *ahci_portp;
3125 ahci_addr_t addr;
3126 uint8_t cport = satadev->satadev_addr.cport;
3127 uint8_t pmport = satadev->satadev_addr.pmport;
3128 uint8_t port;
3129 int instance = ddi_get_instance(dip);
3130
3131 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3132 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3133
3134 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3135 "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3136
3137 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3138
3139 mutex_enter(&ahci_portp->ahciport_mutex);
3140 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3141 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3142
3143 if (AHCI_ADDR_IS_PORT(&addr)) {
3144 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3145 instance, port);
3146
3147 /* Enable the interrupts on the port */
3148 ahci_enable_port_intrs(ahci_ctlp, port);
3149
3150 /*
3151 * Reset the port so that the PHY communication would be
3152 * re-established. But this reset is an internal operation
3153 * and the sata module doesn't need to know about it.
3154 * Moreover, the port with a device attached will be started
3155 * too.
3156 */
3157 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3158 ahci_portp, port,
3159 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3160 NULL);
3161
3162 /*
3163 * Need to check the link status and device status of the port
3164 * and consider raising power if the port was in D3 state
3165 */
3166 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3167 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3168 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3169 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3170 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3171 instance, port, pmport);
3172 /* AHCI_ADDR_PMPORT */
3173 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3174 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3175 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3176 }
3177
3178 satadev->satadev_state = ahci_portp->ahciport_port_state;
3179
3180 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3181
3182 mutex_exit(&ahci_portp->ahciport_mutex);
3183 return (SATA_SUCCESS);
3184 }
3185
3186 /*
3187 * Called by sata framework to deactivate a port as part of hotplug.
3188 * (cfgadm -c disconnect satax/y)
3189 * Support port multiplier.
3190 */
3191 static int
3192 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3193 {
3194 ahci_ctl_t *ahci_ctlp;
3195 ahci_port_t *ahci_portp;
3196 ahci_addr_t addr;
3197 uint8_t cport = satadev->satadev_addr.cport;
3198 uint8_t pmport = satadev->satadev_addr.pmport;
3199 uint8_t port;
3200 uint32_t port_scontrol;
3201 int instance = ddi_get_instance(dip);
3202
3203 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3204 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3205
3206 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3207 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3208
3209 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3210 mutex_enter(&ahci_portp->ahciport_mutex);
3211 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3212 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3213
3214 if (AHCI_ADDR_IS_PORT(&addr)) {
3215 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3216 instance, port);
3217
3218 /* Disable the interrupts on the port */
3219 ahci_disable_port_intrs(ahci_ctlp, port);
3220
3221 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3222
3223 /* First to abort all the pending commands */
3224 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3225
3226 /* Then stop the port */
3227 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3228 ahci_portp, port);
3229 }
3230
3231 /* Next put the PHY offline */
3232 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3233 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3234 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3235 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3236 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3237 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3238 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3239 instance, port, pmport);
3240
3241 ahci_disable_port_intrs(ahci_ctlp, port);
3242 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3243 != SATA_DTYPE_NONE)
3244 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3245
3246 /* Re-enable the interrupts for the other pmports */
3247 ahci_enable_port_intrs(ahci_ctlp, port);
3248 }
3249
3250 /* Update port state */
3251 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3252 satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3253
3254 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3255
3256 mutex_exit(&ahci_portp->ahciport_mutex);
3257 return (SATA_SUCCESS);
3258 }
3259
3260 /*
3261 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3262 * when a device is unplugged or a port is deactivated.
3263 */
3264 static void
3265 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3266 ahci_port_t *ahci_portp, uint8_t port)
3267 {
3268 uint32_t slot_status = 0;
3269 uint32_t abort_tags = 0;
3270
3271 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3272
3273 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3274 "ahci_reject_all_abort_pkts at port: %d", port);
3275
3276 /* Read/write port multiplier command takes highest priority */
3277 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3278 slot_status = 0x1;
3279 abort_tags = 0x1;
3280 goto out;
3281 }
3282
3283 /*
3284 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3285 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3286 * to get the error data, if yes when the device is removed, the
3287 * command needs to be aborted too.
3288 */
3289 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3290 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3291 slot_status = 0x1;
3292 abort_tags = 0x1;
3293 goto out;
3294 } else {
3295 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3296 "ahci_reject_all_abort_pkts return directly "
3297 "port %d no needs to reject any outstanding "
3298 "commands", port);
3299 return;
3300 }
3301 }
3302
3303 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3304 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3305 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3306 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3307 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3308 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3309 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3310 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3311 }
3312
3313 out:
3314 /* No need to do mop when there is no outstanding commands */
3315 if (slot_status != 0) {
3316 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3317 ahci_portp->ahciport_mop_in_progress++;
3318
3319 ahci_mop_commands(ahci_ctlp,
3320 ahci_portp,
3321 slot_status,
3322 0, /* failed tags */
3323 0, /* timeout tags */
3324 abort_tags, /* aborting tags */
3325 0); /* reset tags */
3326 }
3327 }
3328
3329 #if defined(__lock_lint)
3330 static int
3331 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3332 {
3333 return (SATA_SUCCESS);
3334 }
3335 #endif
3336
3337 /*
3338 * Initialize fma capabilities and register with IO fault services.
3339 */
3340 static void
3341 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3342 {
3343 /*
3344 * Need to change iblock to priority for new MSI intr
3345 */
3346 ddi_iblock_cookie_t fm_ibc;
3347
3348 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3349 ahci_ctlp->ahcictl_dip,
3350 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3351 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3352 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3353
3354 /* Only register with IO Fault Services if we have some capability */
3355 if (ahci_ctlp->ahcictl_fm_cap) {
3356 /* Adjust access and dma attributes for FMA */
3357 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3358 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3359 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3360 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3361 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3362
3363 /*
3364 * Register capabilities with IO Fault Services.
3365 * ahcictl_fm_cap will be updated to indicate
3366 * capabilities actually supported (not requested.)
3367 */
3368 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3369 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3370
3371 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3372 cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3373 ddi_get_instance(ahci_ctlp->ahcictl_dip));
3374 return;
3375 }
3376 /*
3377 * Initialize pci ereport capabilities if ereport
3378 * capable (should always be.)
3379 */
3380 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3381 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3382 pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3383 }
3384
3385 /*
3386 * Register error callback if error callback capable.
3387 */
3388 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3389 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3390 ahci_fm_error_cb, (void *) ahci_ctlp);
3391 }
3392
3393 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3394 "ahci_fm_fini: fma enabled.", NULL);
3395 }
3396 }
3397
3398 /*
3399 * Releases fma capabilities and un-registers with IO fault services.
3400 */
3401 static void
3402 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3403 {
3404 /* Only unregister FMA capabilities if registered */
3405 if (ahci_ctlp->ahcictl_fm_cap) {
3406 /*
3407 * Un-register error callback if error callback capable.
3408 */
3409 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3410 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3411 }
3412
3413 /*
3414 * Release any resources allocated by pci_ereport_setup()
3415 */
3416 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3417 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3418 pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3419 }
3420
3421 /* Unregister from IO Fault Services */
3422 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3423
3424 /* Adjust access and dma attributes for FMA */
3425 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3426 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3427 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3428 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3429 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3430
3431 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3432 "ahci_fm_fini: fma disabled.", NULL);
3433 }
3434 }
3435
3436 /*ARGSUSED*/
3437 static int
3438 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3439 {
3440 /*
3441 * as the driver can always deal with an error in any dma or
3442 * access handle, we can just return the fme_status value.
3443 */
3444 pci_ereport_post(dip, err, NULL);
3445 return (err->fme_status);
3446 }
3447
3448 int
3449 ahci_check_acc_handle(ddi_acc_handle_t handle)
3450 {
3451 ddi_fm_error_t de;
3452
3453 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3454 return (de.fme_status);
3455 }
3456
3457 int
3458 ahci_check_dma_handle(ddi_dma_handle_t handle)
3459 {
3460 ddi_fm_error_t de;
3461
3462 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3463 return (de.fme_status);
3464 }
3465
3466 /*
3467 * Generate an ereport
3468 */
3469 void
3470 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3471 {
3472 uint64_t ena;
3473 char buf[FM_MAX_CLASS];
3474
3475 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3476 ena = fm_ena_generate(0, FM_ENA_FMT1);
3477 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3478 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3479 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3480 FM_EREPORT_VERSION, NULL);
3481 }
3482 }
3483
3484 /*
3485 * Check if all handles are correctly allocated.
3486 */
3487 static int
3488 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3489 {
3490 int port;
3491
3492 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3493 return (DDI_FAILURE);
3494 }
3495
3496 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3497 ahci_port_t *ahci_portp;
3498
3499 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3500 continue;
3501
3502 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3503
3504 mutex_enter(&ahci_portp->ahciport_mutex);
3505 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3506 mutex_exit(&ahci_portp->ahciport_mutex);
3507 return (DDI_FAILURE);
3508 }
3509 mutex_exit(&ahci_portp->ahciport_mutex);
3510 }
3511
3512 return (DDI_SUCCESS);
3513 }
3514
3515 /*
3516 * Check the access handles for the controller. Note that
3517 * ahcictl_pci_conf_handle is only used in attach process.
3518 */
3519 static int
3520 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3521 {
3522 if ((ahci_check_acc_handle(ahci_ctlp->
3523 ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3524 (ahci_check_acc_handle(ahci_ctlp->
3525 ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3526 return (DDI_FAILURE);
3527 }
3528 return (DDI_SUCCESS);
3529 }
3530
3531 /*
3532 * Check the DMA handles and the access handles of a controller port.
3533 */
3534 static int
3535 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3536 {
3537 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3538 int slot;
3539
3540 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3541
3542 if ((ahci_check_dma_handle(ahci_portp->
3543 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3544 (ahci_check_dma_handle(ahci_portp->
3545 ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3546 (ahci_check_acc_handle(ahci_portp->
3547 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3548 (ahci_check_acc_handle(ahci_portp->
3549 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3550 return (DDI_FAILURE);
3551 }
3552 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3553 if (ahci_check_slot_handle(ahci_portp, slot)
3554 != DDI_SUCCESS) {
3555 return (DDI_FAILURE);
3556 }
3557 }
3558 return (DDI_SUCCESS);
3559 }
3560
3561 /*
3562 * Check the DMA handles and the access handles of a cmd table slot.
3563 */
3564 static int
3565 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3566 {
3567 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3568
3569 if ((ahci_check_acc_handle(ahci_portp->
3570 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3571 (ahci_check_dma_handle(ahci_portp->
3572 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3573 return (DDI_FAILURE);
3574 }
3575 return (DDI_SUCCESS);
3576 }
3577
3578 /*
3579 * Allocate the ports structure, only called by ahci_attach
3580 */
3581 static int
3582 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3583 {
3584 int port, cport = 0;
3585
3586 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3587 "ahci_alloc_ports_state enter", NULL);
3588
3589 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3590
3591 /* Allocate structures only for the implemented ports */
3592 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3593 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3594 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3595 "hba port %d not implemented", port);
3596 continue;
3597 }
3598
3599 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3600 ahci_ctlp->ahcictl_port_to_cport[port] =
3601 (uint8_t)cport++;
3602
3603 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3604 goto err_out;
3605 }
3606 }
3607
3608 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3609 return (AHCI_SUCCESS);
3610
3611 err_out:
3612 for (port--; port >= 0; port--) {
3613 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3614 ahci_dealloc_port_state(ahci_ctlp, port);
3615 }
3616 }
3617
3618 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3619 return (AHCI_FAILURE);
3620 }
3621
3622 /*
3623 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3624 */
3625 static void
3626 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3627 {
3628 int port;
3629
3630 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3631 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3632 /* if this port is implemented by the HBA */
3633 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3634 ahci_dealloc_port_state(ahci_ctlp, port);
3635 }
3636 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3637 }
3638
3639 /*
3640 * Drain the taskq.
3641 */
3642 static void
3643 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3644 {
3645 ahci_port_t *ahci_portp;
3646 int port;
3647
3648 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3649 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3650 continue;
3651 }
3652
3653 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3654
3655 mutex_enter(&ahci_portp->ahciport_mutex);
3656 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3657 mutex_exit(&ahci_portp->ahciport_mutex);
3658 }
3659 }
3660
3661 /*
3662 * Initialize the controller and all ports. And then try to start the ports
3663 * if there are devices attached.
3664 *
3665 * This routine can be called from three seperate cases: DDI_ATTACH,
3666 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3667 * other two cases; device signature probing are attempted only during
3668 * DDI_ATTACH case.
3669 */
3670 static int
3671 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3672 {
3673 ahci_port_t *ahci_portp;
3674 ahci_addr_t addr;
3675 int port;
3676
3677 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3678 "ahci_initialize_controller enter", NULL);
3679
3680 /* Disable the whole controller interrupts */
3681 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3682 ahci_disable_all_intrs(ahci_ctlp);
3683 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3684
3685 /* Initialize the implemented ports and structures */
3686 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3687 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3688 continue;
3689 }
3690
3691 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3692 mutex_enter(&ahci_portp->ahciport_mutex);
3693
3694 /*
3695 * Ensure that the controller is not in the running state
3696 * by checking every implemented port's PxCMD register
3697 */
3698 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3699
3700 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3701 != AHCI_SUCCESS) {
3702 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3703 "ahci_initialize_controller: failed to "
3704 "initialize port %d", port);
3705 /*
3706 * Set the port state to SATA_PSTATE_FAILED if
3707 * failed to initialize it.
3708 */
3709 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3710 }
3711
3712 mutex_exit(&ahci_portp->ahciport_mutex);
3713 }
3714
3715 /* Enable the whole controller interrupts */
3716 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3717 ahci_enable_all_intrs(ahci_ctlp);
3718 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3719
3720 return (AHCI_SUCCESS);
3721 }
3722
3723 /*
3724 * Reverse of ahci_initialize_controller()
3725 *
3726 * We only need to stop the ports and disable the interrupt.
3727 */
3728 static void
3729 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3730 {
3731 ahci_port_t *ahci_portp;
3732 int port;
3733
3734 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3735 "ahci_uninitialize_controller enter", NULL);
3736
3737 /* disable all the interrupts. */
3738 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3739 ahci_disable_all_intrs(ahci_ctlp);
3740 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3741
3742 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3743 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3744 continue;
3745 }
3746
3747 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3748
3749 /* Stop the port by clearing PxCMD.ST */
3750 mutex_enter(&ahci_portp->ahciport_mutex);
3751
3752 /*
3753 * Here we must disable the port interrupt because
3754 * ahci_disable_all_intrs only clear GHC.IE, and IS
3755 * register will be still set if PxIE is enabled.
3756 * When ahci shares one IRQ with other drivers, the
3757 * intr handler may claim the intr mistakenly.
3758 */
3759 ahci_disable_port_intrs(ahci_ctlp, port);
3760 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3761 ahci_portp, port);
3762 mutex_exit(&ahci_portp->ahciport_mutex);
3763 }
3764 }
3765
3766 /*
3767 * ahci_alloc_pmult()
3768 * 1. Setting HBA port registers which are necessary for a port multiplier.
3769 * (Set PxCMD.PMA while PxCMD.ST is '0')
3770 * 2. Allocate ahci_pmult_info structure.
3771 *
3772 * NOTE: Must stop port before the function is called.
3773 */
3774 static void
3775 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3776 {
3777 uint32_t port_cmd_status;
3778 uint8_t port = ahci_portp->ahciport_port_num;
3779
3780 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3781
3782 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3783 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3784
3785 /* The port must have been stopped before. */
3786 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3787
3788 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3789 /* set PMA bit */
3790 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3791 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3792 port_cmd_status|AHCI_CMD_STATUS_PMA);
3793
3794 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3795 "ahci_alloc_pmult: "
3796 "PxCMD.PMA bit set at port %d.", port);
3797 }
3798
3799 /* Allocate port multiplier information structure */
3800 if (ahci_portp->ahciport_pmult_info == NULL) {
3801 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3802 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3803 }
3804
3805 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3806 }
3807
3808 /*
3809 * ahci_dealloc_pmult()
3810 * 1. Clearing related registers when a port multiplier is detached.
3811 * (Clear PxCMD.PMA while PxCMD.ST is '0')
3812 * 2. Deallocate ahci_pmult_info structure.
3813 *
3814 * NOTE: Must stop port before the function is called.
3815 */
3816 static void
3817 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3818 {
3819 uint32_t port_cmd_status;
3820 uint8_t port = ahci_portp->ahciport_port_num;
3821
3822 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3823
3824 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3825 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3826
3827 if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3828 /* Clear PMA bit */
3829 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3830 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3831 (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3832
3833 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3834 "ahci_dealloc_pmult: "
3835 "PxCMD.PMA bit cleared at port %d.", port);
3836 }
3837
3838 /* Release port multiplier information structure */
3839 if (ahci_portp->ahciport_pmult_info != NULL) {
3840 kmem_free(ahci_portp->ahciport_pmult_info,
3841 sizeof (ahci_pmult_info_t));
3842 ahci_portp->ahciport_pmult_info = NULL;
3843 }
3844 }
3845
3846 /*
3847 * Staggered Spin-up.
3848 */
3849 static void
3850 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3851 {
3852 uint32_t cap_status;
3853 uint32_t port_cmd_status;
3854
3855 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3856
3857 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3858 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3859
3860 /* Check for staggered spin-up support */
3861 if (!(cap_status & AHCI_HBA_CAP_SSS))
3862 return;
3863
3864 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3865 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3866
3867 /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3868 if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3869 return;
3870
3871 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3872
3873 /* Set PxCMD.SUD */
3874 port_cmd_status |= AHCI_CMD_STATUS_SUD;
3875 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3876 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3877 port_cmd_status);
3878 }
3879
3880 /*
3881 * The routine is to initialize a port. First put the port in NotRunning
3882 * state, then enable port interrupt and clear Serror register. And under
3883 * AHCI_ATTACH case, find device signature and then try to start the port.
3884 *
3885 * Called by
3886 * 1. ahci_initialize_controller
3887 * 2. ahci_intr_phyrdy_change (hotplug)
3888 */
3889 static int
3890 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3891 ahci_addr_t *addrp)
3892 {
3893 uint32_t port_sstatus, port_task_file, port_cmd_status;
3894 uint8_t port = addrp->aa_port;
3895 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */
3896 int ret;
3897
3898 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3899
3900 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */
3901 ASSERT(AHCI_ADDR_IS_PORT(addrp));
3902
3903 /*
3904 * At the time being, only probe ports/devices and get the types of
3905 * attached devices during DDI_ATTACH. In fact, the device can be
3906 * changed during power state changes, but at the time being, we
3907 * don't support the situation.
3908 */
3909 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
3910 resuming = B_FALSE;
3911 } else {
3912 /* check for DDI_RESUME case */
3913 mutex_exit(&ahci_portp->ahciport_mutex);
3914 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3915 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
3916 resuming = B_FALSE;
3917 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3918 mutex_enter(&ahci_portp->ahciport_mutex);
3919 }
3920
3921 if (resuming) {
3922 /*
3923 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
3924 * and PxFBU registers in case these registers were cleared
3925 * during the suspend.
3926 */
3927 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
3928 "ahci_initialize_port: port %d "
3929 "set PxCLB, PxCLBU, PxFB and PxFBU "
3930 "during resume", port);
3931
3932 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
3933 AHCI_SUCCESS)
3934 return (AHCI_FAILURE);
3935 }
3936
3937 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3938 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3939
3940 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3941 "ahci_initialize_port: port %d ", port);
3942
3943 /*
3944 * Check whether the port is in NotRunning state, if not,
3945 * put the port in NotRunning state
3946 */
3947 if (port_cmd_status &
3948 (AHCI_CMD_STATUS_ST |
3949 AHCI_CMD_STATUS_CR |
3950 AHCI_CMD_STATUS_FRE |
3951 AHCI_CMD_STATUS_FR)) {
3952 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3953 ahci_portp, port);
3954 }
3955
3956 /* Make sure the drive is spun-up */
3957 ahci_staggered_spin_up(ahci_ctlp, port);
3958
3959 /* Disable interrupt */
3960 ahci_disable_port_intrs(ahci_ctlp, port);
3961
3962 /* Device is unknown at first */
3963 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
3964
3965 /* Disable the interface power management */
3966 ahci_disable_interface_pm(ahci_ctlp, port);
3967
3968 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3969 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
3970 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3971 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
3972
3973 /* Check physcial link status */
3974 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
3975 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
3976
3977 /* Check interface status */
3978 port_task_file & AHCI_TFD_STS_BSY ||
3979 port_task_file & AHCI_TFD_STS_DRQ ||
3980
3981 /* Check whether port reset must be executed */
3982 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
3983
3984 /* Always reset port on RESUME */
3985 resuming != B_FALSE) {
3986
3987 /* Something went wrong, we need do some reset things */
3988 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
3989
3990 /* Does port reset succeed on HBA port? */
3991 if (ret != AHCI_SUCCESS) {
3992 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
3993 "ahci_initialize_port:"
3994 "port reset failed at port %d", port);
3995 return (AHCI_FAILURE);
3996 }
3997
3998 /* Is port failed? */
3999 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4000 SATA_PSTATE_FAILED) {
4001 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4002 "ahci_initialize_port: port %d state 0x%x",
4003 port, ahci_portp->ahciport_port_state);
4004 return (AHCI_FAILURE);
4005 }
4006 }
4007
4008 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4009 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4010
4011 /*
4012 * Try to get the device signature if the port is not empty.
4013 */
4014 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4015 SATA_DTYPE_NONE)
4016 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4017
4018 /* Return directly if no device connected */
4019 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4020 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4021 "No device connected to port %d", port);
4022 goto out;
4023 }
4024
4025 /* If this is a port multiplier, we need do some initialization */
4026 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4027 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4028 "Port multiplier found at port %d", port);
4029 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4030 }
4031
4032 /* Try to start the port */
4033 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4034 != AHCI_SUCCESS) {
4035 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4036 "failed to start port %d", port);
4037 return (AHCI_FAILURE);
4038 }
4039 out:
4040 /* Enable port interrupts */
4041 ahci_enable_port_intrs(ahci_ctlp, port);
4042
4043 return (AHCI_SUCCESS);
4044 }
4045
4046 /*
4047 * Handle hardware defect, and check the capabilities. For example,
4048 * power management capabilty and MSI capability.
4049 */
4050 static int
4051 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4052 {
4053 ushort_t caps_ptr, cap_count, cap;
4054 #if AHCI_DEBUG
4055 ushort_t pmcap, pmcsr;
4056 ushort_t msimc;
4057 #endif
4058 uint8_t revision;
4059
4060 ahci_ctlp->ahcictl_venid =
4061 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4062 PCI_CONF_VENID);
4063
4064 ahci_ctlp->ahcictl_devid =
4065 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4066 PCI_CONF_DEVID);
4067
4068 /*
4069 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4070 * controllers with 0x00 revision id work on 4-byte aligned buffer,
4071 * which is a bug and was fixed after 0x00 revision id controllers.
4072 *
4073 * Moreover, VT8251 cannot use multiple command slots in the command
4074 * list for non-queued commands because the previous register content
4075 * of PxCI can be re-written in the register write, so a flag will be
4076 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4077 *
4078 * For VT8251, software reset also has the same defect as the below
4079 * AMD/ATI chipset. That is, software reset will get failed if 0xf
4080 * is filled in pmport field. Therefore, another software reset need
4081 * to be done with 0 filled in pmport field.
4082 */
4083 if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4084 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4085 PCI_CONF_REVID);
4086 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4087 "revision id = 0x%x", revision);
4088 if (revision == 0x00) {
4089 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4090 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4091 "change ddi_attr_align to 0x4", NULL);
4092 }
4093
4094 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4095 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4096 "VT8251 cannot use multiple command lists for "
4097 "non-queued commands", NULL);
4098
4099 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4100 }
4101
4102 /*
4103 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4104 * DMA addressing for communication memory descriptors though S64A bit
4105 * of CAP register declares it supports. Even though 64-bit DMA for
4106 * data buffer works on ASUS M2A-VM with newer BIOS, three other
4107 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4108 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4109 *
4110 * Due to certain hardware issue, the chipset must do port reset during
4111 * initialization, otherwise, when retrieving device signature,
4112 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4113 * need to set.
4114 *
4115 * For this chipset software reset will get failure if the pmport of
4116 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4117 * multiplier is connected to the port. In order to fix the issue,
4118 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4119 * reset got failure, the driver will try to do another software reset
4120 * with pmport 0.
4121 */
4122 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4123 ahci_ctlp->ahcictl_devid == 0x4380) {
4124 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4125 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4126 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4127 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4128
4129 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4130 "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4131 "communication memory descriptors though CAP indicates "
4132 "support, so force it to use 32-bit DMA", NULL);
4133 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4134 "ATI SB600 need to do a port reset during initialization",
4135 NULL);
4136 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4137 "ATI SB600 will get software reset failure if pmport "
4138 "is set 0xf and no port multiplier is attached", NULL);
4139 }
4140
4141 /*
4142 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4143 * vendor ID and device ID (0x1002,0x4391).
4144 *
4145 * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4146 * DMA addressing for communication memory descriptors though S64A bit
4147 * of CAP register declares the support. However, it does support
4148 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4149 * set for this controller.
4150 *
4151 * SB710 has the same initialization issue as SB600, so it also need
4152 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4153 *
4154 * SB700 also has the same issue about software reset, and thus
4155 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4156 */
4157 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4158 ahci_ctlp->ahcictl_devid == 0x4391) {
4159 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4160 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4161 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4162
4163 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4164 "ATI SB700/750 cannot do 64-bit DMA for communication "
4165 "memory descriptors though CAP indicates support, "
4166 "so force it to use 32-bit DMA", NULL);
4167 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4168 "ATI SB710 need to do a port reset during initialization",
4169 NULL);
4170 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4171 "ATI SB700 will get software reset failure if pmport "
4172 "is set 0xf and no port multiplier is attached", NULL);
4173 }
4174
4175 /*
4176 * Check if capabilities list is supported and if so,
4177 * get initial capabilities pointer and clear bits 0,1.
4178 */
4179 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4180 PCI_CONF_STAT) & PCI_STAT_CAP) {
4181 caps_ptr = P2ALIGN(pci_config_get8(
4182 ahci_ctlp->ahcictl_pci_conf_handle,
4183 PCI_CONF_CAP_PTR), 4);
4184 } else {
4185 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4186 }
4187
4188 /*
4189 * Walk capabilities if supported.
4190 */
4191 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4192
4193 /*
4194 * Check that we haven't exceeded the maximum number of
4195 * capabilities and that the pointer is in a valid range.
4196 */
4197 if (++cap_count > PCI_CAP_MAX_PTR) {
4198 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4199 "too many device capabilities", NULL);
4200 return (AHCI_FAILURE);
4201 }
4202 if (caps_ptr < PCI_CAP_PTR_OFF) {
4203 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4204 "capabilities pointer 0x%x out of range",
4205 caps_ptr);
4206 return (AHCI_FAILURE);
4207 }
4208
4209 /*
4210 * Get next capability and check that it is valid.
4211 * For now, we only support power management.
4212 */
4213 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4214 caps_ptr);
4215 switch (cap) {
4216 case PCI_CAP_ID_PM:
4217
4218 /* power management supported */
4219 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4220
4221 /* Save PMCSR offset */
4222 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4223
4224 #if AHCI_DEBUG
4225 pmcap = pci_config_get16(
4226 ahci_ctlp->ahcictl_pci_conf_handle,
4227 caps_ptr + PCI_PMCAP);
4228 pmcsr = pci_config_get16(
4229 ahci_ctlp->ahcictl_pci_conf_handle,
4230 ahci_ctlp->ahcictl_pmcsr_offset);
4231 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4232 "Power Management capability found PCI_PMCAP "
4233 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4234 if ((pmcap & 0x3) == 0x3)
4235 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4236 "PCI Power Management Interface "
4237 "spec 1.2 compliant", NULL);
4238 #endif
4239 break;
4240
4241 case PCI_CAP_ID_MSI:
4242 #if AHCI_DEBUG
4243 msimc = pci_config_get16(
4244 ahci_ctlp->ahcictl_pci_conf_handle,
4245 caps_ptr + PCI_MSI_CTRL);
4246 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4247 "Message Signaled Interrupt capability found "
4248 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4249 #endif
4250 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4251 "MSI capability found", NULL);
4252 break;
4253
4254 case PCI_CAP_ID_PCIX:
4255 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4256 "PCI-X capability found", NULL);
4257 break;
4258
4259 case PCI_CAP_ID_PCI_E:
4260 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4261 "PCI Express capability found", NULL);
4262 break;
4263
4264 case PCI_CAP_ID_MSI_X:
4265 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4266 "MSI-X capability found", NULL);
4267 break;
4268
4269 case PCI_CAP_ID_SATA:
4270 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4271 "SATA capability found", NULL);
4272 break;
4273
4274 case PCI_CAP_ID_VS:
4275 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4276 "Vendor Specific capability found", NULL);
4277 break;
4278
4279 default:
4280 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4281 "unrecognized capability 0x%x", cap);
4282 break;
4283 }
4284
4285 /*
4286 * Get next capabilities pointer and clear bits 0,1.
4287 */
4288 caps_ptr = P2ALIGN(pci_config_get8(
4289 ahci_ctlp->ahcictl_pci_conf_handle,
4290 (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4291 }
4292
4293 return (AHCI_SUCCESS);
4294 }
4295
4296 /*
4297 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4298 * PORTMULT command. SYNC & POLLING mode is used.
4299 */
4300 static int
4301 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4302 uint8_t regn, uint32_t *pregv, uint8_t type)
4303 {
4304 ahci_port_t *ahci_portp;
4305 ahci_addr_t pmult_addr;
4306 sata_pkt_t *spkt;
4307 sata_cmd_t *scmd;
4308 sata_device_t sata_device;
4309 uint8_t port = addrp->aa_port;
4310 uint8_t pmport = addrp->aa_pmport;
4311 uint8_t cport;
4312 uint32_t intr_mask;
4313 int rval;
4314 char portstr[10];
4315
4316 SET_PORTSTR(portstr, addrp);
4317 cport = ahci_ctlp->ahcictl_port_to_cport[port];
4318 ahci_portp = ahci_ctlp->ahcictl_ports[port];
4319
4320 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4321 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4322
4323 /* Check the existence of the port multiplier */
4324 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4325 return (AHCI_FAILURE);
4326
4327 /* Request a READ/WRITE PORTMULT sata packet. */
4328 bzero(&sata_device, sizeof (sata_device_t));
4329 sata_device.satadev_addr.cport = cport;
4330 sata_device.satadev_addr.pmport = pmport;
4331 sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4332 sata_device.satadev_rev = SATA_DEVICE_REV;
4333
4334 /*
4335 * Make sure no command is outstanding here. All R/W PMULT requests
4336 * come from
4337 *
4338 * 1. ahci_attach()
4339 * The port should be empty.
4340 *
4341 * 2. ahci_tran_probe_port()
4342 * Any request from SATA framework (via ahci_tran_start) should be
4343 * rejected if R/W PMULT command is outstanding.
4344 *
4345 * If we are doing mopping, do not check those flags because no
4346 * command will be actually outstanding.
4347 *
4348 * If the port has been occupied by any other commands, the probe
4349 * function will return a SATA_RETRY. SATA framework will retry
4350 * later.
4351 */
4352 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4353 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4354 "R/W PMULT failed: R/W PMULT in progress at port %d.",
4355 port, ahci_portp->ahciport_flags);
4356 return (AHCI_FAILURE);
4357 }
4358
4359 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4360 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4361 NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4362 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4363 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4364 "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4365 port, ahci_portp->ahciport_flags);
4366 return (AHCI_FAILURE);
4367 }
4368
4369 /*
4370 * The port multiplier is gone. This may happen when
4371 * 1. Cutting off the power of an enclosure. The device lose the power
4372 * before port multiplier.
4373 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4374 *
4375 * The issued command should be aborted and the following command
4376 * should not be continued.
4377 */
4378 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4379 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4380 "READ/WRITE PMULT failed: "
4381 "port-mult is removed from port %d", port);
4382 return (AHCI_FAILURE);
4383 }
4384
4385 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4386
4387 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4388 &sata_device, regn, *pregv, type);
4389
4390 /*
4391 * READ/WRITE PORTMULT command is intended to sent to the control port
4392 * of the port multiplier.
4393 */
4394 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4395
4396 ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4397
4398 /* No interrupt here. Store the interrupt enable mask. */
4399 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4400 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4401 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4402 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4403
4404 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4405
4406 if (rval == AHCI_SUCCESS &&
4407 spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4408 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4409 scmd = &spkt->satapkt_cmd;
4410 *pregv = scmd->satacmd_lba_high_lsb << 24 |
4411 scmd->satacmd_lba_mid_lsb << 16 |
4412 scmd->satacmd_lba_low_lsb << 8 |
4413 scmd->satacmd_sec_count_lsb;
4414 }
4415 } else {
4416 /* Failed or not completed. */
4417 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4418 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4419 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4420 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4421 rval = AHCI_FAILURE;
4422 }
4423 out:
4424 /* Restore the interrupt mask */
4425 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4426 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4427
4428 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4429 ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4430 sata_free_rdwr_pmult_pkt(spkt);
4431 return (rval);
4432 }
4433
4434 static int
4435 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4436 uint8_t regn, uint32_t *pregv)
4437 {
4438 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4439 SATA_RDWR_PMULT_PKT_TYPE_READ);
4440 }
4441
4442 static int
4443 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4444 uint8_t regn, uint32_t regv)
4445 {
4446 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v,
4447 SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4448 }
4449
4450 #define READ_PMULT(addrp, r, pv, out) \
4451 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \
4452 goto out;
4453
4454 #define WRITE_PMULT(addrp, r, v, out) \
4455 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \
4456 goto out;
4457
4458 /*
4459 * Update sata registers on port multiplier, including GSCR/PSCR registers.
4460 * ahci_update_pmult_gscr()
4461 * ahci_update_pmult_pscr()
4462 */
4463 static int
4464 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4465 sata_pmult_gscr_t *sg)
4466 {
4467 ASSERT(MUTEX_HELD(
4468 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4469
4470 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4471 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4472 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4473 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4474
4475 return (AHCI_SUCCESS);
4476
4477 err: /* R/W PMULT error */
4478 return (AHCI_FAILURE);
4479 }
4480
4481 static int
4482 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4483 sata_device_t *sd)
4484 {
4485 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4486 ASSERT(MUTEX_HELD(
4487 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4488
4489 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4490 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4491 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4492 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4493
4494 return (AHCI_SUCCESS);
4495
4496 err: /* R/W PMULT error */
4497 return (AHCI_FAILURE);
4498 }
4499
4500 /*
4501 * ahci_initialize_pmult()
4502 *
4503 * Initialize a port multiplier, including
4504 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4505 * 2. Redefine MASK register. (SATA Chap 16.?)
4506 */
4507 static int
4508 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4509 ahci_addr_t *addrp, sata_device_t *sd)
4510 {
4511 sata_pmult_gscr_t sg;
4512 uint32_t gscr64;
4513 uint8_t port = addrp->aa_port;
4514
4515 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4516
4517 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4518 "[Initialize] Port-multiplier at port %d.", port);
4519
4520 /*
4521 * Enable features of port multiplier. Currently only
4522 * Asynchronous Notification is enabled.
4523 */
4524 /* Check gscr64 for supported features. */
4525 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4526
4527 if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4528 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4529 "port %d: Port Multiplier supports "
4530 "Asynchronous Notification.", port);
4531
4532 /* Write to gscr96 to enabled features */
4533 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4534 SATA_PMULT_CAP_SNOTIF, err);
4535
4536 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4537 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4538 AHCI_SNOTIF_CLEAR_ALL);
4539 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4540 "port %d: PMult PxSNTF cleared.", port);
4541
4542 }
4543
4544 /*
4545 * Now we need to update gscr33 register to enable hot-plug interrupt
4546 * for sub devices behind port multiplier.
4547 */
4548 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4549 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4550 "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4551
4552 /*
4553 * Fetch the number of device ports of the port multiplier
4554 */
4555 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4556 return (AHCI_FAILURE);
4557
4558 /* Register the port multiplier to SATA Framework. */
4559 mutex_exit(&ahci_portp->ahciport_mutex);
4560 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4561 mutex_enter(&ahci_portp->ahciport_mutex);
4562
4563 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4564 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4565
4566 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4567 "port %d: pmult sub-port number updated to %x.", port,
4568 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4569
4570 /* Till now port-mult is successfully initialized */
4571 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4572 return (AHCI_SUCCESS);
4573
4574 err: /* R/W PMULT error */
4575 return (AHCI_FAILURE);
4576 }
4577
4578 /*
4579 * Initialize a port multiplier port. According to spec, firstly we need
4580 * issue a COMRESET, then a software reset to get its signature.
4581 *
4582 * NOTE: This function should only be called in ahci_probe_pmport()
4583 */
4584 static int
4585 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4586 ahci_addr_t *addrp)
4587 {
4588 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4589 uint8_t port = addrp->aa_port;
4590 uint8_t pmport = addrp->aa_pmport;
4591 int ret = AHCI_FAILURE;
4592
4593 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4594 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4595
4596 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4597 "ahci_initialize_pmport: port %d:%d", port, pmport);
4598
4599 /* Check HBA port state */
4600 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4601 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4602 "ahci_initialize_pmport:"
4603 "port %d:%d Port Multiplier is failed.",
4604 port, pmport);
4605 return (AHCI_FAILURE);
4606 }
4607
4608 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4609 return (AHCI_FAILURE);
4610 }
4611 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4612
4613 /* Checking for outstanding commands */
4614 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4615 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4616 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4617 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4618 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4619 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4620 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4621 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4622 }
4623
4624 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4625 ahci_portp->ahciport_mop_in_progress++;
4626
4627 /* Clear status */
4628 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4629
4630 /* Firstly assume an unknown device */
4631 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4632
4633 ahci_disable_port_intrs(ahci_ctlp, port);
4634
4635 /* port reset is necessary for port multiplier port */
4636 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4637 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4638 "ahci_initialize_pmport:"
4639 "port reset failed at port %d:%d",
4640 port, pmport);
4641 goto out;
4642 }
4643
4644 /* Is port failed? */
4645 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4646 SATA_PSTATE_FAILED) {
4647 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4648 "ahci_initialize_pmport: port %d:%d failed. "
4649 "state = 0x%x", port, pmport,
4650 ahci_portp->ahciport_port_state);
4651 goto out;
4652 }
4653
4654 /* Is there any device attached? */
4655 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4656 == SATA_DTYPE_NONE) {
4657 /* Do not waste time on an empty port */
4658 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4659 "ahci_initialize_pmport: No device is found "
4660 "at port %d:%d", port, pmport);
4661 ret = AHCI_SUCCESS;
4662 goto out;
4663 }
4664
4665 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4666 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4667 "port %d:%d is ready now.", port, pmport);
4668
4669 /*
4670 * Till now we can assure a device attached to that HBA port and work
4671 * correctly. Now try to get the device signature. This is an optional
4672 * step. If failed, unknown device is assumed, then SATA module will
4673 * continue to use IDENTIFY DEVICE to get the information of the
4674 * device.
4675 */
4676 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4677
4678 ret = AHCI_SUCCESS;
4679
4680 out:
4681 /* Next try to mop the pending commands */
4682 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4683 finished_tags = ahci_portp->ahciport_pending_tags &
4684 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4685 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4686 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4687 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4688 reset_tags &= ~finished_tags;
4689
4690 ahci_mop_commands(ahci_ctlp,
4691 ahci_portp,
4692 slot_status,
4693 0, /* failed tags */
4694 0, /* timeout tags */
4695 0, /* aborted tags */
4696 reset_tags); /* reset tags */
4697
4698 /* Clear PxSNTF register if supported. */
4699 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4700 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4701 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4702 AHCI_SNOTIF_CLEAR_ALL);
4703 }
4704
4705 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4706 ahci_enable_port_intrs(ahci_ctlp, port);
4707 return (ret);
4708 }
4709
4710 /*
4711 * ahci_probe_pmult()
4712 *
4713 * This function will be called to probe a port multiplier, which will
4714 * handle hotplug events on port multiplier ports.
4715 *
4716 * NOTE: Only called from ahci_tran_probe_port()
4717 */
4718 static int
4719 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4720 ahci_addr_t *addrp)
4721 {
4722 sata_device_t sdevice;
4723 ahci_addr_t pmport_addr;
4724 uint32_t gscr32, port_hotplug_tags;
4725 uint32_t pmport_sstatus;
4726 int dev_exists_now = 0, dev_existed_previously = 0;
4727 uint8_t port = addrp->aa_port;
4728 int npmport;
4729
4730 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4731
4732 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4733 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4734 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4735
4736 do {
4737 npmport = ddi_ffs(port_hotplug_tags) - 1;
4738 if (npmport == -1)
4739 /* no pending hot plug events. */
4740 return (AHCI_SUCCESS);
4741
4742 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4743 "hot-plug event at port %d:%d", port, npmport);
4744
4745 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4746
4747 /* Check previous device at that port */
4748 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4749 != SATA_DTYPE_NONE)
4750 dev_existed_previously = 1;
4751
4752 /* PxSStatus tells the presence of device. */
4753 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4754 &pmport_sstatus, err);
4755
4756 if (SSTATUS_GET_DET(pmport_sstatus) ==
4757 SSTATUS_DET_DEVPRE_PHYCOM)
4758 dev_exists_now = 1;
4759
4760 /*
4761 * Clear PxSERR is critical. The transition from 0 to 1 will
4762 * emit a FIS which generates an asynchronous notification
4763 * event at controller. If we fail to clear the PxSERR, the
4764 * Async Notif events will no longer be activated on this
4765 * pmport.
4766 */
4767 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4768 AHCI_SERROR_CLEAR_ALL, err);
4769
4770 bzero((void *)&sdevice, sizeof (sata_device_t));
4771 sdevice.satadev_addr.cport = ahci_ctlp->
4772 ahcictl_port_to_cport[port];
4773 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4774 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4775 sdevice.satadev_state = SATA_PSTATE_PWRON;
4776
4777 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4778 "[Existence] %d -> %d", dev_existed_previously,
4779 dev_exists_now);
4780
4781 if (dev_exists_now) {
4782 if (dev_existed_previously) {
4783 /* Link (may) not change: Exist -> Exist * */
4784 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4785 "ahci_probe_pmult: port %d:%d "
4786 "device link lost/established",
4787 port, npmport);
4788
4789 mutex_exit(&ahci_portp->ahciport_mutex);
4790 sata_hba_event_notify(
4791 ahci_ctlp->ahcictl_sata_hba_tran->
4792 sata_tran_hba_dip,
4793 &sdevice,
4794 SATA_EVNT_LINK_LOST|
4795 SATA_EVNT_LINK_ESTABLISHED);
4796 mutex_enter(&ahci_portp->ahciport_mutex);
4797 } else {
4798 /* Link change: None -> Exist */
4799 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4800 "ahci_probe_pmult: port %d:%d "
4801 "device link established", port, npmport);
4802
4803 /* Clear port state */
4804 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4805 SATA_STATE_UNKNOWN);
4806 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4807 "ahci_probe_pmult: port %d "
4808 "ahciport_port_state [Cleared].", port);
4809
4810 mutex_exit(&ahci_portp->ahciport_mutex);
4811 sata_hba_event_notify(
4812 ahci_ctlp->ahcictl_sata_hba_tran->
4813 sata_tran_hba_dip,
4814 &sdevice,
4815 SATA_EVNT_LINK_ESTABLISHED);
4816 mutex_enter(&ahci_portp->ahciport_mutex);
4817 }
4818 } else { /* No device exists now */
4819 if (dev_existed_previously) {
4820
4821 /* Link change: Exist -> None */
4822 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4823 "ahci_probe_pmult: port %d:%d "
4824 "device link lost", port, npmport);
4825
4826 /* An existing device is lost. */
4827 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4828 SATA_STATE_UNKNOWN);
4829 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4830 SATA_DTYPE_NONE);
4831
4832 mutex_exit(&ahci_portp->ahciport_mutex);
4833 sata_hba_event_notify(
4834 ahci_ctlp->ahcictl_sata_hba_tran->
4835 sata_tran_hba_dip,
4836 &sdevice,
4837 SATA_EVNT_LINK_LOST);
4838 mutex_enter(&ahci_portp->ahciport_mutex);
4839 }
4840 }
4841
4842 CLEAR_BIT(port_hotplug_tags, npmport);
4843 } while (port_hotplug_tags != 0);
4844
4845 return (AHCI_SUCCESS);
4846
4847 err: /* R/W PMULT error */
4848 return (AHCI_FAILURE);
4849 }
4850
4851 /*
4852 * Probe and initialize a port multiplier port.
4853 * A port multiplier port could only be initilaizer here.
4854 */
4855 static int
4856 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4857 ahci_addr_t *addrp, sata_device_t *sd)
4858 {
4859 uint32_t port_state;
4860 uint8_t port = addrp->aa_port;
4861 ahci_addr_t addr_pmult;
4862
4863 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4864
4865 /*
4866 * Check the parent - port multiplier first.
4867 */
4868
4869 /*
4870 * Parent port multiplier might have been removed. This event will be
4871 * ignored and failure.
4872 */
4873 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4874 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4875 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4876 "ahci_tran_probe_port: "
4877 "parent device removed, ignore event.", NULL);
4878
4879 return (AHCI_FAILURE);
4880 }
4881
4882 /* The port is ready? */
4883 port_state = ahci_portp->ahciport_port_state;
4884 if (!(port_state & SATA_STATE_READY)) {
4885 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4886 "ahci_tran_probe_port: "
4887 "parent port-mult is NOT ready.", NULL);
4888
4889 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4890 ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4891 AHCI_SUCCESS) {
4892 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4893 "ahci_tran_probe_port: "
4894 "restart port-mult failed.", NULL);
4895 return (AHCI_FAILURE);
4896 }
4897 }
4898
4899 /*
4900 * If port-mult is restarted due to some reason, we need
4901 * re-initialized the PMult.
4902 */
4903 if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
4904 /* Initialize registers on a port multiplier */
4905 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
4906 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
4907 &addr_pmult, sd) != AHCI_SUCCESS)
4908 return (AHCI_FAILURE);
4909 }
4910
4911 /*
4912 * Then we check the port-mult port
4913 */
4914 /* Is this pmport initialized? */
4915 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
4916 if (!(port_state & SATA_STATE_READY)) {
4917
4918 /* ahci_initialize_pmport() will set READY state */
4919 if (ahci_initialize_pmport(ahci_ctlp,
4920 ahci_portp, addrp) != AHCI_SUCCESS)
4921 return (AHCI_FAILURE);
4922 }
4923
4924 return (AHCI_SUCCESS);
4925 }
4926
4927 /*
4928 * AHCI device reset ...; a single device on one of the ports is reset,
4929 * but the HBA and physical communication remain intact. This is the
4930 * least intrusive.
4931 *
4932 * When issuing a software reset sequence, there should not be other
4933 * commands in the command list, so we will first clear and then re-set
4934 * PxCMD.ST to clear PxCI. And before issuing the software reset,
4935 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
4936 * cleared unless command list override (PxCMD.CLO) is supported.
4937 */
4938 static int
4939 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4940 ahci_addr_t *addrp)
4941 {
4942 ahci_fis_h2d_register_t *h2d_register_fisp;
4943 ahci_cmd_table_t *cmd_table;
4944 ahci_cmd_header_t *cmd_header;
4945 uint32_t port_cmd_status, port_cmd_issue, port_task_file;
4946 int slot, loop_count;
4947 uint8_t port = addrp->aa_port;
4948 uint8_t pmport = addrp->aa_pmport;
4949 int rval = AHCI_FAILURE;
4950
4951 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4952
4953 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
4954 "port %d:%d device software resetting (FIS)", port, pmport);
4955
4956 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
4957 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
4958 port) != AHCI_SUCCESS) {
4959 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4960 "ahci_software_reset: cannot stop HBA port %d.", port);
4961 goto out;
4962 }
4963
4964 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
4965 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4966 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4967
4968 if (port_task_file & AHCI_TFD_STS_BSY ||
4969 port_task_file & AHCI_TFD_STS_DRQ) {
4970 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
4971 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4972 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
4973 "cannot issue a software reset.", port_task_file);
4974 goto out;
4975 }
4976
4977 /*
4978 * If HBA Support CLO, as Command List Override (CAP.SCLO is
4979 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
4980 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
4981 */
4982 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4983 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
4984
4985 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4986 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4987 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4988 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
4989 port_cmd_status|AHCI_CMD_STATUS_CLO);
4990
4991 /* Waiting till PxCMD.SCLO bit is cleared */
4992 loop_count = 0;
4993 do {
4994 /* Wait for 10 millisec */
4995 drv_usecwait(AHCI_10MS_USECS);
4996
4997 /* We are effectively timing out after 1 sec. */
4998 if (loop_count++ > 100) {
4999 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5000 "SCLO time out. port %d is busy.", port);
5001 goto out;
5002 }
5003
5004 port_cmd_status =
5005 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5006 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5007 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5008
5009 /* Re-check */
5010 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5011 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5012 if (port_task_file & AHCI_TFD_STS_BSY ||
5013 port_task_file & AHCI_TFD_STS_DRQ) {
5014 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5015 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5016 port_task_file);
5017 goto out;
5018 }
5019 }
5020
5021 /* Then start port */
5022 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5023 != AHCI_SUCCESS) {
5024 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5025 "ahci_software_reset: cannot start AHCI port %d.", port);
5026 goto out;
5027 }
5028
5029 /*
5030 * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5031 * ahci_port.ahciport_pending_ncq_tags may fail
5032 * ahci_claim_free_slot(). Actually according to spec, by clearing
5033 * PxCMD.ST there is no command outstanding while executing software
5034 * reseting. Hence we directly use slot 0 instead of
5035 * ahci_claim_free_slot().
5036 */
5037 slot = 0;
5038
5039 /* Now send the first H2D Register FIS with SRST set to 1 */
5040 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5041 bzero((void *)cmd_table, ahci_cmd_table_size);
5042
5043 h2d_register_fisp =
5044 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5045
5046 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5047 SET_FIS_PMP(h2d_register_fisp, pmport);
5048 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5049
5050 /* Set Command Header in Command List */
5051 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5052 BZERO_DESCR_INFO(cmd_header);
5053 BZERO_PRD_BYTE_COUNT(cmd_header);
5054 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5055 SET_PORT_MULTI_PORT(cmd_header, pmport);
5056
5057 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5058 SET_RESET(cmd_header, 1);
5059 SET_WRITE(cmd_header, 1);
5060
5061 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5062 0,
5063 ahci_cmd_table_size,
5064 DDI_DMA_SYNC_FORDEV);
5065
5066 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5067 slot * sizeof (ahci_cmd_header_t),
5068 sizeof (ahci_cmd_header_t),
5069 DDI_DMA_SYNC_FORDEV);
5070
5071 /* Indicate to the HBA that a command is active. */
5072 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5073 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5074 (0x1 << slot));
5075
5076 loop_count = 0;
5077
5078 /* Loop till the first command is finished */
5079 do {
5080 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5081 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5082
5083 /* We are effectively timing out after 1 sec. */
5084 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5085 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5086 "the first SRST FIS is timed out, "
5087 "loop_count = %d", loop_count);
5088 goto out;
5089 }
5090 /* Wait for 10 millisec */
5091 drv_usecwait(AHCI_10MS_USECS);
5092 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5093
5094 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5095 "ahci_software_reset: 1st loop count: %d, "
5096 "port_cmd_issue = 0x%x, slot = 0x%x",
5097 loop_count, port_cmd_issue, slot);
5098
5099 /* According to ATA spec, we need wait at least 5 microsecs here. */
5100 drv_usecwait(AHCI_1MS_USECS);
5101
5102 /* Now send the second H2D Register FIS with SRST cleard to zero */
5103 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5104 bzero((void *)cmd_table, ahci_cmd_table_size);
5105
5106 h2d_register_fisp =
5107 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5108
5109 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5110 SET_FIS_PMP(h2d_register_fisp, pmport);
5111
5112 /* Set Command Header in Command List */
5113 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5114 BZERO_DESCR_INFO(cmd_header);
5115 BZERO_PRD_BYTE_COUNT(cmd_header);
5116 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5117 SET_PORT_MULTI_PORT(cmd_header, pmport);
5118
5119 SET_WRITE(cmd_header, 1);
5120
5121 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5122 0,
5123 ahci_cmd_table_size,
5124 DDI_DMA_SYNC_FORDEV);
5125
5126 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5127 slot * sizeof (ahci_cmd_header_t),
5128 sizeof (ahci_cmd_header_t),
5129 DDI_DMA_SYNC_FORDEV);
5130
5131 /* Indicate to the HBA that a command is active. */
5132 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5133 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5134 (0x1 << slot));
5135
5136 loop_count = 0;
5137
5138 /* Loop till the second command is finished */
5139 do {
5140 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5141 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5142
5143 /* We are effectively timing out after 1 sec. */
5144 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5145 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5146 "the second SRST FIS is timed out, "
5147 "loop_count = %d", loop_count);
5148 goto out;
5149 }
5150
5151 /* Wait for 10 millisec */
5152 drv_usecwait(AHCI_10MS_USECS);
5153 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5154
5155 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5156 "ahci_software_reset: 2nd loop count: %d, "
5157 "port_cmd_issue = 0x%x, slot = 0x%x",
5158 loop_count, port_cmd_issue, slot);
5159
5160 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5161 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5162 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5163 DDI_SERVICE_UNAFFECTED);
5164 goto out;
5165 }
5166
5167 rval = AHCI_SUCCESS;
5168 out:
5169 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5170 "ahci_software_reset: %s at port %d:%d",
5171 rval == AHCI_SUCCESS ? "succeed" : "failed",
5172 port, pmport);
5173
5174 return (rval);
5175 }
5176
5177 /*
5178 * AHCI port reset ...; the physical communication between the HBA and device
5179 * on a port are disabled. This is more intrusive.
5180 *
5181 * When an HBA or port reset occurs, Phy communication is going to
5182 * be re-established with the device through a COMRESET followed by the
5183 * normal out-of-band communication sequence defined in Serial ATA. At
5184 * the end of reset, the device, if working properly, will send a D2H
5185 * Register FIS, which contains the device signature. When the HBA receives
5186 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5187 * the PxSIG register with the signature.
5188 *
5189 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5190 * function is called. If not, it is assumed the interface is in hung
5191 * condition.
5192 */
5193 static int
5194 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5195 ahci_addr_t *addrp)
5196 {
5197 ahci_addr_t pmult_addr;
5198 uint32_t port_cmd_status;
5199 uint32_t port_scontrol, port_sstatus;
5200 uint32_t port_task_file;
5201 uint32_t port_state;
5202 uint8_t port = addrp->aa_port;
5203
5204 int loop_count;
5205 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5206
5207 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5208
5209 /* Target is a port multiplier port? */
5210 if (AHCI_ADDR_IS_PMPORT(addrp))
5211 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5212
5213 /* Otherwise it must be an HBA port. */
5214 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5215
5216 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5217 "Port %d port resetting...", port);
5218 ahci_portp->ahciport_port_state = 0;
5219
5220 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5221 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5222
5223 /*
5224 * According to the spec, SUD bit should be set here,
5225 * but JMicron JMB363 doesn't follow it, so print
5226 * a debug message.
5227 */
5228 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5229 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5230 "ahci_port_reset: port %d SUD bit not set", port);
5231
5232 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5233 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5234 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5235
5236 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5237 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5238 port_scontrol);
5239
5240 /* Enable PxCMD.FRE to read device */
5241 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5242 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5243 port_cmd_status|AHCI_CMD_STATUS_FRE);
5244
5245 /*
5246 * The port enters P:StartComm state, and the HBA tells the link layer
5247 * to start communication, which involves sending COMRESET to the
5248 * device. And the HBA resets PxTFD.STS to 7Fh.
5249 *
5250 * Give time for COMRESET to percolate, according to the AHCI
5251 * spec, software shall wait at least 1 millisecond before
5252 * clearing PxSCTL.DET
5253 */
5254 drv_usecwait(AHCI_1MS_USECS * 2);
5255
5256 /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5257 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5258 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5259 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5260 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5261 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5262 port_scontrol);
5263
5264 /*
5265 * When a COMINIT is received from the device, then the port enters
5266 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5267 * PxSSTS.DET to 1h to indicate a device is detected but communication
5268 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5269 * a COMINIT has been received.
5270 */
5271 /*
5272 * The DET field is valid only if IPM field indicates
5273 * that the interface is in active state.
5274 */
5275 loop_count = 0;
5276 for (;;) {
5277 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5278 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5279
5280 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5281 /*
5282 * If the interface is not active, the DET field
5283 * is considered not accurate. So we want to
5284 * continue looping.
5285 */
5286 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5287 }
5288
5289 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5290 break;
5291
5292 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5293 /*
5294 * We are effectively timing out after 0.1 sec.
5295 */
5296 break;
5297 }
5298
5299 /* Wait for 10 millisec */
5300 drv_usecwait(AHCI_10MS_USECS);
5301 }
5302
5303 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5304 "ahci_port_reset: 1st loop count: %d, "
5305 "port_sstatus = 0x%x port %d",
5306 loop_count, port_sstatus, port);
5307
5308 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5309 /*
5310 * Either the port is not active or there
5311 * is no device present.
5312 */
5313 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5314 return (AHCI_SUCCESS);
5315 }
5316
5317 /* Clear port serror register for the port */
5318 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5319 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5320 AHCI_SERROR_CLEAR_ALL);
5321
5322 /*
5323 * Devices should return a FIS contains its signature to HBA after
5324 * COMINIT signal. Check whether a D2H Register FIS is received by
5325 * polling PxTFD.STS.
5326 */
5327 loop_count = 0;
5328 for (;;) {
5329 port_task_file =
5330 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5331 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5332
5333 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5334 AHCI_TFD_STS_ERR)) == 0)
5335 break;
5336
5337 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5338 /*
5339 * We are effectively timing out after 11 sec.
5340 */
5341 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5342 "the device hardware has been initialized and "
5343 "the power-up diagnostics failed",
5344 instance, port);
5345
5346 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5347 "port %d: some or all of BSY, DRQ and ERR in "
5348 "PxTFD.STS are not clear. We need another "
5349 "software reset.", port);
5350
5351 /* Clear port serror register for the port */
5352 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5353 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5354 AHCI_SERROR_CLEAR_ALL);
5355
5356 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5357
5358 /* Try another software reset. */
5359 if (ahci_software_reset(ahci_ctlp, ahci_portp,
5360 &pmult_addr) != AHCI_SUCCESS) {
5361 AHCIPORT_SET_STATE(ahci_portp, addrp,
5362 SATA_PSTATE_FAILED);
5363 return (AHCI_FAILURE);
5364 }
5365 break;
5366 }
5367
5368 /* Wait for 10 millisec */
5369 drv_usecwait(AHCI_10MS_USECS);
5370 }
5371
5372 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5373 "ahci_port_reset: 2nd loop count: %d, "
5374 "port_task_file = 0x%x port %d",
5375 loop_count, port_task_file, port);
5376
5377 /* Clear port serror register for the port */
5378 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5379 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5380 AHCI_SERROR_CLEAR_ALL);
5381
5382 /* Set port as ready */
5383 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5384 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5385
5386 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5387 "ahci_port_reset: succeed at port %d.", port);
5388 return (AHCI_SUCCESS);
5389 }
5390
5391 /*
5392 * COMRESET on a port multiplier port.
5393 *
5394 * NOTE: Only called in ahci_port_reset()
5395 */
5396 static int
5397 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5398 ahci_addr_t *addrp)
5399 {
5400 uint32_t port_scontrol, port_sstatus, port_serror;
5401 uint32_t port_cmd_status, port_intr_status;
5402 uint32_t port_state;
5403 uint8_t port = addrp->aa_port;
5404 uint8_t pmport = addrp->aa_pmport;
5405 int loop_count;
5406 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5407
5408 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5409 "port %d:%d: pmport resetting", port, pmport);
5410
5411 /* Initialize pmport state */
5412 AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5413
5414 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5415 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5416 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5417
5418 /* PxCMD.FRE should be set before. */
5419 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5420 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5421 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5422 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5423 return (AHCI_FAILURE);
5424
5425 /*
5426 * Give time for COMRESET to percolate, according to the AHCI
5427 * spec, software shall wait at least 1 millisecond before
5428 * clearing PxSCTL.DET
5429 */
5430 drv_usecwait(AHCI_1MS_USECS*2);
5431
5432 /*
5433 * Fetch the SCONTROL again and rewrite the DET part with 0
5434 * This will generate an Asychronous Notification events.
5435 */
5436 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5437 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5438 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5439
5440 /*
5441 * The port enters P:StartComm state, and HBA tells link layer to
5442 * start communication, which involves sending COMRESET to device.
5443 * And the HBA resets PxTFD.STS to 7Fh.
5444 *
5445 * When a COMINIT is received from the device, then the port enters
5446 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5447 * PxSSTS.DET to 1h to indicate a device is detected but communication
5448 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5449 * a COMINIT has been received.
5450 */
5451 /*
5452 * The DET field is valid only if IPM field indicates
5453 * that the interface is in active state.
5454 */
5455 loop_count = 0;
5456 do {
5457 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5458
5459 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5460 /*
5461 * If the interface is not active, the DET field
5462 * is considered not accurate. So we want to
5463 * continue looping.
5464 */
5465 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5466 }
5467
5468 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5469 /*
5470 * We are effectively timing out after 0.1 sec.
5471 */
5472 break;
5473 }
5474
5475 /* Wait for 10 millisec */
5476 drv_usecwait(AHCI_10MS_USECS);
5477
5478 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5479
5480 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5481 "ahci_pmport_reset: 1st loop count: %d, "
5482 "port_sstatus = 0x%x port %d:%d",
5483 loop_count, port_sstatus, port, pmport);
5484
5485 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5486 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5487 /*
5488 * Either the port is not active or there
5489 * is no device present.
5490 */
5491 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5492 "ahci_pmport_reset: "
5493 "no device attached to port %d:%d",
5494 port, pmport);
5495 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5496 return (AHCI_SUCCESS);
5497 }
5498
5499 /* Now we can make sure there is a device connected to the port */
5500 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5501 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5502
5503 if (!(port_serror & (1 << 26))) {
5504 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5505 "COMINIT signal from the device not received port %d:%d",
5506 instance, port, pmport);
5507
5508 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5509 return (AHCI_FAILURE);
5510 }
5511
5512 /*
5513 * After clear PxSERR register, we will receive a D2H FIS.
5514 * Normally this FIS will cause a IPMS error according to AHCI spec
5515 * v1.2 because there is no command outstanding for it. So we need
5516 * to ignore this error.
5517 */
5518 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5519 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5520
5521 /* Now we need to check the D2H FIS by checking IPMS error. */
5522 loop_count = 0;
5523 do {
5524 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5525 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5526
5527 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5528 /*
5529 * No D2H FIS received. This is possible according
5530 * to SATA 2.6 spec.
5531 */
5532 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5533 "PxIS.IPMS is not set, we need another "
5534 "software reset.", port, pmport);
5535
5536 break;
5537 }
5538
5539 /* Wait for 10 millisec */
5540 mutex_exit(&ahci_portp->ahciport_mutex);
5541 delay(AHCI_10MS_TICKS);
5542 mutex_enter(&ahci_portp->ahciport_mutex);
5543
5544 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5545
5546 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5547 "ahci_pmport_reset: 2st loop count: %d, "
5548 "port_sstatus = 0x%x port %d:%d",
5549 loop_count, port_sstatus, port, pmport);
5550
5551 /* Clear IPMS */
5552 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5553 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5554 AHCI_INTR_STATUS_IPMS);
5555 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5556
5557 /* This pmport is now ready for ahci_tran_start() */
5558 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5559 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5560
5561 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5562 "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5563 return (AHCI_SUCCESS);
5564
5565 err: /* R/W PMULT error */
5566 /* IPMS flags might be set before. */
5567 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5568 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5569 "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5570
5571 return (AHCI_FAILURE);
5572 }
5573
5574 /*
5575 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5576 * This is the most intrusive.
5577 *
5578 * When an HBA reset occurs, Phy communication will be re-established with
5579 * the device through a COMRESET followed by the normal out-of-band
5580 * communication sequence defined in Serial ATA. At the end of reset, the
5581 * device, if working properly, will send a D2H Register FIS, which contains
5582 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5583 * and PxTFD.ERR register fields, and updates the PxSIG register with the
5584 * signature.
5585 *
5586 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5587 */
5588 static int
5589 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5590 {
5591 ahci_port_t *ahci_portp;
5592 uint32_t ghc_control;
5593 uint8_t port;
5594 int loop_count;
5595 int rval = AHCI_SUCCESS;
5596
5597 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5598 NULL);
5599
5600 mutex_enter(&ahci_ctlp->ahcictl_mutex);
5601
5602 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5603 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5604
5605 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5606 ghc_control |= AHCI_HBA_GHC_HR;
5607 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5608 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5609
5610 /*
5611 * Wait until HBA Reset complete or timeout
5612 */
5613 loop_count = 0;
5614 do {
5615 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5616 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5617
5618 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5619 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5620 "ahci hba reset is timing out, "
5621 "ghc_control = 0x%x", ghc_control);
5622 /* We are effectively timing out after 1 sec. */
5623 break;
5624 }
5625
5626 /* Wait for 10 millisec */
5627 drv_usecwait(AHCI_10MS_USECS);
5628 } while (ghc_control & AHCI_HBA_GHC_HR);
5629
5630 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5631 "ahci_hba_reset: 1st loop count: %d, "
5632 "ghc_control = 0x%x", loop_count, ghc_control);
5633
5634 if (ghc_control & AHCI_HBA_GHC_HR) {
5635 /* The hba is not reset for some reasons */
5636 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5637 "hba reset failed: HBA in a hung or locked state", NULL);
5638 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5639 return (AHCI_FAILURE);
5640 }
5641
5642 /*
5643 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5644 */
5645 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5646 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5647 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5648 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5649 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5650
5651 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5652
5653 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5654 /* Only check implemented ports */
5655 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5656 continue;
5657 }
5658
5659 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5660 mutex_enter(&ahci_portp->ahciport_mutex);
5661
5662 /* Make sure the drive is spun-up */
5663 ahci_staggered_spin_up(ahci_ctlp, port);
5664
5665 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5666 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5667 AHCI_SUCCESS) {
5668 rval = AHCI_FAILURE;
5669 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5670 "ahci_hba_reset: port %d failed", port);
5671 /*
5672 * Set the port state to SATA_PSTATE_FAILED if
5673 * failed to initialize it.
5674 */
5675 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5676 }
5677
5678 mutex_exit(&ahci_portp->ahciport_mutex);
5679 }
5680
5681 return (rval);
5682 }
5683
5684 /*
5685 * This routine is only called from AHCI_ATTACH or phyrdy change
5686 * case. It first calls software reset, then stop the port and try to
5687 * read PxSIG register to find the type of device attached to the port.
5688 *
5689 * The caller should make sure a valid device exists on specified port and
5690 * physical communication has been established so that the signature could
5691 * be retrieved by software reset.
5692 *
5693 * NOTE: The port interrupts should be disabled before the function is called.
5694 */
5695 static void
5696 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5697 ahci_addr_t *addrp)
5698 {
5699 ahci_addr_t dev_addr;
5700 uint32_t signature;
5701 uint8_t port = addrp->aa_port;
5702 uint8_t pmport = addrp->aa_pmport;
5703 int rval;
5704
5705 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5706 ASSERT(AHCI_ADDR_IS_VALID(addrp));
5707
5708 /*
5709 * If the HBA doesn't support port multiplier, then the driver
5710 * doesn't need to bother to check port multiplier device.
5711 *
5712 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5713 * connected to Silicon Image 4723, to which the two sata drives
5714 * attached can be set with RAID1, RAID0 or Spanning mode.
5715 *
5716 * We found software reset will get failure if port multiplier address
5717 * 0xf is used by software reset, so just ignore the check since
5718 * ICH7 doesn't support port multiplier device at all.
5719 */
5720 if (AHCI_ADDR_IS_PORT(addrp) &&
5721 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5722 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5723 "ahci_find_dev_signature enter: port %d", port);
5724
5725 /*
5726 * NOTE: when the ahci address is a HBA port, we do not know
5727 * it is a device or a port multiplier that attached. we need
5728 * try a software reset at port multiplier address (0xf
5729 * pmport)
5730 */
5731 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5732 } else {
5733 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5734 "ahci_find_dev_signature enter: port %d:%d",
5735 port, pmport);
5736 dev_addr = *addrp;
5737 }
5738
5739 /* Assume it is unknown. */
5740 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5741
5742 /* Issue a software reset to get the signature */
5743 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5744 if (rval != AHCI_SUCCESS) {
5745
5746 /*
5747 * Try to do software reset again with pmport set with 0 if
5748 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5749 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5750 */
5751 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5752 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5753 dev_addr.aa_pmport = 0;
5754 rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5755 &dev_addr);
5756 }
5757
5758 if (rval != AHCI_SUCCESS) {
5759 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5760 "ahci_find_dev_signature: software reset failed "
5761 "at port %d:%d, cannot get signature.",
5762 port, pmport);
5763
5764 AHCIPORT_SET_STATE(ahci_portp, addrp,
5765 SATA_PSTATE_FAILED);
5766 return;
5767 }
5768 }
5769
5770 /*
5771 * ahci_software_reset has started the port, so we need manually stop
5772 * the port again.
5773 */
5774 if (AHCI_ADDR_IS_PORT(addrp)) {
5775 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5776 ahci_portp, port) != AHCI_SUCCESS) {
5777 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5778 "ahci_find_dev_signature: cannot stop port %d.",
5779 port);
5780 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5781 return;
5782 }
5783 }
5784
5785 /* Now we can make sure that a valid signature is received. */
5786 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5787 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5788
5789 if (AHCI_ADDR_IS_PMPORT(addrp)) {
5790 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5791 "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5792 signature, port, pmport);
5793 } else {
5794 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5795 "ahci_find_dev_signature: signature = 0x%x at port %d",
5796 signature, port);
5797 }
5798
5799 /* NOTE: Only support ATAPI device at controller port. */
5800 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5801 signature = SATA_DTYPE_UNKNOWN;
5802
5803 switch (signature) {
5804
5805 case AHCI_SIGNATURE_DISK:
5806 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5807 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5808 "Disk is found at port: %d", port);
5809 break;
5810
5811 case AHCI_SIGNATURE_ATAPI:
5812 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5813 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5814 "ATAPI device is found at port: %d", port);
5815 break;
5816
5817 case AHCI_SIGNATURE_PORT_MULTIPLIER:
5818 /* Port Multiplier cannot recursively attached. */
5819 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5820 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5821 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5822 "Port Multiplier is found at port: %d", port);
5823 break;
5824
5825 default:
5826 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5827 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5828 "Unknown device is found at port: %d", port);
5829 }
5830 }
5831
5832 /*
5833 * According to the spec, to reliably detect hot plug removals, software
5834 * must disable interface power management. Software should perform the
5835 * following initialization on a port after a device is attached:
5836 * Set PxSCTL.IPM to 3h to disable interface state transitions
5837 * Set PxCMD.ALPE to '0' to disable aggressive power management
5838 * Disable device initiated interface power management by SET FEATURE
5839 *
5840 * We can ignore the last item because by default the feature is disabled
5841 */
5842 static void
5843 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5844 {
5845 uint32_t port_scontrol, port_cmd_status;
5846
5847 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5848 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5849 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5850 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5851 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5852
5853 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5854 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5855 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5856 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5857 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5858 }
5859
5860 /*
5861 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5862 * to 1, then set it firstly.
5863 *
5864 * Each port contains two major DMA engines. One DMA engine walks through
5865 * the command list, and is controlled by PxCMD.ST. The second DMA engine
5866 * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5867 *
5868 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5869 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5870 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5871 *
5872 * Software shall not set PxCMD.ST to '1' unless a functional device is
5873 * present on the port(as determined by PxTFD.STS.BSY = '0',
5874 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5875 */
5876 static int
5877 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5878 {
5879 uint32_t port_cmd_status;
5880
5881 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5882
5883 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5884
5885 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5886 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5887 "the state for port %d is 0x%x",
5888 port, ahci_portp->ahciport_port_state);
5889 return (AHCI_FAILURE);
5890 }
5891
5892 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5893 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5894 "no device is attached at port %d", port);
5895 return (AHCI_FAILURE);
5896 }
5897
5898 /* First to set PxCMD.FRE before setting PxCMD.ST. */
5899 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5900 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5901
5902 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
5903 port_cmd_status |= AHCI_CMD_STATUS_FRE;
5904 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5905 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5906 port_cmd_status);
5907 }
5908
5909 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5910 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5911
5912 port_cmd_status |= AHCI_CMD_STATUS_ST;
5913
5914 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5915 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5916 port_cmd_status);
5917
5918 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
5919
5920 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
5921 "PxCMD.ST set to '1' at port %d", port);
5922
5923 return (AHCI_SUCCESS);
5924 }
5925
5926 /*
5927 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
5928 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
5929 * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
5930 */
5931 static int
5932 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
5933 {
5934 uint8_t port = ahci_portp->ahciport_port_num;
5935 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5936 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5937
5938 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5939
5940 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
5941 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
5942 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5943 port) != AHCI_SUCCESS)
5944 return (AHCI_FAILURE);
5945
5946 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5947 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5948 }
5949
5950 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
5951 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
5952 int loop_count = 0;
5953
5954 /* Clear PxCMD.FRE */
5955 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
5956 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5957 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5958 port_cmd_status);
5959
5960 /* Wait until PxCMD.FR is cleared */
5961 for (;;) {
5962 port_cmd_status =
5963 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5964 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5965
5966 if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
5967 break;
5968
5969 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
5970 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
5971 "ahci_setup_port_base_addresses: cannot "
5972 "clear PxCMD.FR for port %d.", port);
5973
5974 /*
5975 * We are effectively timing out after 0.5 sec.
5976 * This value is specified in AHCI spec.
5977 */
5978 return (AHCI_FAILURE);
5979 }
5980
5981 /* Wait for 1 millisec */
5982 drv_usecwait(AHCI_1MS_USECS);
5983 }
5984 }
5985
5986 /* Step 3: Config Port Command List Base Address */
5987 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5988 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
5989 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
5990
5991 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5992 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
5993 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
5994
5995 /* Step 4: Config Port Received FIS Base Address */
5996 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5997 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
5998 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
5999
6000 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6001 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6002 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6003
6004 return (AHCI_SUCCESS);
6005 }
6006
6007 /*
6008 * Allocate the ahci_port_t including Received FIS and Command List.
6009 * The argument - port is the physical port number, and not logical
6010 * port number seen by the SATA framework.
6011 */
6012 static int
6013 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6014 {
6015 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6016 ahci_port_t *ahci_portp;
6017 char taskq_name[64] = "event_handle_taskq";
6018
6019 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6020
6021 ahci_portp =
6022 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6023
6024 ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6025 ahci_portp->ahciport_port_num = port;
6026
6027 /* Initialize the port condition variable */
6028 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6029
6030 /* Initialize the port mutex */
6031 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6032 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6033
6034 mutex_enter(&ahci_portp->ahciport_mutex);
6035
6036 /*
6037 * Allocate memory for received FIS structure and
6038 * command list for this port
6039 */
6040 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6041 goto err_case1;
6042 }
6043
6044 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6045 goto err_case2;
6046 }
6047
6048 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6049 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6050 AHCI_SUCCESS) {
6051 goto err_case3;
6052 }
6053
6054 (void) snprintf(taskq_name + strlen(taskq_name),
6055 sizeof (taskq_name) - strlen(taskq_name),
6056 "_port%d", port);
6057
6058 /* Create the taskq for the port */
6059 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6060 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6061 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6062 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6063 goto err_case3;
6064 }
6065
6066 /* Allocate the argument for the taskq */
6067 ahci_portp->ahciport_event_args =
6068 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6069
6070 ahci_portp->ahciport_event_args->ahciea_addrp =
6071 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6072
6073 if (ahci_portp->ahciport_event_args == NULL)
6074 goto err_case4;
6075
6076 /* Initialize the done queue */
6077 ahci_portp->ahciport_doneq = NULL;
6078 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6079 ahci_portp->ahciport_doneq_len = 0;
6080
6081 mutex_exit(&ahci_portp->ahciport_mutex);
6082
6083 return (AHCI_SUCCESS);
6084
6085 err_case4:
6086 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6087
6088 err_case3:
6089 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6090
6091 err_case2:
6092 ahci_dealloc_rcvd_fis(ahci_portp);
6093
6094 err_case1:
6095 mutex_exit(&ahci_portp->ahciport_mutex);
6096 mutex_destroy(&ahci_portp->ahciport_mutex);
6097 cv_destroy(&ahci_portp->ahciport_cv);
6098
6099 kmem_free(ahci_portp, sizeof (ahci_port_t));
6100
6101 return (AHCI_FAILURE);
6102 }
6103
6104 /*
6105 * Reverse of ahci_alloc_port_state().
6106 */
6107 static void
6108 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6109 {
6110 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6111
6112 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6113 ASSERT(ahci_portp != NULL);
6114
6115 mutex_enter(&ahci_portp->ahciport_mutex);
6116 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6117 sizeof (ahci_addr_t));
6118 ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6119 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6120 ahci_portp->ahciport_event_args = NULL;
6121 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6122 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6123 ahci_dealloc_rcvd_fis(ahci_portp);
6124 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6125 mutex_exit(&ahci_portp->ahciport_mutex);
6126
6127 mutex_destroy(&ahci_portp->ahciport_mutex);
6128 cv_destroy(&ahci_portp->ahciport_cv);
6129
6130 kmem_free(ahci_portp, sizeof (ahci_port_t));
6131
6132 ahci_ctlp->ahcictl_ports[port] = NULL;
6133 }
6134
6135 /*
6136 * Allocates memory for the Received FIS Structure
6137 */
6138 static int
6139 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6140 {
6141 size_t rcvd_fis_size;
6142 size_t ret_len;
6143 uint_t cookie_count;
6144
6145 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6146
6147 rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6148
6149 /* allocate rcvd FIS dma handle. */
6150 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6151 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6152 DDI_DMA_SLEEP,
6153 NULL,
6154 &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6155 DDI_SUCCESS) {
6156 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6157 "rcvd FIS dma handle alloc failed", NULL);
6158
6159 return (AHCI_FAILURE);
6160 }
6161
6162 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6163 rcvd_fis_size,
6164 &accattr,
6165 DDI_DMA_CONSISTENT,
6166 DDI_DMA_SLEEP,
6167 NULL,
6168 (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6169 &ret_len,
6170 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6171
6172 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6173 "rcvd FIS dma mem alloc fail", NULL);
6174 /* error.. free the dma handle. */
6175 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6176 return (AHCI_FAILURE);
6177 }
6178
6179 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6180 NULL,
6181 (caddr_t)ahci_portp->ahciport_rcvd_fis,
6182 rcvd_fis_size,
6183 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6184 DDI_DMA_SLEEP,
6185 NULL,
6186 &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6187 &cookie_count) != DDI_DMA_MAPPED) {
6188
6189 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6190 "rcvd FIS dma handle bind fail", NULL);
6191 /* error.. free the dma handle & free the memory. */
6192 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6193 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6194 return (AHCI_FAILURE);
6195 }
6196
6197 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6198
6199 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6200 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6201 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6202 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6203
6204 return (AHCI_SUCCESS);
6205 }
6206
6207 /*
6208 * Deallocates the Received FIS Structure
6209 */
6210 static void
6211 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6212 {
6213 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6214
6215 /* Unbind the cmd list dma handle first. */
6216 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6217
6218 /* Then free the underlying memory. */
6219 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6220
6221 /* Now free the handle itself. */
6222 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6223 }
6224
6225 /*
6226 * Allocates memory for the Command List, which contains up to 32 entries.
6227 * Each entry contains a command header, which is a 32-byte structure that
6228 * includes the pointer to the command table.
6229 */
6230 static int
6231 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6232 {
6233 size_t cmd_list_size;
6234 size_t ret_len;
6235 uint_t cookie_count;
6236
6237 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6238
6239 cmd_list_size =
6240 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6241
6242 /* allocate cmd list dma handle. */
6243 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6244 &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6245 DDI_DMA_SLEEP,
6246 NULL,
6247 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6248
6249 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6250 "cmd list dma handle alloc failed", NULL);
6251 return (AHCI_FAILURE);
6252 }
6253
6254 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6255 cmd_list_size,
6256 &accattr,
6257 DDI_DMA_CONSISTENT,
6258 DDI_DMA_SLEEP,
6259 NULL,
6260 (caddr_t *)&ahci_portp->ahciport_cmd_list,
6261 &ret_len,
6262 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6263
6264 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6265 "cmd list dma mem alloc fail", NULL);
6266 /* error.. free the dma handle. */
6267 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6268 return (AHCI_FAILURE);
6269 }
6270
6271 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6272 NULL,
6273 (caddr_t)ahci_portp->ahciport_cmd_list,
6274 cmd_list_size,
6275 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6276 DDI_DMA_SLEEP,
6277 NULL,
6278 &ahci_portp->ahciport_cmd_list_dma_cookie,
6279 &cookie_count) != DDI_DMA_MAPPED) {
6280
6281 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6282 "cmd list dma handle bind fail", NULL);
6283 /* error.. free the dma handle & free the memory. */
6284 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6285 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6286 return (AHCI_FAILURE);
6287 }
6288
6289 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6290
6291 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6292 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6293
6294 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6295 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6296
6297 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6298 goto err_out;
6299 }
6300
6301 return (AHCI_SUCCESS);
6302
6303 err_out:
6304 /* Unbind the cmd list dma handle first. */
6305 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6306
6307 /* Then free the underlying memory. */
6308 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6309
6310 /* Now free the handle itself. */
6311 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6312
6313 return (AHCI_FAILURE);
6314 }
6315
6316 /*
6317 * Deallocates the Command List
6318 */
6319 static void
6320 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6321 {
6322 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6323
6324 /* First dealloc command table */
6325 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6326
6327 /* Unbind the cmd list dma handle first. */
6328 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6329
6330 /* Then free the underlying memory. */
6331 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6332
6333 /* Now free the handle itself. */
6334 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6335 }
6336
6337 /*
6338 * Allocates memory for all Command Tables, which contains Command FIS,
6339 * ATAPI Command and Physical Region Descriptor Table.
6340 */
6341 static int
6342 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6343 {
6344 size_t ret_len;
6345 ddi_dma_cookie_t cmd_table_dma_cookie;
6346 uint_t cookie_count;
6347 int slot;
6348
6349 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6350
6351 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6352 "ahci_alloc_cmd_tables: port %d enter",
6353 ahci_portp->ahciport_port_num);
6354
6355 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6356 /* Allocate cmd table dma handle. */
6357 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6358 &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6359 DDI_DMA_SLEEP,
6360 NULL,
6361 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6362 DDI_SUCCESS) {
6363
6364 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6365 "cmd table dma handle alloc failed", NULL);
6366
6367 goto err_out;
6368 }
6369
6370 if (ddi_dma_mem_alloc(
6371 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6372 ahci_cmd_table_size,
6373 &accattr,
6374 DDI_DMA_CONSISTENT,
6375 DDI_DMA_SLEEP,
6376 NULL,
6377 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6378 &ret_len,
6379 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6380 NULL) {
6381
6382 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6383 "cmd table dma mem alloc fail", NULL);
6384
6385 /* error.. free the dma handle. */
6386 ddi_dma_free_handle(
6387 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6388 goto err_out;
6389 }
6390
6391 if (ddi_dma_addr_bind_handle(
6392 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6393 NULL,
6394 (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6395 ahci_cmd_table_size,
6396 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6397 DDI_DMA_SLEEP,
6398 NULL,
6399 &cmd_table_dma_cookie,
6400 &cookie_count) != DDI_DMA_MAPPED) {
6401
6402 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6403 "cmd table dma handle bind fail", NULL);
6404 /* error.. free the dma handle & free the memory. */
6405 ddi_dma_mem_free(
6406 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6407 ddi_dma_free_handle(
6408 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6409 goto err_out;
6410 }
6411
6412 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6413 ahci_cmd_table_size);
6414
6415 /* Config Port Command Table Base Address */
6416 SET_COMMAND_TABLE_BASE_ADDR(
6417 (&ahci_portp->ahciport_cmd_list[slot]),
6418 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6419
6420 #ifndef __lock_lint
6421 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6422 (&ahci_portp->ahciport_cmd_list[slot]),
6423 cmd_table_dma_cookie.dmac_laddress >> 32);
6424 #endif
6425 }
6426
6427 return (AHCI_SUCCESS);
6428 err_out:
6429
6430 for (slot--; slot >= 0; slot--) {
6431 /* Unbind the cmd table dma handle first */
6432 (void) ddi_dma_unbind_handle(
6433 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6434
6435 /* Then free the underlying memory */
6436 ddi_dma_mem_free(
6437 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6438
6439 /* Now free the handle itself */
6440 ddi_dma_free_handle(
6441 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6442 }
6443
6444 return (AHCI_FAILURE);
6445 }
6446
6447 /*
6448 * Deallocates memory for all Command Tables.
6449 */
6450 static void
6451 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6452 {
6453 int slot;
6454
6455 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6456
6457 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6458 "ahci_dealloc_cmd_tables: %d enter",
6459 ahci_portp->ahciport_port_num);
6460
6461 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6462 /* Unbind the cmd table dma handle first. */
6463 (void) ddi_dma_unbind_handle(
6464 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6465
6466 /* Then free the underlying memory. */
6467 ddi_dma_mem_free(
6468 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6469
6470 /* Now free the handle itself. */
6471 ddi_dma_free_handle(
6472 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6473 }
6474 }
6475
6476 /*
6477 * Update SATA registers at controller ports
6478 */
6479 static void
6480 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6481 sata_device_t *sd)
6482 {
6483 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6484
6485 sd->satadev_scr.sstatus =
6486 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6487 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6488 sd->satadev_scr.serror =
6489 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6490 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6491 sd->satadev_scr.scontrol =
6492 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6493 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6494 sd->satadev_scr.sactive =
6495 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6496 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6497 }
6498
6499 /*
6500 * For poll mode, ahci_port_intr will be called to emulate the interrupt
6501 */
6502 static void
6503 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6504 {
6505 uint32_t port_intr_status;
6506 uint32_t port_intr_enable;
6507
6508 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6509 "ahci_port_intr enter: port %d", port);
6510
6511 mutex_enter(&ahci_portp->ahciport_mutex);
6512 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6513 /* For SATA_OPMODE_POLLING commands */
6514 port_intr_enable =
6515 (AHCI_INTR_STATUS_DHRS |
6516 AHCI_INTR_STATUS_PSS |
6517 AHCI_INTR_STATUS_SDBS |
6518 AHCI_INTR_STATUS_UFS |
6519 AHCI_INTR_STATUS_PCS |
6520 AHCI_INTR_STATUS_PRCS |
6521 AHCI_INTR_STATUS_OFS |
6522 AHCI_INTR_STATUS_INFS |
6523 AHCI_INTR_STATUS_IFS |
6524 AHCI_INTR_STATUS_HBDS |
6525 AHCI_INTR_STATUS_HBFS |
6526 AHCI_INTR_STATUS_TFES);
6527 } else {
6528 /*
6529 * port_intr_enable indicates that the corresponding interrrupt
6530 * reporting is enabled.
6531 */
6532 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6533 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6534 }
6535
6536 /* IPMS error in port reset should be ignored according AHCI spec. */
6537 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6538 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6539 mutex_exit(&ahci_portp->ahciport_mutex);
6540
6541 /*
6542 * port_intr_stats indicates that the corresponding interrupt
6543 * condition is active.
6544 */
6545 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6546 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6547
6548 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6549 "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6550 "port_intr_enable = 0x%x",
6551 port, port_intr_status, port_intr_enable);
6552
6553 port_intr_status &= port_intr_enable;
6554
6555 /*
6556 * Pending interrupt events are indicated by the PxIS register.
6557 * Make sure we don't miss any event.
6558 */
6559 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6560 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6561 DDI_SERVICE_UNAFFECTED);
6562 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6563 DDI_FME_VERSION);
6564 return;
6565 }
6566
6567 /* First clear the port interrupts status */
6568 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6569 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6570 port_intr_status);
6571
6572 /* Check the completed non-queued commands */
6573 if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6574 AHCI_INTR_STATUS_PSS)) {
6575 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6576 ahci_portp, port);
6577 }
6578
6579 /* Check the completed queued commands */
6580 if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6581 (void) ahci_intr_set_device_bits(ahci_ctlp,
6582 ahci_portp, port);
6583 }
6584
6585 /* Check the port connect change status interrupt bit */
6586 if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6587 (void) ahci_intr_port_connect_change(ahci_ctlp,
6588 ahci_portp, port);
6589 }
6590
6591 /* Check the device mechanical presence status interrupt bit */
6592 if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6593 (void) ahci_intr_device_mechanical_presence_status(
6594 ahci_ctlp, ahci_portp, port);
6595 }
6596
6597 /* Check the PhyRdy change status interrupt bit */
6598 if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6599 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6600 port);
6601 }
6602
6603 /*
6604 * Check the non-fatal error interrupt bits, there are four
6605 * kinds of non-fatal errors at the time being:
6606 *
6607 * PxIS.UFS - Unknown FIS Error
6608 * PxIS.OFS - Overflow Error
6609 * PxIS.INFS - Interface Non-Fatal Error
6610 * PxIS.IPMS - Incorrect Port Multiplier Status Error
6611 *
6612 * For these non-fatal errors, the HBA can continue to operate,
6613 * so the driver just log the error messages.
6614 */
6615 if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6616 AHCI_INTR_STATUS_OFS |
6617 AHCI_INTR_STATUS_IPMS |
6618 AHCI_INTR_STATUS_INFS)) {
6619 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6620 port, port_intr_status);
6621 }
6622
6623 /*
6624 * Check the fatal error interrupt bits, there are four kinds
6625 * of fatal errors for AHCI controllers:
6626 *
6627 * PxIS.HBFS - Host Bus Fatal Error
6628 * PxIS.HBDS - Host Bus Data Error
6629 * PxIS.IFS - Interface Fatal Error
6630 * PxIS.TFES - Task File Error
6631 *
6632 * The fatal error means the HBA can not recover from it by
6633 * itself, and it will try to abort the transfer, and the software
6634 * must intervene to restart the port.
6635 */
6636 if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6637 AHCI_INTR_STATUS_HBDS |
6638 AHCI_INTR_STATUS_HBFS |
6639 AHCI_INTR_STATUS_TFES))
6640 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6641 port, port_intr_status);
6642
6643 /* Check the cold port detect interrupt bit */
6644 if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6645 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6646 }
6647
6648 /* Second clear the corresponding bit in IS.IPS */
6649 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6650 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6651
6652 /* Try to recover at the end of the interrupt handler. */
6653 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6654 DDI_FM_OK) {
6655 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6656 DDI_SERVICE_UNAFFECTED);
6657 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6658 DDI_FME_VERSION);
6659 }
6660 }
6661
6662 /*
6663 * Interrupt service handler
6664 */
6665 static uint_t
6666 ahci_intr(caddr_t arg1, caddr_t arg2)
6667 {
6668 #ifndef __lock_lint
6669 _NOTE(ARGUNUSED(arg2))
6670 #endif
6671 /* LINTED */
6672 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6673 ahci_port_t *ahci_portp;
6674 int32_t global_intr_status;
6675 uint8_t port;
6676
6677 /*
6678 * global_intr_status indicates that the corresponding port has
6679 * an interrupt pending.
6680 */
6681 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6682 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6683
6684 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6685 /* The interrupt is not ours */
6686 return (DDI_INTR_UNCLAIMED);
6687 }
6688
6689 /*
6690 * Check the handle after reading global_intr_status - we don't want
6691 * to miss any port with pending interrupts.
6692 */
6693 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6694 DDI_FM_OK) {
6695 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6696 DDI_SERVICE_UNAFFECTED);
6697 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6698 DDI_FME_VERSION);
6699 return (DDI_INTR_UNCLAIMED);
6700 }
6701
6702 /* Loop for all the ports */
6703 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6704 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6705 continue;
6706 }
6707 if (!((0x1 << port) & global_intr_status)) {
6708 continue;
6709 }
6710
6711 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6712
6713 /* Call ahci_port_intr */
6714 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6715 }
6716
6717 return (DDI_INTR_CLAIMED);
6718 }
6719
6720 /*
6721 * For non-queued commands, when the corresponding bit in the PxCI register
6722 * is cleared, it means the command is completed successfully. And according
6723 * to the HBA state machine, there are three conditions which possibly will
6724 * try to clear the PxCI register bit.
6725 * 1. Receive one D2H Register FIS which is with 'I' bit set
6726 * 2. Update PIO Setup FIS
6727 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6728 *
6729 * Process completed non-queued commands when the interrupt status bit -
6730 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6731 *
6732 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6733 * with the 'I' bit set. And the following commands will send thus
6734 * FIS with 'I' bit set upon the successful completion:
6735 * 1. Non-data commands
6736 * 2. DMA data-in command
6737 * 3. DMA data-out command
6738 * 4. PIO data-out command
6739 * 5. PACKET non-data commands
6740 * 6. PACKET PIO data-in command
6741 * 7. PACKET PIO data-out command
6742 * 8. PACKET DMA data-in command
6743 * 9. PACKET DMA data-out command
6744 *
6745 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6746 * with the 'I' bit set. And the following commands will send this
6747 * FIS upon the successful completion:
6748 * 1. PIO data-in command
6749 */
6750 static int
6751 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6752 ahci_port_t *ahci_portp, uint8_t port)
6753 {
6754 uint32_t port_cmd_issue = 0;
6755 uint32_t finished_tags;
6756 int finished_slot;
6757 sata_pkt_t *satapkt;
6758 ahci_fis_d2h_register_t *rcvd_fisp;
6759 #if AHCI_DEBUG
6760 ahci_cmd_header_t *cmd_header;
6761 uint32_t cmd_dmacount;
6762 #endif
6763
6764 mutex_enter(&ahci_portp->ahciport_mutex);
6765
6766 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6767 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6768 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6769 /*
6770 * Spurious interrupt. Nothing to be done.
6771 */
6772 mutex_exit(&ahci_portp->ahciport_mutex);
6773 return (AHCI_SUCCESS);
6774 }
6775
6776 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6777 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6778
6779 /* If the PxCI corrupts, don't complete the commmands. */
6780 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6781 != DDI_FM_OK) {
6782 mutex_exit(&ahci_portp->ahciport_mutex);
6783 return (AHCI_FAILURE);
6784 }
6785
6786 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6787 /* Slot 0 is always used during error recovery */
6788 finished_tags = 0x1 & ~port_cmd_issue;
6789 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6790 "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6791 "retrieval is finished, and finished_tags = 0x%x",
6792 port, finished_tags);
6793 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6794 finished_tags = 0x1 & ~port_cmd_issue;
6795 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6796 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6797 "port multiplier is finished, and finished_tags = 0x%x",
6798 port, finished_tags);
6799
6800 } else {
6801
6802 finished_tags = ahci_portp->ahciport_pending_tags &
6803 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6804 }
6805
6806 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6807 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6808 "port_cmd_issue = 0x%x finished_tags = 0x%x",
6809 ahci_portp->ahciport_pending_tags, port_cmd_issue,
6810 finished_tags);
6811
6812 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6813 (finished_tags == 0x1)) {
6814 satapkt = ahci_portp->ahciport_err_retri_pkt;
6815 ASSERT(satapkt != NULL);
6816
6817 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6818 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6819 "with SATA_PKT_COMPLETED", (void *)satapkt);
6820
6821 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6822 goto out;
6823 }
6824
6825 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6826 (finished_tags == 0x1)) {
6827 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6828 ASSERT(satapkt != NULL);
6829
6830 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6831 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6832 "with SATA_PKT_COMPLETED", (void *)satapkt);
6833
6834 /* READ PORTMULT need copy out FIS content. */
6835 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6836 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6837 ahcirf_d2h_register_fis);
6838 satapkt->satapkt_cmd.satacmd_status_reg =
6839 GET_RFIS_STATUS(rcvd_fisp);
6840 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6841 }
6842
6843 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6844 goto out;
6845 }
6846
6847 while (finished_tags) {
6848 finished_slot = ddi_ffs(finished_tags) - 1;
6849 if (finished_slot == -1) {
6850 goto out;
6851 }
6852
6853 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6854 ASSERT(satapkt != NULL);
6855 #if AHCI_DEBUG
6856 /*
6857 * For non-native queued commands, the PRD byte count field
6858 * shall contain an accurate count of the number of bytes
6859 * transferred for the command before the PxCI bit is cleared
6860 * to '0' for the command.
6861 *
6862 * The purpose of this field is to let software know how many
6863 * bytes transferred for a given operation in order to
6864 * determine if underflow occurred. When issuing native command
6865 * queuing commands, this field should not be used and is not
6866 * required to be valid since in this case underflow is always
6867 * illegal.
6868 *
6869 * For data reads, the HBA will update its PRD byte count with
6870 * the total number of bytes received from the last FIS, and
6871 * may be able to continue normally. For data writes, the
6872 * device will detect an error, and HBA most likely will get
6873 * a fatal error.
6874 *
6875 * Therefore, here just put code to debug part. And please
6876 * refer to the comment above ahci_intr_fatal_error for the
6877 * definition of underflow error.
6878 */
6879 cmd_dmacount =
6880 ahci_portp->ahciport_prd_bytecounts[finished_slot];
6881 if (cmd_dmacount) {
6882 cmd_header =
6883 &ahci_portp->ahciport_cmd_list[finished_slot];
6884 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6885 "ahci_intr_cmd_cmplt: port %d, "
6886 "PRD Byte Count = 0x%x, "
6887 "ahciport_prd_bytecounts = 0x%x", port,
6888 cmd_header->ahcich_prd_byte_count,
6889 cmd_dmacount);
6890
6891 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6892 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6893 "ahci_intr_cmd_cmplt: port %d, "
6894 "an underflow occurred", port);
6895 }
6896 }
6897 #endif
6898
6899 /*
6900 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6901 * feature, sata_special_regs flag will be set, and the
6902 * driver should copy the status and the other corresponding
6903 * register values in the D2H Register FIS received (It's
6904 * working on Non-data protocol) from the device back to
6905 * the sata_cmd.
6906 *
6907 * For every AHCI port, there is only one Received FIS
6908 * structure, which contains the FISes received from the
6909 * device, So we're trying to copy the content of D2H
6910 * Register FIS in the Received FIS structure back to
6911 * the sata_cmd.
6912 */
6913 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6914 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6915 ahcirf_d2h_register_fis);
6916 satapkt->satapkt_cmd.satacmd_status_reg =
6917 GET_RFIS_STATUS(rcvd_fisp);
6918 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6919 }
6920
6921 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6922 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6923 "with SATA_PKT_COMPLETED", (void *)satapkt);
6924
6925 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
6926 CLEAR_BIT(finished_tags, finished_slot);
6927 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
6928
6929 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6930 }
6931 out:
6932 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
6933 "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
6934 ahci_portp->ahciport_pending_tags);
6935
6936 ahci_flush_doneq(ahci_portp);
6937
6938 mutex_exit(&ahci_portp->ahciport_mutex);
6939
6940 return (AHCI_SUCCESS);
6941 }
6942
6943 /*
6944 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
6945 * with the 'I' bit set and has been copied into system memory. It will
6946 * be sent under the following situations:
6947 *
6948 * 1. NCQ command is completed
6949 *
6950 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
6951 * via the Set Device Bits FIS. When such event is generated, the software
6952 * needs to read PxSACT register and compares the current value to the
6953 * list of commands previously issue by software. ahciport_pending_ncq_tags
6954 * keeps the tags of previously issued commands.
6955 *
6956 * 2. Asynchronous Notification
6957 *
6958 * Asynchronous Notification is a feature in SATA spec 2.6.
6959 *
6960 * 1) ATAPI device will send a signal to the host when media is inserted or
6961 * removed and avoids polling the device for media changes. The signal
6962 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
6963 * set to '1'. At the moment, it's not supported yet.
6964 *
6965 * 2) Port multiplier will send a signal to the host when a hot plug event
6966 * has occured on a port multiplier port. It is used when command based
6967 * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
6968 */
6969 static int
6970 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
6971 ahci_port_t *ahci_portp, uint8_t port)
6972 {
6973 ahci_addr_t addr;
6974
6975 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
6976 "ahci_intr_set_device_bits enter: port %d", port);
6977
6978 /* Initialize HBA port address */
6979 AHCI_ADDR_SET_PORT(&addr, port);
6980
6981 /* NCQ plug handler */
6982 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
6983
6984 /* Check port multiplier's asynchronous notification events */
6985 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
6986 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
6987 ahci_portp, port);
6988 }
6989
6990 /* ATAPI events is not supported yet */
6991
6992 return (AHCI_SUCCESS);
6993 }
6994 /*
6995 * NCQ interrupt handler. Called upon a NCQ command is completed.
6996 * Only be called from ahci_intr_set_device_bits().
6997 */
6998 static int
6999 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7000 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7001 {
7002 uint32_t port_sactive;
7003 uint32_t port_cmd_issue;
7004 uint32_t issued_tags;
7005 int issued_slot;
7006 uint32_t finished_tags;
7007 int finished_slot;
7008 uint8_t port = addrp->aa_port;
7009 sata_pkt_t *satapkt;
7010
7011 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7012 "ahci_intr_set_device_bits enter: port %d", port);
7013
7014 mutex_enter(&ahci_portp->ahciport_mutex);
7015 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7016 mutex_exit(&ahci_portp->ahciport_mutex);
7017 return (AHCI_SUCCESS);
7018 }
7019
7020 /*
7021 * First the handler got which commands are finished by checking
7022 * PxSACT register
7023 */
7024 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7025 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7026
7027 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7028 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7029
7030 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7031 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7032 "port_sactive = 0x%x", port,
7033 ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7034
7035 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7036 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7037
7038 /*
7039 * For NCQ commands, the software can determine which command has
7040 * already been transmitted to the device by checking PxCI register.
7041 */
7042 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7043 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7044
7045 issued_tags = ahci_portp->ahciport_pending_tags &
7046 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7047
7048 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7049 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7050 != DDI_FM_OK) {
7051 mutex_exit(&ahci_portp->ahciport_mutex);
7052 return (AHCI_FAILURE);
7053 }
7054
7055 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7056 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7057 "port_cmd_issue = 0x%x", port,
7058 ahci_portp->ahciport_pending_tags, port_cmd_issue);
7059
7060 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7061 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7062
7063 /*
7064 * Clear ahciport_pending_tags bit when the corresponding command
7065 * is already sent down to the device.
7066 */
7067 while (issued_tags) {
7068 issued_slot = ddi_ffs(issued_tags) - 1;
7069 if (issued_slot == -1) {
7070 goto next;
7071 }
7072 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7073 CLEAR_BIT(issued_tags, issued_slot);
7074 }
7075
7076 next:
7077 while (finished_tags) {
7078 finished_slot = ddi_ffs(finished_tags) - 1;
7079 if (finished_slot == -1) {
7080 goto out;
7081 }
7082
7083 /* The command is certainly transmitted to the device */
7084 ASSERT(!(ahci_portp->ahciport_pending_tags &
7085 (0x1 << finished_slot)));
7086
7087 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7088 ASSERT(satapkt != NULL);
7089
7090 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7091 "ahci_intr_set_device_bits: sending up pkt 0x%p "
7092 "with SATA_PKT_COMPLETED", (void *)satapkt);
7093
7094 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7095 CLEAR_BIT(finished_tags, finished_slot);
7096 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7097
7098 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7099 }
7100 out:
7101 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7102 "ahci_intr_set_device_bits: port %d "
7103 "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7104 port, ahci_portp->ahciport_pending_ncq_tags,
7105 ahci_portp->ahciport_pending_tags);
7106
7107 ahci_flush_doneq(ahci_portp);
7108
7109 mutex_exit(&ahci_portp->ahciport_mutex);
7110
7111 return (AHCI_SUCCESS);
7112 }
7113
7114 /*
7115 * Port multiplier asynchronous notification event handler. Called upon a
7116 * device is hot plugged/pulled.
7117 *
7118 * The async-notification event will only be recorded by ahcipmi_snotif_tags
7119 * here and will be handled by ahci_probe_pmult().
7120 *
7121 * NOTE: called only from ahci_port_intr().
7122 */
7123 static int
7124 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7125 uint8_t port)
7126 {
7127 sata_device_t sdevice;
7128
7129 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7130 "ahci_intr_pmult_sntf_events enter: port %d ", port);
7131
7132 /* no hot-plug while attaching process */
7133 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7134 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7135 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7136 return (AHCI_SUCCESS);
7137 }
7138 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7139
7140 mutex_enter(&ahci_portp->ahciport_mutex);
7141 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7142 mutex_exit(&ahci_portp->ahciport_mutex);
7143 return (AHCI_SUCCESS);
7144 }
7145
7146 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7147
7148 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7149 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7150 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7151 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7152 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7153 AHCI_SNOTIF_CLEAR_ALL);
7154
7155 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7156 mutex_exit(&ahci_portp->ahciport_mutex);
7157 return (AHCI_SUCCESS);
7158 }
7159
7160 /* Port Multiplier sub-device hot-plug handler */
7161 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7162 mutex_exit(&ahci_portp->ahciport_mutex);
7163 return (AHCI_SUCCESS);
7164 }
7165
7166 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7167 /* Not allowed to re-enter. */
7168 mutex_exit(&ahci_portp->ahciport_mutex);
7169 return (AHCI_SUCCESS);
7170 }
7171
7172 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7173
7174 /*
7175 * NOTE:
7176 * Even if Asynchronous Notification is supported (and enabled) by
7177 * both controller and the port multiplier, the content of PxSNTF
7178 * register is always set to 0x8000 by async notification event. We
7179 * need to check GSCR[32] on the port multiplier to find out the
7180 * owner of this event.
7181 * This is not accord with SATA spec 2.6 and needs further
7182 * clarification.
7183 */
7184 /* hot-plug will not reported while reseting. */
7185 if (ahci_portp->ahciport_reset_in_progress == 1) {
7186 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7187 "port %d snotif event ignored", port);
7188 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7189 mutex_exit(&ahci_portp->ahciport_mutex);
7190 return (AHCI_SUCCESS);
7191 }
7192
7193 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7194 "PxSNTF is set to 0x%x by port multiplier",
7195 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7196
7197 /*
7198 * Now we need do some necessary operation and inform SATA framework
7199 * that link/device events has happened.
7200 */
7201 bzero((void *)&sdevice, sizeof (sata_device_t));
7202 sdevice.satadev_addr.cport = ahci_ctlp->
7203 ahcictl_port_to_cport[port];
7204 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7205 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7206 sdevice.satadev_state = SATA_PSTATE_PWRON;
7207
7208 /* Just reject packets, do not stop that port. */
7209 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7210
7211 mutex_exit(&ahci_portp->ahciport_mutex);
7212 sata_hba_event_notify(
7213 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7214 &sdevice,
7215 SATA_EVNT_PMULT_LINK_CHANGED);
7216 mutex_enter(&ahci_portp->ahciport_mutex);
7217
7218 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7219 mutex_exit(&ahci_portp->ahciport_mutex);
7220
7221 return (AHCI_SUCCESS);
7222 }
7223
7224 /*
7225 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7226 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7227 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7228 * indicates a COMINIT signal was received.
7229 *
7230 * Hot plug insertion is detected by reception of a COMINIT signal from the
7231 * device. On reception of unsolicited COMINIT, the HBA shall generate a
7232 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7233 * begin the normal communication negotiation sequence as outlined in the
7234 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7235 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7236 * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7237 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7238 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7239 * to handle hot plug insertion. In this interrupt handler, just do nothing
7240 * but print some log message and clear the bit.
7241 */
7242 static int
7243 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7244 ahci_port_t *ahci_portp, uint8_t port)
7245 {
7246 #if AHCI_DEBUG
7247 uint32_t port_serror;
7248 #endif
7249
7250 mutex_enter(&ahci_portp->ahciport_mutex);
7251
7252 #if AHCI_DEBUG
7253 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7254 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7255
7256 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7257 "ahci_intr_port_connect_change: port %d, "
7258 "port_serror = 0x%x", port, port_serror);
7259 #endif
7260
7261 /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7262 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7263 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7264 SERROR_EXCHANGED_ERR);
7265
7266 mutex_exit(&ahci_portp->ahciport_mutex);
7267
7268 return (AHCI_SUCCESS);
7269 }
7270
7271 /*
7272 * Hot Plug Operation for platforms that support Mechanical Presence
7273 * Switches.
7274 *
7275 * When set, it indicates that a mechanical presence switch attached to this
7276 * port has been opened or closed, which may lead to a change in the connection
7277 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7278 * are set to '1'.
7279 *
7280 * At the moment, this interrupt is not needed and disabled and we just log
7281 * the debug message.
7282 */
7283 static int
7284 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7285 ahci_port_t *ahci_portp, uint8_t port)
7286 {
7287 uint32_t cap_status, port_cmd_status;
7288
7289 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7290 "ahci_intr_device_mechanical_presence_status enter, "
7291 "port %d", port);
7292
7293 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7294 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7295
7296 mutex_enter(&ahci_portp->ahciport_mutex);
7297 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7298 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7299
7300 if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7301 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7302 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7303 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7304 "the interrupt: cap_status = 0x%x, "
7305 "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7306 mutex_exit(&ahci_portp->ahciport_mutex);
7307
7308 return (AHCI_SUCCESS);
7309 }
7310
7311 #if AHCI_DEBUG
7312 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7313 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7314 "The mechanical presence switch is open: "
7315 "port %d, port_cmd_status = 0x%x",
7316 port, port_cmd_status);
7317 } else {
7318 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7319 "The mechanical presence switch is close: "
7320 "port %d, port_cmd_status = 0x%x",
7321 port, port_cmd_status);
7322 }
7323 #endif
7324
7325 mutex_exit(&ahci_portp->ahciport_mutex);
7326
7327 return (AHCI_SUCCESS);
7328 }
7329
7330 /*
7331 * Native Hot Plug Support.
7332 *
7333 * When set, it indicates that the internal PHYRDY signal changed state.
7334 * This bit reflects the state of PxSERR.DIAG.N.
7335 *
7336 * There are three kinds of conditions to generate this interrupt event:
7337 * 1. a device is inserted
7338 * 2. a device is disconnected
7339 * 3. when the link enters/exits a Partial or Slumber interface power
7340 * management state
7341 *
7342 * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7343 * bit may be set due to the link entering the Partial or Slumber power
7344 * management state, rather than due to a hot plug insertion or removal
7345 * event. So far, the interface power management is disabled, so the
7346 * driver can reliably get removal detection notification via the
7347 * PxSERR.DIAG.N bit.
7348 */
7349 static int
7350 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7351 ahci_port_t *ahci_portp, uint8_t port)
7352 {
7353 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7354 sata_device_t sdevice;
7355 int dev_exists_now = 0;
7356 int dev_existed_previously = 0;
7357 ahci_addr_t port_addr;
7358
7359 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7360 "ahci_intr_phyrdy_change enter, port %d", port);
7361
7362 /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7363 mutex_enter(&ahci_portp->ahciport_mutex);
7364 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7365 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7366 SERROR_PHY_RDY_CHG);
7367 mutex_exit(&ahci_portp->ahciport_mutex);
7368
7369 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7370 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7371 (ahci_portp == NULL)) {
7372 /* The whole controller setup is not yet done. */
7373 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7374 return (AHCI_SUCCESS);
7375 }
7376 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7377
7378 mutex_enter(&ahci_portp->ahciport_mutex);
7379
7380 /* SStatus tells the presence of device. */
7381 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7382 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7383
7384 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7385 dev_exists_now = 1;
7386 }
7387
7388 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7389 dev_existed_previously = 1;
7390 }
7391
7392 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7393 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7394 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7395 "ahci_intr_phyrdy_change: port %d "
7396 "AHCI_PORT_FLAG_NODEV is cleared", port);
7397 if (dev_exists_now == 0)
7398 dev_existed_previously = 1;
7399 }
7400
7401 bzero((void *)&sdevice, sizeof (sata_device_t));
7402 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7403 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7404 sdevice.satadev_addr.pmport = 0;
7405 sdevice.satadev_state = SATA_PSTATE_PWRON;
7406 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7407
7408 AHCI_ADDR_SET_PORT(&port_addr, port);
7409
7410 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7411 if (dev_exists_now) {
7412 if (dev_existed_previously) { /* 1 -> 1 */
7413 /* Things are fine now. The loss was temporary. */
7414 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7415 "ahci_intr_phyrdy_change port %d "
7416 "device link lost/established", port);
7417
7418 mutex_exit(&ahci_portp->ahciport_mutex);
7419 sata_hba_event_notify(
7420 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7421 &sdevice,
7422 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7423 mutex_enter(&ahci_portp->ahciport_mutex);
7424
7425 } else { /* 0 -> 1 */
7426 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7427 "ahci_intr_phyrdy_change: port %d "
7428 "device link established", port);
7429
7430 /*
7431 * A new device has been detected. The new device
7432 * might be a port multiplier instead of a drive, so
7433 * we cannot update the signature directly.
7434 */
7435 (void) ahci_initialize_port(ahci_ctlp,
7436 ahci_portp, &port_addr);
7437
7438 /* Try to start the port */
7439 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7440 != AHCI_SUCCESS) {
7441 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7442 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7443 "ahci_intr_phyrdy_change: port %d failed "
7444 "at start port", port);
7445 }
7446
7447 /* Clear the max queue depth for inserted device */
7448 ahci_portp->ahciport_max_ncq_tags = 0;
7449
7450 mutex_exit(&ahci_portp->ahciport_mutex);
7451 sata_hba_event_notify(
7452 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7453 &sdevice,
7454 SATA_EVNT_LINK_ESTABLISHED);
7455 mutex_enter(&ahci_portp->ahciport_mutex);
7456
7457 }
7458 } else { /* No device exists now */
7459
7460 if (dev_existed_previously) { /* 1 -> 0 */
7461 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7462 "ahci_intr_phyrdy_change: port %d "
7463 "device link lost", port);
7464
7465 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7466 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7467 ahci_portp, port);
7468
7469 if (ahci_portp->ahciport_device_type ==
7470 SATA_DTYPE_PMULT) {
7471 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7472 }
7473
7474 /* An existing device is lost. */
7475 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7476 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7477
7478 mutex_exit(&ahci_portp->ahciport_mutex);
7479 sata_hba_event_notify(
7480 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7481 &sdevice,
7482 SATA_EVNT_LINK_LOST);
7483 mutex_enter(&ahci_portp->ahciport_mutex);
7484 }
7485 }
7486 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7487
7488 mutex_exit(&ahci_portp->ahciport_mutex);
7489
7490 return (AHCI_SUCCESS);
7491 }
7492
7493 /*
7494 * PxIS.UFS - Unknown FIS Error
7495 *
7496 * This interrupt event means an unknown FIS was received and has been
7497 * copied into system memory. An unknown FIS is not considered an illegal
7498 * FIS, unless the length received is more than 64 bytes. If an unknown
7499 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7500 * normal operation. If the unknown FIS is more than 64 bytes, then it
7501 * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7502 * a fatal error.
7503 *
7504 * PxIS.IPMS - Incorrect Port Multiplier Status
7505 *
7506 * IPMS Indicates that the HBA received a FIS from a device that did not
7507 * have a command outstanding. The IPMS bit may be set during enumeration
7508 * of devices on a Port Multiplier due to the normal Port Multiplier
7509 * enumeration process. It is recommended that IPMS only be used after
7510 * enumeration is complete on the Port Multiplier (copied from spec).
7511 *
7512 * PxIS.OFS - Overflow Error
7513 *
7514 * Command list overflow is defined as software building a command table
7515 * that has fewer total bytes than the transaction given to the device.
7516 * On device writes, the HBA will run out of data, and on reads, there
7517 * will be no room to put the data.
7518 *
7519 * For an overflow on data read, either PIO or DMA, the HBA will set
7520 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7521 * non-fatal error when the HBA can continues. Sometimes, it will cause
7522 * a fatal error and need the software to do something.
7523 *
7524 * For an overflow on data write, setting PxIS.OFS is optional for both
7525 * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7526 * software to clean up from this serious error.
7527 *
7528 * PxIS.INFS - Interface Non-Fatal Error
7529 *
7530 * This interrupt event indicates that the HBA encountered an error on
7531 * the Serial ATA interface but was able to continue operation. The kind
7532 * of error usually occurred during a non-Data FIS, and under this condition
7533 * the FIS will be re-transmitted by HBA automatically.
7534 *
7535 * When the FMA is implemented, there should be a stat structure to
7536 * record how many every kind of error happens.
7537 */
7538 static int
7539 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7540 uint8_t port, uint32_t intr_status)
7541 {
7542 uint32_t port_serror;
7543 #if AHCI_DEBUG
7544 uint32_t port_cmd_status;
7545 uint32_t port_cmd_issue;
7546 uint32_t port_sactive;
7547 int current_slot;
7548 uint32_t current_tags;
7549 sata_pkt_t *satapkt;
7550 ahci_cmd_header_t *cmd_header;
7551 uint32_t cmd_dmacount;
7552 #endif
7553
7554 mutex_enter(&ahci_portp->ahciport_mutex);
7555
7556 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7557 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7558
7559 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7560 "ahci_intr_non_fatal_error: port %d, "
7561 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7562
7563 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7564
7565 if (intr_status & AHCI_INTR_STATUS_UFS) {
7566 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7567 "ahci port %d has unknown FIS error", port);
7568
7569 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7570 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7571 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7572 SERROR_FIS_TYPE);
7573 }
7574
7575 #if AHCI_DEBUG
7576 if (intr_status & AHCI_INTR_STATUS_IPMS) {
7577 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7578 "has Incorrect Port Multiplier Status error", port);
7579 }
7580
7581 if (intr_status & AHCI_INTR_STATUS_OFS) {
7582 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7583 "ahci port %d has overflow error", port);
7584 }
7585
7586 if (intr_status & AHCI_INTR_STATUS_INFS) {
7587 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7588 "ahci port %d has interface non fatal error", port);
7589 }
7590
7591 /*
7592 * Record the error occurred command's slot.
7593 */
7594 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7595 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7596 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7597 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7598
7599 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7600 AHCI_CMD_STATUS_CCS_SHIFT;
7601
7602 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7603 satapkt = ahci_portp->ahciport_err_retri_pkt;
7604 ASSERT(satapkt != NULL);
7605 ASSERT(current_slot == 0);
7606 } else {
7607 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7608 }
7609
7610 if (satapkt != NULL) {
7611 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7612 "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7613 "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7614 satapkt->satapkt_cmd.satacmd_cmd_reg);
7615
7616 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7617 "ahci_intr_non_fatal_error: port %d, "
7618 "satapkt 0x%p is being processed when error occurs",
7619 port, (void *)satapkt);
7620
7621 /*
7622 * PRD Byte Count field of command header is not
7623 * required to reflect the total number of bytes
7624 * transferred when an overflow occurs, so here
7625 * just log the value.
7626 */
7627 cmd_dmacount =
7628 ahci_portp->ahciport_prd_bytecounts[current_slot];
7629 if (cmd_dmacount) {
7630 cmd_header = &ahci_portp->
7631 ahciport_cmd_list[current_slot];
7632 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7633 "ahci_intr_non_fatal_error: port %d, "
7634 "PRD Byte Count = 0x%x, "
7635 "ahciport_prd_bytecounts = 0x%x", port,
7636 cmd_header->ahcich_prd_byte_count,
7637 cmd_dmacount);
7638 }
7639 }
7640 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7641 /*
7642 * For queued command, list those command which have already
7643 * been transmitted to the device and still not completed.
7644 */
7645 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7646 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7647
7648 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7649 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7650
7651 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7652 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7653 "port_sactive = 0x%x port_cmd_issue = 0x%x",
7654 ahci_portp->ahciport_pending_ncq_tags,
7655 port_sactive, port_cmd_issue);
7656
7657 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7658 port_sactive & ~port_cmd_issue &
7659 AHCI_NCQ_SLOT_MASK(ahci_portp);
7660
7661 while (current_tags) {
7662 current_slot = ddi_ffs(current_tags) - 1;
7663 if (current_slot == -1) {
7664 goto out;
7665 }
7666
7667 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7668 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7669 ahci_ctlp, "ahci_intr_non_fatal_error: "
7670 "port %d, satapkt 0x%p is outstanding when "
7671 "error occurs", port, (void *)satapkt);
7672
7673 CLEAR_BIT(current_tags, current_slot);
7674 }
7675 }
7676 out:
7677 #endif
7678 mutex_exit(&ahci_portp->ahciport_mutex);
7679
7680 return (AHCI_SUCCESS);
7681 }
7682
7683 /*
7684 * According to the AHCI spec, the error types include system memory
7685 * errors, interface errors, port multiplier errors, device errors,
7686 * command list overflow, command list underflow, native command
7687 * queuing tag errors and pio data transfer errors.
7688 *
7689 * System memory errors such as target abort, master abort, and parity
7690 * may cause the host to stop, and they are serious errors and needed
7691 * to be recovered with software intervention. When system software
7692 * has given a pointer to the HBA that doesn't exist in physical memory,
7693 * a master/target abort error occurs, and PxIS.HBFS will be set. A
7694 * data error such as CRC or parity occurs, the HBA aborts the transfer
7695 * (if necessary) and PxIS.HBDS will be set.
7696 *
7697 * Interface errors are errors that occur due to electrical issues on
7698 * the interface, or protocol miscommunication between the device and
7699 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7700 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7701 * causes PxIS.IFS/PxIS.INFS to be set are
7702 * 1. in PxSERR.ERR, P bit is set to '1'
7703 * 2. in PxSERR.DIAG, C or H bit is set to '1'
7704 * 3. PhyRdy drop unexpectly, N bit is set to '1'
7705 * If the error occurred during a non-data FIS, the FIS must be
7706 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7707 * the error occurred during a data FIS, the transfer will stop, so
7708 * the error is fatal and PxIS.IFS is set.
7709 *
7710 * When a FIS arrives that updates the taskfile, the HBA checks to see
7711 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7712 * stops processing any more commands.
7713 *
7714 * Command list overflow is defined as software building a command table
7715 * that has fewer total bytes than the transaction given to the device.
7716 * On device writes, the HBA will run out of data, and on reads, there
7717 * will be no room to put the data. For an overflow on data read, either
7718 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7719 * For an overflow on data write, setting PxIS.OFS is optional for both
7720 * DMA and PIO, and a COMRESET is required by software to clean up from
7721 * this serious error.
7722 *
7723 * Command list underflow is defined as software building a command
7724 * table that has more total bytes than the transaction given to the
7725 * device. For data writes, both PIO and DMA, the device will detect
7726 * an error and end the transfer. And these errors are most likely going
7727 * to be fatal errors that will cause the port to be restarted. For
7728 * data reads, the HBA updates its PRD byte count, and may be
7729 * able to continue normally, but is not required to. And The HBA is
7730 * not required to detect underflow conditions for native command
7731 * queuing command.
7732 *
7733 * The HBA does not actively check incoming DMA Setup FISes to ensure
7734 * that the PxSACT register bit for that slot is set. Existing error
7735 * mechanisms, such as host bus failure, or bad protocol, are used to
7736 * recover from this case.
7737 *
7738 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7739 * DATA FIS must be an integral number of Dwords. If the HBA receives
7740 * a request which is not an integral number of Dwords, the HBA
7741 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7742 * software restarts the port. And the HBA ensures that the size
7743 * of the DATA FIS received during a PIO command matches the size in
7744 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7745 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7746 * stop running until software restarts the port.
7747 */
7748 /*
7749 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7750 *
7751 * PxIS.IFS indicates that the hba encountered an error on the serial ata
7752 * interface which caused the transfer to stop.
7753 *
7754 * PxIS.HBDS indicates that the hba encountered a data error
7755 * (uncorrectable ecc/parity) when reading from or writing to system memory.
7756 *
7757 * PxIS.HBFS indicates that the hba encountered a host bus error that it
7758 * cannot recover from, such as a bad software pointer.
7759 *
7760 * PxIS.TFES is set whenever the status register is updated by the device
7761 * and the error bit (bit 0) is set.
7762 */
7763 static int
7764 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7765 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7766 {
7767 uint32_t port_cmd_status;
7768 uint32_t port_serror;
7769 uint32_t task_file_status;
7770 int failed_slot;
7771 sata_pkt_t *spkt = NULL;
7772 uint8_t err_byte;
7773 ahci_event_arg_t *args;
7774 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7775 uint32_t failed_tags = 0;
7776 int task_fail_flag = 0, task_abort_flag = 0;
7777 uint32_t slot_status;
7778
7779 mutex_enter(&ahci_portp->ahciport_mutex);
7780
7781 /*
7782 * ahci_intr_phyrdy_change() may have rendered it to
7783 * SATA_DTYPE_NONE.
7784 */
7785 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7786 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7787 "ahci_intr_fatal_error: port %d no device attached, "
7788 "and just return without doing anything", port);
7789 goto out0;
7790 }
7791
7792 if (intr_status & AHCI_INTR_STATUS_TFES) {
7793 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7794 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7795 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7796 "ahci_intr_fatal_error: port %d "
7797 "task_file_status = 0x%x", port, task_file_status);
7798 task_fail_flag = 1;
7799
7800 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7801 >> AHCI_TFD_ERR_SHIFT;
7802 if (err_byte == SATA_ERROR_ABORT)
7803 task_abort_flag = 1;
7804 }
7805
7806 /*
7807 * Here we just log the fatal error info in interrupt context.
7808 * Misc recovery processing will be handled in task queue.
7809 */
7810 if (task_fail_flag == 1) {
7811 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7812 /*
7813 * Read PxCMD.CCS to determine the slot that the HBA
7814 * was processing when the error occurred.
7815 */
7816 port_cmd_status = ddi_get32(
7817 ahci_ctlp->ahcictl_ahci_acc_handle,
7818 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7819 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7820 AHCI_CMD_STATUS_CCS_SHIFT;
7821 failed_tags = 0x1 << failed_slot;
7822
7823 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7824 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7825 "ahci_intr_fatal_error: spkt 0x%p is being "
7826 "processed when fatal error occurred for port %d",
7827 spkt, port);
7828
7829 /*
7830 * Won't emit the error message if it is an IDENTIFY
7831 * DEVICE command sent to an ATAPI device.
7832 */
7833 if ((spkt != NULL) &&
7834 (spkt->satapkt_cmd.satacmd_cmd_reg ==
7835 SATAC_ID_DEVICE) &&
7836 (task_abort_flag == 1))
7837 goto out1;
7838
7839 /*
7840 * Won't emit the error message if it is an ATAPI PACKET
7841 * command
7842 */
7843 if ((spkt != NULL) &&
7844 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7845 goto out1;
7846
7847 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7848 slot_status = ddi_get32(
7849 ahci_ctlp->ahcictl_ahci_acc_handle,
7850 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7851 failed_tags = slot_status &
7852 AHCI_NCQ_SLOT_MASK(ahci_portp);
7853 }
7854 }
7855
7856 /* print the fatal error type */
7857 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7858 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7859
7860 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7861 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7862
7863 /* print PxSERR related error message */
7864 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7865
7866 /* print task file register value */
7867 if (task_fail_flag == 1) {
7868 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7869 "= 0x%x", instance, port, task_file_status);
7870 if (task_abort_flag == 1) {
7871 cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7872 "port %d are aborted", instance, port);
7873 ahci_dump_commands(ahci_ctlp, port, failed_tags);
7874 }
7875 }
7876
7877 out1:
7878 /* Prepare the argument for the taskq */
7879 args = ahci_portp->ahciport_event_args;
7880 args->ahciea_ctlp = (void *)ahci_ctlp;
7881 args->ahciea_portp = (void *)ahci_portp;
7882 args->ahciea_event = intr_status;
7883 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7884
7885 /* Start the taskq to handle error recovery */
7886 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7887 ahci_events_handler,
7888 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7889 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7890 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7891 "port %d failed", instance, port);
7892 }
7893 out0:
7894 mutex_exit(&ahci_portp->ahciport_mutex);
7895
7896 return (AHCI_SUCCESS);
7897 }
7898
7899 /*
7900 * Hot Plug Operation for platforms that support Cold Presence Detect.
7901 *
7902 * When set, a device status has changed as detected by the cold presence
7903 * detect logic. This bit can either be set due to a non-connected port
7904 * receiving a device, or a connected port having its device removed.
7905 * This bit is only valid if the port supports cold presence detect as
7906 * indicated by PxCMD.CPD set to '1'.
7907 *
7908 * At the moment, this interrupt is not needed and disabled and we just
7909 * log the debug message.
7910 */
7911 static int
7912 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
7913 ahci_port_t *ahci_portp, uint8_t port)
7914 {
7915 uint32_t port_cmd_status;
7916 sata_device_t sdevice;
7917
7918 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7919 "ahci_intr_cold_port_detect enter, port %d", port);
7920
7921 mutex_enter(&ahci_portp->ahciport_mutex);
7922
7923 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7924 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7925 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
7926 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7927 "port %d does not support cold presence detect, so "
7928 "we just ignore this interrupt", port);
7929 mutex_exit(&ahci_portp->ahciport_mutex);
7930 return (AHCI_SUCCESS);
7931 }
7932
7933 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7934 "port %d device status has changed", port);
7935
7936 bzero((void *)&sdevice, sizeof (sata_device_t));
7937 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7938 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7939 sdevice.satadev_addr.pmport = 0;
7940 sdevice.satadev_state = SATA_PSTATE_PWRON;
7941
7942 if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
7943 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7944 "port %d: a device is hot plugged", port);
7945 mutex_exit(&ahci_portp->ahciport_mutex);
7946 sata_hba_event_notify(
7947 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7948 &sdevice,
7949 SATA_EVNT_DEVICE_ATTACHED);
7950 mutex_enter(&ahci_portp->ahciport_mutex);
7951
7952 } else {
7953 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7954 "port %d: a device is hot unplugged", port);
7955 mutex_exit(&ahci_portp->ahciport_mutex);
7956 sata_hba_event_notify(
7957 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7958 &sdevice,
7959 SATA_EVNT_DEVICE_DETACHED);
7960 mutex_enter(&ahci_portp->ahciport_mutex);
7961 }
7962
7963 mutex_exit(&ahci_portp->ahciport_mutex);
7964
7965 return (AHCI_SUCCESS);
7966 }
7967
7968 /*
7969 * Enable the interrupts for a particular port.
7970 */
7971 static void
7972 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
7973 {
7974 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
7975
7976 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
7977 "ahci_enable_port_intrs enter, port %d", port);
7978
7979 /*
7980 * Clear port interrupt status before enabling interrupt
7981 */
7982 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7983 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
7984 AHCI_PORT_INTR_MASK);
7985
7986 /*
7987 * Clear the pending bit from IS.IPS
7988 */
7989 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7990 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
7991
7992 /*
7993 * Enable the following interrupts:
7994 * Device to Host Register FIS Interrupt (DHRS)
7995 * PIO Setup FIS Interrupt (PSS)
7996 * Set Device Bits Interrupt (SDBS)
7997 * Unknown FIS Interrupt (UFS)
7998 * Port Connect Change Status (PCS)
7999 * PhyRdy Change Status (PRCS)
8000 * Overflow Status (OFS)
8001 * Interface Non-fatal Error Status (INFS)
8002 * Interface Fatal Error Status (IFS)
8003 * Host Bus Data Error Status (HBDS)
8004 * Host Bus Fatal Error Status (HBFS)
8005 * Task File Error Status (TFES)
8006 */
8007 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8008 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8009 (AHCI_INTR_STATUS_DHRS |
8010 AHCI_INTR_STATUS_PSS |
8011 AHCI_INTR_STATUS_SDBS |
8012 AHCI_INTR_STATUS_UFS |
8013 AHCI_INTR_STATUS_DPS |
8014 AHCI_INTR_STATUS_PCS |
8015 AHCI_INTR_STATUS_PRCS |
8016 AHCI_INTR_STATUS_OFS |
8017 AHCI_INTR_STATUS_INFS |
8018 AHCI_INTR_STATUS_IFS |
8019 AHCI_INTR_STATUS_HBDS |
8020 AHCI_INTR_STATUS_HBFS |
8021 AHCI_INTR_STATUS_TFES));
8022 }
8023
8024 /*
8025 * Enable interrupts for all the ports.
8026 */
8027 static void
8028 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8029 {
8030 uint32_t ghc_control;
8031
8032 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8033
8034 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8035
8036 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8037 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8038
8039 ghc_control |= AHCI_HBA_GHC_IE;
8040
8041 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8042 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8043 }
8044
8045 /*
8046 * Disable interrupts for a particular port.
8047 */
8048 static void
8049 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8050 {
8051 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8052 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8053
8054 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8055 "ahci_disable_port_intrs enter, port %d", port);
8056
8057 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8058 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8059 }
8060
8061 /*
8062 * Disable interrupts for the whole HBA.
8063 *
8064 * The global bit is cleared, then all interrupt sources from all
8065 * ports are disabled.
8066 */
8067 static void
8068 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8069 {
8070 uint32_t ghc_control;
8071
8072 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8073 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8074
8075 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8076 NULL);
8077
8078 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8079 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8080
8081 ghc_control &= ~AHCI_HBA_GHC_IE;
8082
8083 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8084 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8085 }
8086
8087 /*
8088 * Handle FIXED or MSI interrupts.
8089 */
8090 /*
8091 * According to AHCI spec, the HBA may support several interrupt modes:
8092 * * pin based interrupts (FIXED)
8093 * * single MSI message interrupts
8094 * * multiple MSI based message interrupts
8095 *
8096 * For pin based interrupts, the software interrupt handler need to check IS
8097 * register to find out which port has pending interrupts. And then check
8098 * PxIS register to find out which interrupt events happened on that port.
8099 *
8100 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8101 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8102 * in that software interrupt handler need to check IS register to determine
8103 * which port triggered the interrupts since it uses a single message for all
8104 * port interrupts.
8105 *
8106 * HBA may optionally support multiple MSI message for better performance. In
8107 * this mode, each port may have its own interrupt message, and thus generation
8108 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8109 * represents a power-of-2 wrapper on the number of implemented ports, and
8110 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8111 * maximum number of assigned interrupts. When the number of MSI messages
8112 * allocated is less than the number requested, then hardware may have two
8113 * implementation behaviors:
8114 * * assign each ports its own interrupt and then force all additional
8115 * ports to share the last interrupt message, and this condition is
8116 * indicated by clearing GHC.MRSM to '0'
8117 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8118 * When multiple-message MSI is enabled, hardware will still set IS register
8119 * as single message case. And this IS register may be used by software when
8120 * fewer than the requested number of messages is granted in order to determine
8121 * which port had the interrupt.
8122 *
8123 * Note: The current ahci driver only supports the first two interrupt modes:
8124 * pin based interrupts and single MSI message interrupts, and the reason
8125 * is indicated in below code.
8126 */
8127 static int
8128 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8129 {
8130 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8131 int count, avail, actual;
8132 int i, rc;
8133
8134 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8135 "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8136
8137 /* get number of interrupts. */
8138 rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8139 if ((rc != DDI_SUCCESS) || (count == 0)) {
8140 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8141 "ddi_intr_get_nintrs() failed, "
8142 "rc %d count %d\n", rc, count);
8143 return (DDI_FAILURE);
8144 }
8145
8146 /* get number of available interrupts. */
8147 rc = ddi_intr_get_navail(dip, intr_type, &avail);
8148 if ((rc != DDI_SUCCESS) || (avail == 0)) {
8149 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8150 "ddi_intr_get_navail() failed, "
8151 "rc %d avail %d\n", rc, avail);
8152 return (DDI_FAILURE);
8153 }
8154
8155 #if AHCI_DEBUG
8156 if (avail < count) {
8157 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8158 "ddi_intr_get_nintrs returned %d, navail() returned %d",
8159 count, avail);
8160 }
8161 #endif
8162
8163 /*
8164 * Note: So far Solaris restricts the maximum number of messages for
8165 * x86 to 2, that is avail is 2, so here we set the count with 1 to
8166 * force the driver to use single MSI message interrupt. In future if
8167 * Solaris remove the restriction, then we need to delete the below
8168 * code and try to use multiple interrupt routine to gain better
8169 * performance.
8170 */
8171 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8172 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8173 "force to use one interrupt routine though the "
8174 "HBA supports %d interrupt", count);
8175 count = 1;
8176 }
8177
8178 /* Allocate an array of interrupt handles. */
8179 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8180 ahci_ctlp->ahcictl_intr_htable =
8181 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8182
8183 /* call ddi_intr_alloc(). */
8184 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8185 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8186
8187 if ((rc != DDI_SUCCESS) || (actual == 0)) {
8188 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8189 "ddi_intr_alloc() failed, rc %d count %d actual %d "
8190 "avail %d\n", rc, count, actual, avail);
8191 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8192 ahci_ctlp->ahcictl_intr_size);
8193 return (DDI_FAILURE);
8194 }
8195
8196 /* use interrupt count returned */
8197 #if AHCI_DEBUG
8198 if (actual < count) {
8199 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8200 "Requested: %d, Received: %d", count, actual);
8201 }
8202 #endif
8203
8204 ahci_ctlp->ahcictl_intr_cnt = actual;
8205
8206 /*
8207 * Get priority for first, assume remaining are all the same.
8208 */
8209 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8210 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8211 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8212 "ddi_intr_get_pri() failed", NULL);
8213
8214 /* Free already allocated intr. */
8215 for (i = 0; i < actual; i++) {
8216 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8217 }
8218
8219 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8220 ahci_ctlp->ahcictl_intr_size);
8221 return (DDI_FAILURE);
8222 }
8223
8224 /* Test for high level interrupt. */
8225 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8226 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8227 "ahci_add_intrs: Hi level intr not supported", NULL);
8228
8229 /* Free already allocated intr. */
8230 for (i = 0; i < actual; i++) {
8231 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8232 }
8233
8234 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8235 sizeof (ddi_intr_handle_t));
8236
8237 return (DDI_FAILURE);
8238 }
8239
8240 /* Call ddi_intr_add_handler(). */
8241 for (i = 0; i < actual; i++) {
8242 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8243 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8244 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8245 "ddi_intr_add_handler() failed", NULL);
8246
8247 /* Free already allocated intr. */
8248 for (i = 0; i < actual; i++) {
8249 (void) ddi_intr_free(
8250 ahci_ctlp->ahcictl_intr_htable[i]);
8251 }
8252
8253 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8254 ahci_ctlp->ahcictl_intr_size);
8255 return (DDI_FAILURE);
8256 }
8257 }
8258
8259 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8260 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8261 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8262 "ddi_intr_get_cap() failed", NULL);
8263
8264 /* Free already allocated intr. */
8265 for (i = 0; i < actual; i++) {
8266 (void) ddi_intr_free(
8267 ahci_ctlp->ahcictl_intr_htable[i]);
8268 }
8269
8270 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8271 ahci_ctlp->ahcictl_intr_size);
8272 return (DDI_FAILURE);
8273 }
8274
8275 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8276 /* Call ddi_intr_block_enable() for MSI. */
8277 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8278 ahci_ctlp->ahcictl_intr_cnt);
8279 } else {
8280 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8281 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8282 (void) ddi_intr_enable(
8283 ahci_ctlp->ahcictl_intr_htable[i]);
8284 }
8285 }
8286
8287 return (DDI_SUCCESS);
8288 }
8289
8290 /*
8291 * Removes the registered interrupts irrespective of whether they
8292 * were legacy or MSI.
8293 *
8294 * NOTE: The controller interrupts must be disabled before calling
8295 * this routine.
8296 */
8297 static void
8298 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8299 {
8300 int x;
8301
8302 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8303
8304 /* Disable all interrupts. */
8305 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8306 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8307 /* Call ddi_intr_block_disable(). */
8308 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8309 ahci_ctlp->ahcictl_intr_cnt);
8310 } else {
8311 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8312 (void) ddi_intr_disable(
8313 ahci_ctlp->ahcictl_intr_htable[x]);
8314 }
8315 }
8316
8317 /* Call ddi_intr_remove_handler(). */
8318 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8319 (void) ddi_intr_remove_handler(
8320 ahci_ctlp->ahcictl_intr_htable[x]);
8321 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8322 }
8323
8324 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8325 }
8326
8327 /*
8328 * This routine tries to put port into P:NotRunning state by clearing
8329 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8330 * and PxCMD.CR to '0'.
8331 */
8332 static int
8333 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8334 ahci_port_t *ahci_portp, uint8_t port)
8335 {
8336 uint32_t port_cmd_status;
8337 int loop_count;
8338
8339 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8340 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8341
8342 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8343 "ahci_put_port_into_notrunning_state enter: port %d", port);
8344
8345 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8346 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8347
8348 port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8349 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8350 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8351
8352 /* Wait until PxCMD.CR is cleared */
8353 loop_count = 0;
8354 do {
8355 port_cmd_status =
8356 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8357 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8358
8359 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8360 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8361 "clearing port %d CMD.CR timeout, "
8362 "port_cmd_status = 0x%x", port,
8363 port_cmd_status);
8364 /*
8365 * We are effectively timing out after 0.5 sec.
8366 * This value is specified in AHCI spec.
8367 */
8368 break;
8369 }
8370
8371 /* Wait for 10 millisec */
8372 drv_usecwait(AHCI_10MS_USECS);
8373 } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8374
8375 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8376
8377 if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8378 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8379 "ahci_put_port_into_notrunning_state: failed to clear "
8380 "PxCMD.CR to '0' after loop count: %d, and "
8381 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8382 return (AHCI_FAILURE);
8383 } else {
8384 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8385 "ahci_put_port_into_notrunning_state: succeeded to clear "
8386 "PxCMD.CR to '0' after loop count: %d, and "
8387 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8388 return (AHCI_SUCCESS);
8389 }
8390 }
8391
8392 /*
8393 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8394 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8395 * stable state, then set PxCMD.ST to '1' to start the port directly.
8396 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8397 * COMRESET to the device to put it in an idle state.
8398 *
8399 * The fifth argument returns whether the port reset is involved during
8400 * the process.
8401 *
8402 * The routine will be called under following scenarios:
8403 * + To reset the HBA
8404 * + To abort the packet(s)
8405 * + To reset the port
8406 * + To activate the port
8407 * + Fatal error recovery
8408 * + To abort the timeout packet(s)
8409 *
8410 * NOTES!!! During this procedure, PxSERR register will be cleared, and
8411 * according to the spec, the clearance of three bits will also clear
8412 * three interrupt status bits.
8413 * 1. PxSERR.DIAG.F will clear PxIS.UFS
8414 * 2. PxSERR.DIAG.X will clear PxIS.PCS
8415 * 3. PxSERR.DIAG.N will clear PxIS.PRCS
8416 *
8417 * Among these three interrupt events, the driver needs to take care of
8418 * PxIS.PRCS, which is the hot plug event. When the driver found out
8419 * a device was unplugged, it will call the interrupt handler.
8420 */
8421 static int
8422 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8423 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8424 {
8425 uint32_t port_sstatus;
8426 uint32_t task_file_status;
8427 sata_device_t sdevice;
8428 int rval;
8429 ahci_addr_t addr_port;
8430 ahci_pmult_info_t *pminfo = NULL;
8431 int dev_exists_begin = 0;
8432 int dev_exists_end = 0;
8433 uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8434 int npmport = 0;
8435 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8436
8437 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8438
8439 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8440 "ahci_restart_port_wait_till_ready: port %d enter", port);
8441
8442 AHCI_ADDR_SET_PORT(&addr_port, port);
8443
8444 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8445 dev_exists_begin = 1;
8446
8447 /* First clear PxCMD.ST */
8448 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8449 port);
8450 if (rval != AHCI_SUCCESS)
8451 /*
8452 * If PxCMD.CR does not clear within a reasonable time, it
8453 * may assume the interface is in a hung condition and may
8454 * continue with issuing the port reset.
8455 */
8456 goto reset;
8457
8458 /* Then clear PxSERR */
8459 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8460 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8461 AHCI_SERROR_CLEAR_ALL);
8462
8463 /* Then get PxTFD */
8464 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8465 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8466
8467 /*
8468 * Check whether the device is in a stable status, if yes,
8469 * then start the port directly. However for ahci_tran_reset_dport,
8470 * we may have to perform a port reset.
8471 */
8472 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8473 !(flag & AHCI_PORT_RESET))
8474 goto out;
8475
8476 reset:
8477 /*
8478 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8479 * a COMRESET to the device
8480 */
8481 ahci_disable_port_intrs(ahci_ctlp, port);
8482 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8483 ahci_enable_port_intrs(ahci_ctlp, port);
8484
8485 #ifdef AHCI_DEBUG
8486 if (rval != AHCI_SUCCESS)
8487 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8488 "ahci_restart_port_wait_till_ready: port %d failed",
8489 port);
8490 #endif
8491
8492 if (reset_flag != NULL)
8493 *reset_flag = 1;
8494
8495 /* Indicate to the framework that a reset has happened. */
8496 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8497 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8498 !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8499 /* Set the reset in progress flag */
8500 ahci_portp->ahciport_reset_in_progress = 1;
8501
8502 bzero((void *)&sdevice, sizeof (sata_device_t));
8503 sdevice.satadev_addr.cport =
8504 ahci_ctlp->ahcictl_port_to_cport[port];
8505 sdevice.satadev_addr.pmport = 0;
8506 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8507
8508 sdevice.satadev_state = SATA_DSTATE_RESET |
8509 SATA_DSTATE_PWR_ACTIVE;
8510 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8511 mutex_exit(&ahci_portp->ahciport_mutex);
8512 sata_hba_event_notify(
8513 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8514 &sdevice,
8515 SATA_EVNT_DEVICE_RESET);
8516 mutex_enter(&ahci_portp->ahciport_mutex);
8517 }
8518
8519 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8520 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8521 } else {
8522 ahci_portp->ahciport_reset_in_progress = 0;
8523 }
8524
8525 out:
8526 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8527
8528 /* SStatus tells the presence of device. */
8529 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8530 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8531
8532 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8533 dev_exists_end = 1;
8534 }
8535
8536 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8537 return (rval);
8538
8539 /* Check whether a hot plug event happened */
8540 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8541 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8542 "ahci_restart_port_wait_till_ready: port %d "
8543 "device is removed", port);
8544 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8545 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8546 "ahci_restart_port_wait_till_ready: port %d "
8547 "AHCI_PORT_FLAG_NODEV flag is set", port);
8548 mutex_exit(&ahci_portp->ahciport_mutex);
8549 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8550 mutex_enter(&ahci_portp->ahciport_mutex);
8551
8552 return (rval);
8553 }
8554
8555
8556 /* 0/1 -> 1 : device may change */
8557 /*
8558 * May be called by ahci_fatal_error_recovery_handler, so
8559 * don't issue software if the previous device is ATAPI.
8560 */
8561 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8562 return (rval);
8563
8564 /*
8565 * The COMRESET will make port multiplier enter legacy mode.
8566 * Issue a software reset to make it work again.
8567 */
8568 ahci_disable_port_intrs(ahci_ctlp, port);
8569 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8570 ahci_enable_port_intrs(ahci_ctlp, port);
8571
8572 /*
8573 * Following codes are specific for the port multiplier
8574 */
8575 if (previous_dev_type != SATA_DTYPE_PMULT &&
8576 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8577 /* in case previous_dev_type is corrupt */
8578 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8579 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8580 return (rval);
8581 }
8582
8583 /* Device change: PMult -> Non-PMult */
8584 if (previous_dev_type == SATA_DTYPE_PMULT &&
8585 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8586 /*
8587 * This might happen because
8588 * 1. Software reset failed. Port multiplier is not correctly
8589 * enumerated.
8590 * 2. Another non-port-multiplier device is attached. Perhaps
8591 * the port multiplier was replaced by another device by
8592 * whatever reason, but AHCI driver missed hot-plug event.
8593 *
8594 * Now that the port has been initialized, we just need to
8595 * update the port structure according new device, then report
8596 * and wait SATA framework to probe new device.
8597 */
8598
8599 /* Force to release pmult resource */
8600 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8601 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8602
8603 bzero((void *)&sdevice, sizeof (sata_device_t));
8604 sdevice.satadev_addr.cport =
8605 ahci_ctlp->ahcictl_port_to_cport[port];
8606 sdevice.satadev_addr.pmport = 0;
8607 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8608
8609 sdevice.satadev_state = SATA_DSTATE_RESET |
8610 SATA_DSTATE_PWR_ACTIVE;
8611
8612 mutex_exit(&ahci_portp->ahciport_mutex);
8613 sata_hba_event_notify(
8614 ahci_ctlp->ahcictl_dip,
8615 &sdevice,
8616 SATA_EVNT_DEVICE_RESET);
8617 mutex_enter(&ahci_portp->ahciport_mutex);
8618
8619 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8620 "Port multiplier is [Gone] at port %d ", port);
8621 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8622 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8623
8624 return (AHCI_SUCCESS);
8625 }
8626
8627 /* Device change: Non-PMult -> PMult */
8628 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8629
8630 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8631 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8632
8633 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8634 }
8635 pminfo = ahci_portp->ahciport_pmult_info;
8636 ASSERT(pminfo != NULL);
8637
8638 /* Device (may) change: PMult -> PMult */
8639 /*
8640 * First initialize port multiplier. Set state to READY and wait for
8641 * probe entry point to initialize it
8642 */
8643 ahci_portp->ahciport_port_state = SATA_STATE_READY;
8644
8645 /*
8646 * It's a little complicated while target is a port multiplier. we
8647 * need to COMRESET all pmports behind that PMult otherwise those
8648 * sub-links between the PMult and the sub-devices will be in an
8649 * inactive state (indicated by PSCR0/PxSSTS) and the following access
8650 * to those sub-devices will be rejected by Link-Fatal-Error.
8651 */
8652 /*
8653 * The PxSNTF will be set soon after the pmult is plugged. While the
8654 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8655 * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8656 * will initialized it.
8657 */
8658 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8659 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8660
8661 /* Report reset event. */
8662 ahci_portp->ahciport_reset_in_progress = 1;
8663
8664 bzero((void *)&sdevice, sizeof (sata_device_t));
8665 sdevice.satadev_addr.cport = cport;
8666 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8667 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8668 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8669 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8670 SATA_EVNT_DEVICE_RESET);
8671
8672 return (rval);
8673 }
8674
8675 /*
8676 * This routine may be called under four scenarios:
8677 * a) do the recovery from fatal error
8678 * b) or we need to timeout some commands
8679 * c) or we need to abort some commands
8680 * d) or we need reset device/port/controller
8681 *
8682 * In all these scenarios, we need to send any pending unfinished
8683 * commands up to sata framework.
8684 */
8685 static void
8686 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8687 ahci_port_t *ahci_portp,
8688 uint32_t slot_status,
8689 uint32_t failed_tags,
8690 uint32_t timeout_tags,
8691 uint32_t aborted_tags,
8692 uint32_t reset_tags)
8693 {
8694 uint32_t finished_tags = 0;
8695 uint32_t unfinished_tags = 0;
8696 int tmp_slot;
8697 sata_pkt_t *satapkt;
8698 int ncq_cmd_in_progress = 0;
8699 int err_retri_cmd_in_progress = 0;
8700 int rdwr_pmult_cmd_in_progress = 0;
8701
8702 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8703
8704 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8705 "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8706 ahci_portp->ahciport_port_num, slot_status);
8707
8708 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8709 "ahci_mop_commands: failed_tags: 0x%x, "
8710 "timeout_tags: 0x%x aborted_tags: 0x%x, "
8711 "reset_tags: 0x%x", failed_tags,
8712 timeout_tags, aborted_tags, reset_tags);
8713
8714 #ifdef AHCI_DEBUG
8715 if (ahci_debug_flags & AHCIDBG_ERRS) {
8716 int i;
8717 char msg_buf[200] = {0, };
8718 for (i = 0x1f; i >= 0; i--) {
8719 if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8720 msg_buf[i] = 'X';
8721 else
8722 msg_buf[i] = '.';
8723 }
8724 msg_buf[0x20] = '\0';
8725 cmn_err(CE_NOTE, "port[%d] slots: %s",
8726 ahci_portp->ahciport_port_num, msg_buf);
8727 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8728 (void *)ahci_portp->ahciport_err_retri_pkt,
8729 (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8730 }
8731 #endif
8732
8733 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8734 finished_tags = ahci_portp->ahciport_pending_tags &
8735 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8736
8737 unfinished_tags = slot_status &
8738 AHCI_SLOT_MASK(ahci_ctlp) &
8739 ~failed_tags &
8740 ~aborted_tags &
8741 ~reset_tags &
8742 ~timeout_tags;
8743 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8744 ncq_cmd_in_progress = 1;
8745 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8746 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8747
8748 unfinished_tags = slot_status &
8749 AHCI_NCQ_SLOT_MASK(ahci_portp) &
8750 ~failed_tags &
8751 ~aborted_tags &
8752 ~reset_tags &
8753 ~timeout_tags;
8754 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8755
8756 /*
8757 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8758 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8759 * complete successfully due to one of the following three
8760 * conditions:
8761 *
8762 * 1. Fatal error - failed_tags includes its slot
8763 * 2. Timed out - timeout_tags includes its slot
8764 * 3. Aborted when hot unplug - aborted_tags includes its
8765 * slot
8766 *
8767 * Please note that the command is always sent down in Slot 0
8768 */
8769 err_retri_cmd_in_progress = 1;
8770 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8771 "ahci_mop_commands is called for port %d while "
8772 "REQUEST SENSE or READ LOG EXT for error retrieval "
8773 "is being executed slot_status = 0x%x",
8774 ahci_portp->ahciport_port_num, slot_status);
8775 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8776 ASSERT(slot_status == 0x1);
8777 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8778 rdwr_pmult_cmd_in_progress = 1;
8779 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8780 "ahci_mop_commands is called for port %d while "
8781 "READ/WRITE PORTMULT command is being executed",
8782 ahci_portp->ahciport_port_num);
8783
8784 ASSERT(slot_status == 0x1);
8785 }
8786
8787 #ifdef AHCI_DEBUG
8788 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8789 "ahci_mop_commands: finished_tags: 0x%x, "
8790 "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8791 #endif
8792
8793 /* Send up finished packets with SATA_PKT_COMPLETED */
8794 while (finished_tags) {
8795 tmp_slot = ddi_ffs(finished_tags) - 1;
8796 if (tmp_slot == -1) {
8797 break;
8798 }
8799
8800 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8801 ASSERT(satapkt != NULL);
8802
8803 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8804 "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8805 (void *)satapkt);
8806
8807 /*
8808 * Cannot fetch the return register content since the port
8809 * was restarted, so the corresponding tag will be set to
8810 * aborted tags.
8811 */
8812 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8813 CLEAR_BIT(finished_tags, tmp_slot);
8814 aborted_tags |= tmp_slot;
8815 continue;
8816 }
8817
8818 if (ncq_cmd_in_progress)
8819 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8820 tmp_slot);
8821 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8822 CLEAR_BIT(finished_tags, tmp_slot);
8823 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8824
8825 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8826 }
8827
8828 /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8829 while (failed_tags) {
8830 if (err_retri_cmd_in_progress) {
8831 satapkt = ahci_portp->ahciport_err_retri_pkt;
8832 ASSERT(satapkt != NULL);
8833 ASSERT(failed_tags == 0x1);
8834
8835 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8836 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8837 (void *)satapkt);
8838 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8839 break;
8840 }
8841 if (rdwr_pmult_cmd_in_progress) {
8842 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8843 ASSERT(satapkt != NULL);
8844 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8845 "ahci_mop_commands: sending up "
8846 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8847 (void *)satapkt);
8848 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8849 break;
8850 }
8851
8852 tmp_slot = ddi_ffs(failed_tags) - 1;
8853 if (tmp_slot == -1) {
8854 break;
8855 }
8856
8857 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8858 ASSERT(satapkt != NULL);
8859
8860 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8861 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8862 (void *)satapkt);
8863
8864 if (ncq_cmd_in_progress)
8865 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8866 tmp_slot);
8867 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8868 CLEAR_BIT(failed_tags, tmp_slot);
8869 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8870
8871 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8872 }
8873
8874 /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8875 while (timeout_tags) {
8876 if (err_retri_cmd_in_progress) {
8877 satapkt = ahci_portp->ahciport_err_retri_pkt;
8878 ASSERT(satapkt != NULL);
8879 ASSERT(timeout_tags == 0x1);
8880
8881 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8882 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8883 (void *)satapkt);
8884 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8885 break;
8886 }
8887 if (rdwr_pmult_cmd_in_progress) {
8888 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8889 ASSERT(satapkt != NULL);
8890 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8891 "ahci_mop_commands: sending up "
8892 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8893 (void *)satapkt);
8894 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8895 break;
8896 }
8897
8898 tmp_slot = ddi_ffs(timeout_tags) - 1;
8899 if (tmp_slot == -1) {
8900 break;
8901 }
8902
8903 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8904 ASSERT(satapkt != NULL);
8905
8906 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8907 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8908 (void *)satapkt);
8909
8910 if (ncq_cmd_in_progress)
8911 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8912 tmp_slot);
8913 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8914 CLEAR_BIT(timeout_tags, tmp_slot);
8915 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8916
8917 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8918 }
8919
8920 /* Send up aborted packets with SATA_PKT_ABORTED */
8921 while (aborted_tags) {
8922 if (err_retri_cmd_in_progress) {
8923 satapkt = ahci_portp->ahciport_err_retri_pkt;
8924 ASSERT(satapkt != NULL);
8925 ASSERT(aborted_tags == 0x1);
8926
8927 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8928 "sending up pkt 0x%p with SATA_PKT_ABORTED",
8929 (void *)satapkt);
8930 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8931 break;
8932 }
8933 if (rdwr_pmult_cmd_in_progress) {
8934 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8935 ASSERT(satapkt != NULL);
8936 ASSERT(aborted_tags == 0x1);
8937 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8938 "ahci_mop_commands: sending up "
8939 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
8940 (void *)satapkt);
8941 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8942 break;
8943 }
8944
8945 tmp_slot = ddi_ffs(aborted_tags) - 1;
8946 if (tmp_slot == -1) {
8947 break;
8948 }
8949
8950 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8951 ASSERT(satapkt != NULL);
8952
8953 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8954 "sending up pkt 0x%p with SATA_PKT_ABORTED",
8955 (void *)satapkt);
8956
8957 if (ncq_cmd_in_progress)
8958 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8959 tmp_slot);
8960 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8961 CLEAR_BIT(aborted_tags, tmp_slot);
8962 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8963
8964 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8965 }
8966
8967 /* Send up reset packets with SATA_PKT_RESET. */
8968 while (reset_tags) {
8969 if (rdwr_pmult_cmd_in_progress) {
8970 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8971 ASSERT(satapkt != NULL);
8972 ASSERT(aborted_tags == 0x1);
8973 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8974 "ahci_mop_commands: sending up "
8975 "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
8976 (void *)satapkt);
8977 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
8978 break;
8979 }
8980
8981 tmp_slot = ddi_ffs(reset_tags) - 1;
8982 if (tmp_slot == -1) {
8983 break;
8984 }
8985
8986 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8987 ASSERT(satapkt != NULL);
8988
8989 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8990 "sending up pkt 0x%p with SATA_PKT_RESET",
8991 (void *)satapkt);
8992
8993 if (ncq_cmd_in_progress)
8994 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8995 tmp_slot);
8996 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8997 CLEAR_BIT(reset_tags, tmp_slot);
8998 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8999
9000 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9001 }
9002
9003 /* Send up unfinished packets with SATA_PKT_RESET */
9004 while (unfinished_tags) {
9005 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9006 if (tmp_slot == -1) {
9007 break;
9008 }
9009
9010 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9011 ASSERT(satapkt != NULL);
9012
9013 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9014 "sending up pkt 0x%p with SATA_PKT_RESET",
9015 (void *)satapkt);
9016
9017 if (ncq_cmd_in_progress)
9018 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9019 tmp_slot);
9020 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9021 CLEAR_BIT(unfinished_tags, tmp_slot);
9022 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9023
9024 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9025 }
9026
9027 ahci_portp->ahciport_mop_in_progress--;
9028 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9029
9030 if (ahci_portp->ahciport_mop_in_progress == 0)
9031 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9032
9033 ahci_flush_doneq(ahci_portp);
9034 }
9035
9036 /*
9037 * This routine is going to first request a READ LOG EXT sata pkt from sata
9038 * module, and then deliver it to the HBA to get the ncq failure context.
9039 * The return value is the exactly failed tags.
9040 */
9041 static uint32_t
9042 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9043 uint8_t port)
9044 {
9045 sata_device_t sdevice;
9046 sata_pkt_t *rdlog_spkt, *spkt;
9047 ddi_dma_handle_t buf_dma_handle;
9048 ahci_addr_t addr;
9049 int loop_count;
9050 int rval;
9051 int failed_slot;
9052 uint32_t failed_tags = 0;
9053 struct sata_ncq_error_recovery_page *ncq_err_page;
9054
9055 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9056 "ahci_get_rdlogext_data enter: port %d", port);
9057
9058 /* Prepare the sdevice data */
9059 bzero((void *)&sdevice, sizeof (sata_device_t));
9060 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9061
9062 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9063 sdevice.satadev_addr.pmport = 0;
9064
9065 /* Translate sata_device.satadev_addr -> ahci_addr */
9066 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9067
9068 /*
9069 * Call the sata hba interface to get a rdlog spkt
9070 */
9071 loop_count = 0;
9072 loop:
9073 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9074 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9075 if (rdlog_spkt == NULL) {
9076 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9077 /* Sleep for a while */
9078 drv_usecwait(AHCI_10MS_USECS);
9079 goto loop;
9080 }
9081 /* Timed out after 1s */
9082 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9083 "failed to get rdlog spkt for port %d", port);
9084 return (failed_tags);
9085 }
9086
9087 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9088
9089 /*
9090 * This flag is used to handle the specific error recovery when the
9091 * READ LOG EXT command gets a failure (fatal error or time-out).
9092 */
9093 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9094
9095 /*
9096 * This start is not supposed to fail because after port is restarted,
9097 * the whole command list is empty.
9098 */
9099 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9100 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9101 ahci_portp->ahciport_err_retri_pkt = NULL;
9102
9103 /* Remove the flag after READ LOG EXT command is completed */
9104 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9105
9106 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9107 /* Update the request log data */
9108 buf_dma_handle = *(ddi_dma_handle_t *)
9109 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9110 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9111 DDI_DMA_SYNC_FORKERNEL);
9112 if (rval == DDI_SUCCESS) {
9113 ncq_err_page =
9114 (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9115 satapkt_cmd.satacmd_bp->b_un.b_addr;
9116
9117 /* Get the failed tag */
9118 failed_slot = ncq_err_page->ncq_tag;
9119 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9120 "ahci_get_rdlogext_data: port %d "
9121 "failed slot %d", port, failed_slot);
9122 if (failed_slot & NQ) {
9123 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9124 "the failed slot is not a valid tag", NULL);
9125 goto out;
9126 }
9127
9128 failed_slot &= NCQ_TAG_MASK;
9129 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9130 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9131 "ahci_get_rdlogext_data: failed spkt 0x%p",
9132 (void *)spkt);
9133 if (spkt == NULL) {
9134 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9135 "the failed slot spkt is NULL", NULL);
9136 goto out;
9137 }
9138
9139 failed_tags = 0x1 << failed_slot;
9140
9141 /* Fill out the error context */
9142 ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9143 ncq_err_page);
9144 ahci_update_sata_registers(ahci_ctlp, port,
9145 &spkt->satapkt_device);
9146 }
9147 }
9148 out:
9149 sata_free_error_retrieval_pkt(rdlog_spkt);
9150
9151 return (failed_tags);
9152 }
9153
9154 /*
9155 * This routine is going to first request a REQUEST SENSE sata pkt from sata
9156 * module, and then deliver it to the HBA to get the sense data and copy
9157 * the sense data back to the orignal failed sata pkt, and free the REQUEST
9158 * SENSE sata pkt later.
9159 */
9160 static void
9161 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9162 uint8_t port, sata_pkt_t *spkt)
9163 {
9164 sata_device_t sdevice;
9165 sata_pkt_t *rs_spkt;
9166 sata_cmd_t *sata_cmd;
9167 ddi_dma_handle_t buf_dma_handle;
9168 ahci_addr_t addr;
9169 int loop_count;
9170 #if AHCI_DEBUG
9171 struct scsi_extended_sense *rqsense;
9172 #endif
9173
9174 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9175 "ahci_get_rqsense_data enter: port %d", port);
9176
9177 /* Prepare the sdevice data */
9178 bzero((void *)&sdevice, sizeof (sata_device_t));
9179 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9180
9181 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9182 sdevice.satadev_addr.pmport = 0;
9183
9184 /* Translate sata_device.satadev_addr -> ahci_addr */
9185 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9186
9187 sata_cmd = &spkt->satapkt_cmd;
9188
9189 /*
9190 * Call the sata hba interface to get a rs spkt
9191 */
9192 loop_count = 0;
9193 loop:
9194 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9195 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9196 if (rs_spkt == NULL) {
9197 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9198 /* Sleep for a while */
9199 drv_usecwait(AHCI_10MS_USECS);
9200 goto loop;
9201
9202 }
9203 /* Timed out after 1s */
9204 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9205 "failed to get rs spkt for port %d", port);
9206 return;
9207 }
9208
9209 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9210
9211 /*
9212 * This flag is used to handle the specific error recovery when the
9213 * REQUEST SENSE command gets a faiure (fatal error or time-out).
9214 */
9215 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9216
9217 /*
9218 * This start is not supposed to fail because after port is restarted,
9219 * the whole command list is empty.
9220 */
9221 ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9222 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9223 ahci_portp->ahciport_err_retri_pkt = NULL;
9224
9225 /* Remove the flag after REQUEST SENSE command is completed */
9226 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9227
9228 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9229 /* Update the request sense data */
9230 buf_dma_handle = *(ddi_dma_handle_t *)
9231 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9232 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9233 DDI_DMA_SYNC_FORKERNEL);
9234 /* Copy the request sense data */
9235 bcopy(rs_spkt->
9236 satapkt_cmd.satacmd_bp->b_un.b_addr,
9237 &sata_cmd->satacmd_rqsense,
9238 SATA_ATAPI_MIN_RQSENSE_LEN);
9239 #if AHCI_DEBUG
9240 rqsense = (struct scsi_extended_sense *)
9241 sata_cmd->satacmd_rqsense;
9242
9243 /* Dump the sense data */
9244 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9245 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9246 "Sense data for satapkt %p ATAPI cmd 0x%x",
9247 spkt, sata_cmd->satacmd_acdb[0]);
9248 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9249 " es_code 0x%x es_class 0x%x "
9250 "es_key 0x%x es_add_code 0x%x "
9251 "es_qual_code 0x%x",
9252 rqsense->es_code, rqsense->es_class,
9253 rqsense->es_key, rqsense->es_add_code,
9254 rqsense->es_qual_code);
9255 #endif
9256 }
9257
9258 sata_free_error_retrieval_pkt(rs_spkt);
9259 }
9260
9261 /*
9262 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9263 * the port must be restarted. When the HBA detects thus error, it may try
9264 * to abort a transfer. And if the transfer was aborted, the device is
9265 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9266 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9267 * that the device is in a stable status and transfers may be restarted without
9268 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9269 * then the software will send the COMRESET to do the port reset.
9270 *
9271 * Software should perform the appropriate error recovery actions based on
9272 * whether non-queued commands were being issued or natived command queuing
9273 * commands were being issued.
9274 *
9275 * And software will complete the command that had the error with error mark
9276 * to higher level software.
9277 *
9278 * Fatal errors include the following:
9279 * PxIS.IFS - Interface Fatal Error Status
9280 * PxIS.HBDS - Host Bus Data Error Status
9281 * PxIS.HBFS - Host Bus Fatal Error Status
9282 * PxIS.TFES - Task File Error Status
9283 */
9284 static void
9285 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9286 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9287 {
9288 uint32_t port_cmd_status;
9289 uint32_t slot_status = 0;
9290 uint32_t failed_tags = 0;
9291 int failed_slot;
9292 int reset_flag = 0, flag = 0;
9293 ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9294 sata_cmd_t *sata_cmd = NULL;
9295 sata_pkt_t *spkt = NULL;
9296 #if AHCI_DEBUG
9297 ahci_cmd_header_t *cmd_header;
9298 #endif
9299 uint8_t port = addrp->aa_port;
9300 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9301 int rval;
9302
9303 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9304
9305 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9306 "ahci_fatal_error_recovery_handler enter: port %d", port);
9307
9308 /* Port multiplier error */
9309 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9310 /* FBS code is neither completed nor tested. */
9311 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9312 port, intr_status);
9313
9314 /* Force a port reset */
9315 flag = AHCI_PORT_RESET;
9316 }
9317
9318 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9319 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9320
9321 /* Read PxCI to see which commands are still outstanding */
9322 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9323 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9324
9325 /*
9326 * Read PxCMD.CCS to determine the slot that the HBA
9327 * was processing when the error occurred.
9328 */
9329 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9330 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9331 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9332 AHCI_CMD_STATUS_CCS_SHIFT;
9333
9334 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9335 spkt = ahci_portp->ahciport_err_retri_pkt;
9336 ASSERT(spkt != NULL);
9337 } else {
9338 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9339 if (spkt == NULL) {
9340 /* May happen when interface errors occur? */
9341 goto next;
9342 }
9343 }
9344
9345 #if AHCI_DEBUG
9346 /*
9347 * Debugging purpose...
9348 */
9349 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9350 cmd_header =
9351 &ahci_portp->ahciport_cmd_list[failed_slot];
9352 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9353 "ahci_fatal_error_recovery_handler: port %d, "
9354 "PRD Byte Count = 0x%x, "
9355 "ahciport_prd_bytecounts = 0x%x", port,
9356 cmd_header->ahcich_prd_byte_count,
9357 ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9358 }
9359 #endif
9360
9361 sata_cmd = &spkt->satapkt_cmd;
9362
9363 /* Fill out the status and error registers for PxIS.TFES */
9364 if (intr_status & AHCI_INTR_STATUS_TFES) {
9365 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9366 ahcirf_d2h_register_fis);
9367
9368 /* Copy the error context back to the sata_cmd */
9369 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9370 }
9371
9372 /* The failed command must be one of the outstanding commands */
9373 failed_tags = 0x1 << failed_slot;
9374 ASSERT(failed_tags & slot_status);
9375
9376 /* Update the sata registers, especially PxSERR register */
9377 ahci_update_sata_registers(ahci_ctlp, port,
9378 &spkt->satapkt_device);
9379
9380 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9381 /* Read PxSACT to see which commands are still outstanding */
9382 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9383 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9384 }
9385 next:
9386
9387 #if AHCI_DEBUG
9388 /*
9389 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9390 * set, it means a fatal error happened after REQUEST SENSE command
9391 * or READ LOG EXT command is delivered to the HBA during the error
9392 * recovery process. At this time, the only outstanding command is
9393 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9394 */
9395 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9396 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9397 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9398 "command or READ LOG EXT command for error data retrieval "
9399 "failed", port);
9400 ASSERT(slot_status == 0x1);
9401 ASSERT(failed_slot == 0);
9402 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9403 SCMD_REQUEST_SENSE ||
9404 spkt->satapkt_cmd.satacmd_cmd_reg ==
9405 SATAC_READ_LOG_EXT);
9406 }
9407 #endif
9408
9409 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9410 ahci_portp->ahciport_mop_in_progress++;
9411
9412 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9413 port, flag, &reset_flag);
9414
9415 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9416 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9417 if (rval == AHCI_SUCCESS)
9418 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9419 "succeed", instance, port);
9420 else
9421 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9422 "failed", instance, port);
9423 }
9424
9425 /*
9426 * Won't retrieve error information:
9427 * 1. Port reset was involved to recover
9428 * 2. Device is gone
9429 * 3. IDENTIFY DEVICE command sent to ATAPI device
9430 * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9431 */
9432 if (reset_flag ||
9433 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9434 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9435 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9436 goto out;
9437
9438 /*
9439 * Deliver READ LOG EXT to gather information about the error when
9440 * a COMRESET has not been performed as part of the error recovery
9441 * during NCQ command processing.
9442 */
9443 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9444 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9445 ahci_portp, port);
9446 goto out;
9447 }
9448
9449 /*
9450 * Deliver REQUEST SENSE for ATAPI command to gather information about
9451 * the error when a COMRESET has not been performed as part of the
9452 * error recovery.
9453 */
9454 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9455 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9456 out:
9457 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9458 "ahci_fatal_error_recovery_handler: port %d fatal error "
9459 "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9460 "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9461 port, slot_status, ahci_portp->ahciport_pending_tags,
9462 ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9463
9464 ahci_mop_commands(ahci_ctlp,
9465 ahci_portp,
9466 slot_status,
9467 failed_tags, /* failed tags */
9468 0, /* timeout tags */
9469 0, /* aborted tags */
9470 0); /* reset tags */
9471 }
9472
9473 /*
9474 * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9475 * 1. device specific.PxFBS.SDE=1
9476 * 2. Non-Deivce specific.
9477 * Nothing will be done when Command-based switching is employed.
9478 *
9479 * Currently code is neither completed nor tested.
9480 */
9481 static void
9482 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9483 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9484 {
9485 #ifndef __lock_lint
9486 _NOTE(ARGUNUSED(intr_status))
9487 #endif
9488 uint32_t port_fbs_ctrl;
9489 int loop_count = 0;
9490 ahci_addr_t addr;
9491
9492 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9493
9494 /* Nothing will be done under Command-based switching. */
9495 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9496 return;
9497
9498 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9499 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9500
9501 if (!(port_fbs_ctrl & AHCI_FBS_EN))
9502 /* FBS is not enabled. */
9503 return;
9504
9505 /* Problem's getting complicated now. */
9506 /*
9507 * If FIS-based switching is used, we need to check
9508 * the PxFBS to see the error type.
9509 */
9510 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9511 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9512
9513 /* Refer to spec(v1.2) 9.3.6.1 */
9514 if (port_fbs_ctrl & AHCI_FBS_SDE) {
9515 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9516 "A Device Sepcific Error: port %d", port);
9517 /*
9518 * Controller has paused commands for all other
9519 * sub-devices until PxFBS.DEC is set.
9520 */
9521 ahci_reject_all_abort_pkts(ahci_ctlp,
9522 ahci_portp, 0);
9523
9524 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9525 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9526 port_fbs_ctrl | AHCI_FBS_DEC);
9527
9528 /*
9529 * Wait controller clear PxFBS.DEC,
9530 * then we can continue.
9531 */
9532 loop_count = 0;
9533 do {
9534 port_fbs_ctrl = ddi_get32(ahci_ctlp->
9535 ahcictl_ahci_acc_handle, (uint32_t *)
9536 AHCI_PORT_PxFBS(ahci_ctlp, port));
9537
9538 if (loop_count++ > 1000)
9539 /*
9540 * Esclate the error. Follow
9541 * non-device specific error
9542 * procedure.
9543 */
9544 return;
9545
9546 drv_usecwait(AHCI_100US_USECS);
9547 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9548
9549 /*
9550 * Issue a software reset to ensure drive is in
9551 * a known state.
9552 */
9553 (void) ahci_software_reset(ahci_ctlp,
9554 ahci_portp, &addr);
9555
9556 } else {
9557
9558 /* Process Non-Device Specific Error. */
9559 /* This will be handled later on. */
9560 cmn_err(CE_NOTE, "!FBS is not supported now.");
9561 }
9562 }
9563 /*
9564 * Handle events - fatal error recovery
9565 */
9566 static void
9567 ahci_events_handler(void *args)
9568 {
9569 ahci_event_arg_t *ahci_event_arg;
9570 ahci_ctl_t *ahci_ctlp;
9571 ahci_port_t *ahci_portp;
9572 ahci_addr_t *addrp;
9573 uint32_t event;
9574 int instance;
9575
9576 ahci_event_arg = (ahci_event_arg_t *)args;
9577
9578 ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9579 ahci_portp = ahci_event_arg->ahciea_portp;
9580 addrp = ahci_event_arg->ahciea_addrp;
9581 event = ahci_event_arg->ahciea_event;
9582 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9583
9584 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9585 "ahci_events_handler enter: port %d intr_status = 0x%x",
9586 ahci_portp->ahciport_port_num, event);
9587
9588 mutex_enter(&ahci_portp->ahciport_mutex);
9589
9590 /*
9591 * ahci_intr_phyrdy_change() may have rendered it to
9592 * SATA_DTYPE_NONE.
9593 */
9594 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9595 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9596 "ahci_events_handler: port %d no device attached, "
9597 "and just return without doing anything",
9598 ahci_portp->ahciport_port_num);
9599
9600 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9601 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9602 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9603 "succeed", instance, ahci_portp->ahciport_port_num);
9604 }
9605
9606 goto out;
9607 }
9608
9609 if (event & (AHCI_INTR_STATUS_IFS |
9610 AHCI_INTR_STATUS_HBDS |
9611 AHCI_INTR_STATUS_HBFS |
9612 AHCI_INTR_STATUS_TFES))
9613 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9614 addrp, event);
9615
9616 out:
9617 mutex_exit(&ahci_portp->ahciport_mutex);
9618 }
9619
9620 /*
9621 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9622 * detect there are some commands which are timed out.
9623 */
9624 static void
9625 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9626 uint8_t port, uint32_t tmp_timeout_tags)
9627 {
9628 uint32_t slot_status = 0;
9629 uint32_t finished_tags = 0;
9630 uint32_t timeout_tags = 0;
9631
9632 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9633 "ahci_timeout_pkts enter: port %d", port);
9634
9635 mutex_enter(&ahci_portp->ahciport_mutex);
9636
9637 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9638 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9639 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9640 /* Read PxCI to see which commands are still outstanding */
9641 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9642 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9643 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9644 /* Read PxSACT to see which commands are still outstanding */
9645 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9646 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9647 }
9648
9649 #if AHCI_DEBUG
9650 /*
9651 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9652 * set, it means a fatal error happened after REQUEST SENSE command
9653 * or READ LOG EXT command is delivered to the HBA during the error
9654 * recovery process. At this time, the only outstanding command is
9655 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9656 */
9657 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9658 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9659 "ahci_timeout_pkts called while REQUEST SENSE "
9660 "command or READ LOG EXT command for error recovery "
9661 "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9662 "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9663 tmp_timeout_tags, slot_status,
9664 ahci_portp->ahciport_pending_tags,
9665 ahci_portp->ahciport_pending_ncq_tags);
9666 ASSERT(slot_status == 0x1);
9667 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9668 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9669 "ahci_timeout_pkts called while executing R/W PMULT "
9670 "command timeout_tags = 0x%x, slot_status = 0x%x",
9671 tmp_timeout_tags, slot_status);
9672 ASSERT(slot_status == 0x1);
9673 }
9674 #endif
9675
9676 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9677 ahci_portp->ahciport_mop_in_progress++;
9678
9679 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9680 port, AHCI_PORT_RESET, NULL);
9681
9682 /*
9683 * Re-identify timeout tags because some previously checked commands
9684 * could already complete.
9685 */
9686 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9687 finished_tags = ahci_portp->ahciport_pending_tags &
9688 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9689 timeout_tags = tmp_timeout_tags & ~finished_tags;
9690
9691 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9692 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9693 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9694 "pending_tags = 0x%x ",
9695 port, finished_tags, timeout_tags,
9696 slot_status, ahci_portp->ahciport_pending_tags);
9697 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9698 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9699 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9700 timeout_tags = tmp_timeout_tags & ~finished_tags;
9701
9702 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9703 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9704 "timeout_tags = 0x%x, port_sactive = 0x%x, "
9705 "pending_ncq_tags = 0x%x ",
9706 port, finished_tags, timeout_tags,
9707 slot_status, ahci_portp->ahciport_pending_ncq_tags);
9708 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9709 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9710 timeout_tags = tmp_timeout_tags;
9711 }
9712
9713 ahci_mop_commands(ahci_ctlp,
9714 ahci_portp,
9715 slot_status,
9716 0, /* failed tags */
9717 timeout_tags, /* timeout tags */
9718 0, /* aborted tags */
9719 0); /* reset tags */
9720
9721 mutex_exit(&ahci_portp->ahciport_mutex);
9722 }
9723
9724 /*
9725 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9726 * for long time.
9727 */
9728 static void
9729 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9730 {
9731 ahci_port_t *ahci_portp;
9732 sata_pkt_t *spkt;
9733 uint32_t pending_tags;
9734 uint32_t timeout_tags;
9735 uint32_t port_cmd_status;
9736 uint32_t port_sactive;
9737 uint8_t port;
9738 int tmp_slot;
9739 int current_slot;
9740 uint32_t current_tags;
9741 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9742
9743 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9744
9745 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9746 "ahci_watchdog_handler entered", NULL);
9747
9748 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9749 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9750 continue;
9751 }
9752
9753 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9754
9755 mutex_enter(&ahci_portp->ahciport_mutex);
9756 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9757 mutex_exit(&ahci_portp->ahciport_mutex);
9758 continue;
9759 }
9760
9761 /* Skip the check for those ports in error recovery */
9762 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9763 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9764 mutex_exit(&ahci_portp->ahciport_mutex);
9765 continue;
9766 }
9767
9768 pending_tags = 0;
9769 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9770 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9771
9772 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9773 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9774 current_slot = 0;
9775 pending_tags = 0x1;
9776 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9777 current_slot =
9778 (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9779 AHCI_CMD_STATUS_CCS_SHIFT;
9780 pending_tags = ahci_portp->ahciport_pending_tags;
9781 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9782 port_sactive = ddi_get32(
9783 ahci_ctlp->ahcictl_ahci_acc_handle,
9784 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9785 current_tags = port_sactive &
9786 ~port_cmd_status &
9787 AHCI_NCQ_SLOT_MASK(ahci_portp);
9788 pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9789 }
9790
9791 timeout_tags = 0;
9792 while (pending_tags) {
9793 tmp_slot = ddi_ffs(pending_tags) - 1;
9794 if (tmp_slot == -1) {
9795 break;
9796 }
9797
9798 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9799 spkt = ahci_portp->ahciport_err_retri_pkt;
9800 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9801 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9802 else
9803 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9804
9805 if ((spkt != NULL) && spkt->satapkt_time &&
9806 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9807 /*
9808 * If a packet has survived for more than it's
9809 * max life cycles, it is a candidate for time
9810 * out.
9811 */
9812 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9813 ahci_watchdog_timeout;
9814
9815 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9816 > 0)
9817 goto next;
9818
9819 #if AHCI_DEBUG
9820 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9821 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9822 ahci_ctlp, "watchdog: the current "
9823 "tags is 0x%x", current_tags);
9824 } else {
9825 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9826 ahci_ctlp, "watchdog: the current "
9827 "slot is %d", current_slot);
9828 }
9829 #endif
9830
9831 /*
9832 * We need to check whether the HBA has
9833 * begun to execute the command, if not,
9834 * then re-set the timer of the command.
9835 */
9836 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9837 (tmp_slot != current_slot) ||
9838 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9839 ((0x1 << tmp_slot) & current_tags)) {
9840 ahci_portp->ahciport_slot_timeout \
9841 [tmp_slot] = spkt->satapkt_time;
9842 } else {
9843 timeout_tags |= (0x1 << tmp_slot);
9844 cmn_err(CE_WARN, "!ahci%d: watchdog "
9845 "port %d satapkt 0x%p timed out\n",
9846 instance, port, (void *)spkt);
9847 }
9848 }
9849 next:
9850 CLEAR_BIT(pending_tags, tmp_slot);
9851 }
9852
9853 if (timeout_tags) {
9854 mutex_exit(&ahci_portp->ahciport_mutex);
9855 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9856 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9857 port, timeout_tags);
9858 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9859 mutex_enter(&ahci_portp->ahciport_mutex);
9860 }
9861
9862 mutex_exit(&ahci_portp->ahciport_mutex);
9863 }
9864
9865 /* Re-install the watchdog timeout handler */
9866 if (ahci_ctlp->ahcictl_timeout_id != 0) {
9867 ahci_ctlp->ahcictl_timeout_id =
9868 timeout((void (*)(void *))ahci_watchdog_handler,
9869 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9870 }
9871
9872 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9873 }
9874
9875 /*
9876 * Fill the error context into sata_cmd for non-queued command error.
9877 */
9878 static void
9879 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9880 {
9881 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9882 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9883 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9884 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9885 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9886 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9887 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9888
9889 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9890 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9891 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9892 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9893 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9894 }
9895 }
9896
9897 /*
9898 * Fill the ncq error page into sata_cmd for queued command error.
9899 */
9900 static void
9901 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9902 struct sata_ncq_error_recovery_page *ncq_err_page)
9903 {
9904 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9905 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9906 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
9907 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
9908 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
9909 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
9910 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
9911 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
9912 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
9913 scmd->satacmd_status_reg = ncq_err_page->ncq_status;
9914 scmd->satacmd_error_reg = ncq_err_page->ncq_error;
9915 }
9916
9917 /*
9918 * Put the respective register value to sata_cmd_t for satacmd_flags.
9919 */
9920 static void
9921 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9922 {
9923 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
9924 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9925 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
9926 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9927 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
9928 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9929 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
9930 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9931 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
9932 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9933 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
9934 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9935 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
9936 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9937 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
9938 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9939 if (scmd->satacmd_flags.sata_copy_out_device_reg)
9940 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9941 if (scmd->satacmd_flags.sata_copy_out_error_reg)
9942 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9943 }
9944
9945 static void
9946 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
9947 uint32_t intr_status)
9948 {
9949 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9950
9951 if (intr_status & AHCI_INTR_STATUS_IFS)
9952 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
9953 "error", instance, port);
9954
9955 if (intr_status & AHCI_INTR_STATUS_HBDS)
9956 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
9957 instance, port);
9958
9959 if (intr_status & AHCI_INTR_STATUS_HBFS)
9960 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
9961 instance, port);
9962
9963 if (intr_status & AHCI_INTR_STATUS_TFES)
9964 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
9965 instance, port);
9966
9967 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
9968 "recovery", instance, port);
9969 }
9970
9971 static void
9972 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
9973 uint32_t slot_tags)
9974 {
9975 ahci_port_t *ahci_portp;
9976 int tmp_slot;
9977 sata_pkt_t *spkt;
9978 sata_cmd_t cmd;
9979
9980 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9981 ASSERT(ahci_portp != NULL);
9982
9983 while (slot_tags) {
9984 tmp_slot = ddi_ffs(slot_tags) - 1;
9985 if (tmp_slot == -1) {
9986 break;
9987 }
9988
9989 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9990 ASSERT(spkt != NULL);
9991 cmd = spkt->satapkt_cmd;
9992
9993 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
9994 "features_reg = 0x%x sec_count_msb = 0x%x "
9995 "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
9996 "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
9997 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
9998 "lba_high_lsb = 0x%x device_reg = 0x%x "
9999 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10000 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10001 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10002 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10003 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10004 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10005 cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10006 *((uint32_t *)&(cmd.satacmd_flags)));
10007
10008 CLEAR_BIT(slot_tags, tmp_slot);
10009 }
10010 }
10011
10012 /*
10013 * Dump the serror message to the log.
10014 */
10015 static void
10016 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10017 uint32_t port_serror, int debug_only)
10018 {
10019 static char err_buf[512];
10020 static char err_msg_header[16];
10021 char *err_msg = err_buf;
10022
10023 *err_buf = '\0';
10024 *err_msg_header = '\0';
10025
10026 if (port_serror & SERROR_DATA_ERR_FIXED) {
10027 err_msg = strcat(err_msg,
10028 "\tRecovered Data Integrity Error (I)\n");
10029 }
10030
10031 if (port_serror & SERROR_COMM_ERR_FIXED) {
10032 err_msg = strcat(err_msg,
10033 "\tRecovered Communication Error (M)\n");
10034 }
10035
10036 if (port_serror & SERROR_DATA_ERR) {
10037 err_msg = strcat(err_msg,
10038 "\tTransient Data Integrity Error (T)\n");
10039 }
10040
10041 if (port_serror & SERROR_PERSISTENT_ERR) {
10042 err_msg = strcat(err_msg,
10043 "\tPersistent Communication or Data Integrity Error (C)\n");
10044 }
10045
10046 if (port_serror & SERROR_PROTOCOL_ERR) {
10047 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10048 }
10049
10050 if (port_serror & SERROR_INT_ERR) {
10051 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10052 }
10053
10054 if (port_serror & SERROR_PHY_RDY_CHG) {
10055 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10056 }
10057
10058 if (port_serror & SERROR_PHY_INT_ERR) {
10059 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10060 }
10061
10062 if (port_serror & SERROR_COMM_WAKE) {
10063 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10064 }
10065
10066 if (port_serror & SERROR_10B_TO_8B_ERR) {
10067 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10068 }
10069
10070 if (port_serror & SERROR_DISPARITY_ERR) {
10071 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10072 }
10073
10074 if (port_serror & SERROR_CRC_ERR) {
10075 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10076 }
10077
10078 if (port_serror & SERROR_HANDSHAKE_ERR) {
10079 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10080 }
10081
10082 if (port_serror & SERROR_LINK_SEQ_ERR) {
10083 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10084 }
10085
10086 if (port_serror & SERROR_TRANS_ERR) {
10087 err_msg = strcat(err_msg,
10088 "\tTransport state transition error (T)\n");
10089 }
10090
10091 if (port_serror & SERROR_FIS_TYPE) {
10092 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10093 }
10094
10095 if (port_serror & SERROR_EXCHANGED_ERR) {
10096 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10097 }
10098
10099 if (*err_msg == '\0')
10100 return;
10101
10102 if (debug_only) {
10103 (void) sprintf(err_msg_header, "port %d", port);
10104 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10105 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10106 } else if (ahci_ctlp) {
10107 cmn_err(CE_WARN, "!ahci%d: %s %s",
10108 ddi_get_instance(ahci_ctlp->ahcictl_dip),
10109 err_msg_header, err_msg);
10110
10111 /* sata trace debug */
10112 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10113 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10114 err_msg_header, err_msg);
10115 } else {
10116 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10117
10118 /* sata trace debug */
10119 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10120 }
10121 }
10122
10123 /*
10124 * Translate the sata_address_t type into the ahci_addr_t type.
10125 * sata_device.satadev_addr structure is used as source.
10126 */
10127 static void
10128 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10129 ahci_addr_t *ahci_addrp)
10130 {
10131 sata_address_t *sata_addrp = &sd->satadev_addr;
10132 ahci_addrp->aa_port =
10133 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10134 ahci_addrp->aa_pmport = sata_addrp->pmport;
10135
10136 switch (sata_addrp->qual) {
10137 case SATA_ADDR_DCPORT:
10138 case SATA_ADDR_CPORT:
10139 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10140 break;
10141 case SATA_ADDR_PMULT:
10142 case SATA_ADDR_PMULT_SPEC:
10143 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10144 break;
10145 case SATA_ADDR_DPMPORT:
10146 case SATA_ADDR_PMPORT:
10147 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10148 break;
10149 case SATA_ADDR_NULL:
10150 default:
10151 /* something went wrong */
10152 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10153 break;
10154 }
10155 }
10156
10157 /*
10158 * This routine is to calculate the total number of ports implemented
10159 * by the HBA.
10160 */
10161 static int
10162 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10163 {
10164 uint8_t i;
10165 int num = 0;
10166
10167 for (i = 0; i < AHCI_MAX_PORTS; i++) {
10168 if (((uint32_t)0x1 << i) & ports_implemented)
10169 num++;
10170 }
10171
10172 return (num);
10173 }
10174
10175 #if AHCI_DEBUG
10176 static void
10177 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10178 {
10179 static char name[16];
10180 va_list ap;
10181
10182 mutex_enter(&ahci_log_mutex);
10183
10184 va_start(ap, fmt);
10185 if (ahci_ctlp) {
10186 (void) sprintf(name, "ahci%d: ",
10187 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10188 } else {
10189 (void) sprintf(name, "ahci: ");
10190 }
10191
10192 (void) vsprintf(ahci_log_buf, fmt, ap);
10193 va_end(ap);
10194
10195 cmn_err(level, "%s%s", name, ahci_log_buf);
10196
10197 mutex_exit(&ahci_log_mutex);
10198 }
10199 #endif
10200
10201 /*
10202 * quiesce(9E) entry point.
10203 *
10204 * This function is called when the system is single-threaded at high
10205 * PIL with preemption disabled. Therefore, this function must not be
10206 * blocked.
10207 *
10208 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10209 * DDI_FAILURE indicates an error condition and should almost never happen.
10210 */
10211 static int
10212 ahci_quiesce(dev_info_t *dip)
10213 {
10214 ahci_ctl_t *ahci_ctlp;
10215 ahci_port_t *ahci_portp;
10216 int instance, port;
10217
10218 instance = ddi_get_instance(dip);
10219 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10220
10221 if (ahci_ctlp == NULL)
10222 return (DDI_FAILURE);
10223
10224 #if AHCI_DEBUG
10225 ahci_debug_flags = 0;
10226 #endif
10227
10228 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10229
10230 /* disable all the interrupts. */
10231 ahci_disable_all_intrs(ahci_ctlp);
10232
10233 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10234 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10235 continue;
10236 }
10237
10238 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10239
10240 /*
10241 * Stop the port by clearing PxCMD.ST
10242 *
10243 * Here we must disable the port interrupt because
10244 * ahci_disable_all_intrs only clear GHC.IE, and IS
10245 * register will be still set if PxIE is enabled.
10246 * When ahci shares one IRQ with other drivers, the
10247 * intr handler may claim the intr mistakenly.
10248 */
10249 ahci_disable_port_intrs(ahci_ctlp, port);
10250 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10251 ahci_portp, port);
10252 }
10253
10254 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10255
10256 return (DDI_SUCCESS);
10257 }
10258
10259 /*
10260 * The function will add a sata packet to the done queue.
10261 */
10262 static void
10263 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10264 {
10265 ASSERT(satapkt != NULL);
10266 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10267
10268 /* set the reason for all packets */
10269 satapkt->satapkt_reason = reason;
10270 satapkt->satapkt_hba_driver_private = NULL;
10271
10272 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10273 satapkt->satapkt_comp) {
10274 /*
10275 * only add to queue when mode is not synch and there is
10276 * completion callback
10277 */
10278 *ahci_portp->ahciport_doneqtail = satapkt;
10279 ahci_portp->ahciport_doneqtail =
10280 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10281 ahci_portp->ahciport_doneq_len++;
10282
10283 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10284 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10285 /*
10286 * for sync/non-poll mode, just call cv_broadcast
10287 */
10288 cv_broadcast(&ahci_portp->ahciport_cv);
10289 }
10290
10291 /*
10292 * The function will call completion callback of sata packet on the
10293 * completed queue
10294 */
10295 static void
10296 ahci_flush_doneq(ahci_port_t *ahci_portp)
10297 {
10298 sata_pkt_t *satapkt, *next;
10299
10300 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10301
10302 if (ahci_portp->ahciport_doneq) {
10303 satapkt = ahci_portp->ahciport_doneq;
10304
10305 ahci_portp->ahciport_doneq = NULL;
10306 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10307 ahci_portp->ahciport_doneq_len = 0;
10308
10309 mutex_exit(&ahci_portp->ahciport_mutex);
10310
10311 while (satapkt != NULL) {
10312 next = satapkt->satapkt_hba_driver_private;
10313 satapkt->satapkt_hba_driver_private = NULL;
10314
10315 /* Call the callback */
10316 (*satapkt->satapkt_comp)(satapkt);
10317
10318 satapkt = next;
10319 }
10320
10321 mutex_enter(&ahci_portp->ahciport_mutex);
10322 }
10323 }