1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
25 */
26
27 /*
28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
29 *
30 * Power Management Support
31 * ------------------------
32 *
33 * At the moment, the ahci driver only implements suspend/resume to
34 * support Suspend to RAM on X86 feature. Device power management isn't
35 * implemented, link power management is disabled, and hot plug isn't
36 * allowed during the period from suspend to resume.
37 *
38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
39 * and DDI_RESUME entries, and don't need to take care of new requests
40 * sent down after suspend because the target driver (sd) has already
41 * handled these conditions, and blocked these requests. For the detailed
42 * information, please check with sdopen, sdclose and sdioctl routines.
43 *
44 */
45
46 #include <sys/note.h>
47 #include <sys/scsi/scsi.h>
48 #include <sys/pci.h>
49 #include <sys/disp.h>
50 #include <sys/sata/sata_hba.h>
51 #include <sys/sata/adapters/ahci/ahcireg.h>
52 #include <sys/sata/adapters/ahci/ahcivar.h>
53
54 /*
55 * FMA header files
56 */
57 #include <sys/ddifm.h>
58 #include <sys/fm/protocol.h>
59 #include <sys/fm/util.h>
60 #include <sys/fm/io/ddi.h>
61
62 /*
63 * This is the string displayed by modinfo, etc.
64 */
65 static char ahci_ident[] = "ahci driver";
66
67 /*
68 * Function prototypes for driver entry points
69 */
70 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
71 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
72 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
73 static int ahci_quiesce(dev_info_t *);
74
75 /*
76 * Function prototypes for SATA Framework interfaces
77 */
78 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
79 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
80
81 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
82 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
83 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
84 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
85 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
86 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
87 #if defined(__lock_lint)
88 static int ahci_selftest(dev_info_t *, sata_device_t *);
89 #endif
90
91 /*
92 * FMA Prototypes
93 */
94 static void ahci_fm_init(ahci_ctl_t *);
95 static void ahci_fm_fini(ahci_ctl_t *);
96 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
97 int ahci_check_acc_handle(ddi_acc_handle_t);
98 int ahci_check_dma_handle(ddi_dma_handle_t);
99 void ahci_fm_ereport(ahci_ctl_t *, char *);
100 static int ahci_check_all_handle(ahci_ctl_t *);
101 static int ahci_check_ctl_handle(ahci_ctl_t *);
102 static int ahci_check_port_handle(ahci_ctl_t *, int);
103 static int ahci_check_slot_handle(ahci_port_t *, int);
104
105 /*
106 * Local function prototypes
107 */
108 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
109 static int ahci_alloc_ports_state(ahci_ctl_t *);
110 static void ahci_dealloc_ports_state(ahci_ctl_t *);
111 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
112 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
113 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
114 static void ahci_dealloc_rcvd_fis(ahci_port_t *);
115 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
116 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
117 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
118 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
119 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
120 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
121
122 static int ahci_initialize_controller(ahci_ctl_t *);
123 static void ahci_uninitialize_controller(ahci_ctl_t *);
124 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
125 static int ahci_config_space_init(ahci_ctl_t *);
126 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
127
128 static void ahci_drain_ports_taskq(ahci_ctl_t *);
129 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
130 uint8_t);
131 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
132 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
133 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
134 sata_device_t *);
135 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
136 sata_pmult_gscr_t *);
137 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
138 sata_device_t *);
139 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
140 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
141 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
142 sata_device_t *);
143
144 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
145 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
146 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
147 ahci_addr_t *);
148 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
149 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
150 ahci_addr_t *, sata_pkt_t *);
151 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
152 ahci_addr_t *, sata_pkt_t *);
153 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
154 ahci_addr_t *, int);
155 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
156 static void ahci_copy_ncq_err_page(sata_cmd_t *,
157 struct sata_ncq_error_recovery_page *);
158 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
159 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
160 static void ahci_flush_doneq(ahci_port_t *);
161
162 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
163 static int ahci_hba_reset(ahci_ctl_t *);
164 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
165 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
166 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
167 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
168 ahci_addr_t *);
169 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
170 ahci_addr_t *);
171 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
172 ahci_addr_t *);
173 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
174 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
175 uint8_t);
176 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
177 uint8_t, int, int *);
178 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
179 uint32_t, uint32_t, uint32_t, uint32_t);
180 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
181 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
182 uint8_t, sata_pkt_t *);
183 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
184 ahci_addr_t *, uint32_t);
185 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
186 uint8_t, uint32_t);
187 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
188 uint8_t, uint32_t);
189 static void ahci_events_handler(void *);
190 static void ahci_watchdog_handler(ahci_ctl_t *);
191
192 static uint_t ahci_intr(caddr_t, caddr_t);
193 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
194 static int ahci_add_intrs(ahci_ctl_t *, int);
195 static void ahci_rem_intrs(ahci_ctl_t *);
196 static void ahci_enable_all_intrs(ahci_ctl_t *);
197 static void ahci_disable_all_intrs(ahci_ctl_t *);
198 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
199 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
200
201 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
202 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
203 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
204 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
205 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
206 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
207 ahci_port_t *, uint8_t);
208 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
209 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
210 uint8_t, uint32_t);
211 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
212 uint8_t, uint32_t);
213 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
214
215 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
216 static int ahci_get_num_implemented_ports(uint32_t);
217 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
218 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
219 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
220 #if AHCI_DEBUG
221 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
222 #endif
223
224
225 /*
226 * DMA attributes for the data buffer
227 *
228 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
229 * does not support 64-bit addressing
230 */
231 static ddi_dma_attr_t buffer_dma_attr = {
232 DMA_ATTR_V0, /* dma_attr_version */
233 0x0ull, /* dma_attr_addr_lo: lowest bus address */
234 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
235 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */
236 0x2ull, /* dma_attr_align: word aligned */
237 1, /* dma_attr_burstsizes */
238 1, /* dma_attr_minxfer */
239 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
240 0xffffffffull, /* dma_attr_seg */
241 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */
242 512, /* dma_attr_granular */
243 0, /* dma_attr_flags */
244 };
245
246 /*
247 * DMA attributes for the rcvd FIS
248 *
249 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
250 * does not support 64-bit addressing
251 */
252 static ddi_dma_attr_t rcvd_fis_dma_attr = {
253 DMA_ATTR_V0, /* dma_attr_version */
254 0x0ull, /* dma_attr_addr_lo: lowest bus address */
255 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
256 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
257 0x100ull, /* dma_attr_align: 256-byte aligned */
258 1, /* dma_attr_burstsizes */
259 1, /* dma_attr_minxfer */
260 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
261 0xffffffffull, /* dma_attr_seg */
262 1, /* dma_attr_sgllen */
263 1, /* dma_attr_granular */
264 0, /* dma_attr_flags */
265 };
266
267 /*
268 * DMA attributes for the command list
269 *
270 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
271 * does not support 64-bit addressing
272 */
273 static ddi_dma_attr_t cmd_list_dma_attr = {
274 DMA_ATTR_V0, /* dma_attr_version */
275 0x0ull, /* dma_attr_addr_lo: lowest bus address */
276 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
277 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
278 0x400ull, /* dma_attr_align: 1K-byte aligned */
279 1, /* dma_attr_burstsizes */
280 1, /* dma_attr_minxfer */
281 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
282 0xffffffffull, /* dma_attr_seg */
283 1, /* dma_attr_sgllen */
284 1, /* dma_attr_granular */
285 0, /* dma_attr_flags */
286 };
287
288 /*
289 * DMA attributes for cmd tables
290 *
291 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
292 * does not support 64-bit addressing
293 */
294 static ddi_dma_attr_t cmd_table_dma_attr = {
295 DMA_ATTR_V0, /* dma_attr_version */
296 0x0ull, /* dma_attr_addr_lo: lowest bus address */
297 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
298 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
299 0x80ull, /* dma_attr_align: 128-byte aligned */
300 1, /* dma_attr_burstsizes */
301 1, /* dma_attr_minxfer */
302 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
303 0xffffffffull, /* dma_attr_seg */
304 1, /* dma_attr_sgllen */
305 1, /* dma_attr_granular */
306 0, /* dma_attr_flags */
307 };
308
309
310 /* Device access attributes */
311 static ddi_device_acc_attr_t accattr = {
312 DDI_DEVICE_ATTR_V1,
313 DDI_STRUCTURE_LE_ACC,
314 DDI_STRICTORDER_ACC,
315 DDI_DEFAULT_ACC
316 };
317
318
319 static struct dev_ops ahcictl_dev_ops = {
320 DEVO_REV, /* devo_rev */
321 0, /* refcnt */
322 ahci_getinfo, /* info */
323 nulldev, /* identify */
324 nulldev, /* probe */
325 ahci_attach, /* attach */
326 ahci_detach, /* detach */
327 nodev, /* no reset */
328 (struct cb_ops *)0, /* driver operations */
329 NULL, /* bus operations */
330 NULL, /* power */
331 ahci_quiesce, /* quiesce */
332 };
333
334 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
335 SATA_TRAN_HOTPLUG_OPS_REV_1,
336 ahci_tran_hotplug_port_activate,
337 ahci_tran_hotplug_port_deactivate
338 };
339
340 extern struct mod_ops mod_driverops;
341
342 static struct modldrv modldrv = {
343 &mod_driverops, /* driverops */
344 ahci_ident, /* short description */
345 &ahcictl_dev_ops, /* driver ops */
346 };
347
348 static struct modlinkage modlinkage = {
349 MODREV_1,
350 &modldrv,
351 NULL
352 };
353
354 /* The following variables are watchdog handler related */
355 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
356 static clock_t ahci_watchdog_tick;
357
358 /*
359 * This static variable indicates the size of command table,
360 * and it's changeable with prdt number, which ahci_dma_prdt_number
361 * indicates.
362 */
363 static size_t ahci_cmd_table_size;
364
365 /*
366 * The below global variables are tunable via /etc/system
367 *
368 * ahci_dma_prdt_number
369 * ahci_msi_enabled
370 * ahci_buf_64bit_dma
371 * ahci_commu_64bit_dma
372 */
373
374 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
375 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
376
377 /* AHCI MSI is tunable */
378 boolean_t ahci_msi_enabled = B_TRUE;
379
380 /*
381 * 64-bit dma addressing for data buffer is tunable
382 *
383 * The variable controls only the below value:
384 * DBAU (upper 32-bits physical address of data block)
385 */
386 boolean_t ahci_buf_64bit_dma = B_TRUE;
387
388 /*
389 * 64-bit dma addressing for communication system descriptors is tunable
390 *
391 * The variable controls the below three values:
392 *
393 * PxCLBU (upper 32-bits for the command list base physical address)
394 * PxFBU (upper 32-bits for the received FIS base physical address)
395 * CTBAU (upper 32-bits of command table base)
396 */
397 boolean_t ahci_commu_64bit_dma = B_TRUE;
398
399 /*
400 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
401 * chipset. If the users want to have a try with 64-bit dma, please change
402 * the below variable value to enable it.
403 */
404 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
405
406 /*
407 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
408 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
409 * please change the below value to enable it.
410 */
411 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
412
413
414 /*
415 * End of global tunable variable definition
416 */
417
418 #if AHCI_DEBUG
419 uint32_t ahci_debug_flags = 0;
420 #else
421 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
422 #endif
423
424
425 #if AHCI_DEBUG
426 /* The following is needed for ahci_log() */
427 static kmutex_t ahci_log_mutex;
428 static char ahci_log_buf[512];
429 #endif
430
431 /* Opaque state pointer initialized by ddi_soft_state_init() */
432 static void *ahci_statep = NULL;
433
434 /*
435 * ahci module initialization.
436 */
437 int
438 _init(void)
439 {
440 int ret;
441
442 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
443 if (ret != 0) {
444 goto err_out;
445 }
446
447 #if AHCI_DEBUG
448 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
449 #endif
450
451 if ((ret = sata_hba_init(&modlinkage)) != 0) {
452 #if AHCI_DEBUG
453 mutex_destroy(&ahci_log_mutex);
454 #endif
455 ddi_soft_state_fini(&ahci_statep);
456 goto err_out;
457 }
458
459 /* watchdog tick */
460 ahci_watchdog_tick = drv_usectohz(
461 (clock_t)ahci_watchdog_timeout * 1000000);
462
463 ret = mod_install(&modlinkage);
464 if (ret != 0) {
465 sata_hba_fini(&modlinkage);
466 #if AHCI_DEBUG
467 mutex_destroy(&ahci_log_mutex);
468 #endif
469 ddi_soft_state_fini(&ahci_statep);
470 goto err_out;
471 }
472
473 return (ret);
474
475 err_out:
476 cmn_err(CE_WARN, "!ahci: Module init failed");
477 return (ret);
478 }
479
480 /*
481 * ahci module uninitialize.
482 */
483 int
484 _fini(void)
485 {
486 int ret;
487
488 ret = mod_remove(&modlinkage);
489 if (ret != 0) {
490 return (ret);
491 }
492
493 /* Remove the resources allocated in _init(). */
494 sata_hba_fini(&modlinkage);
495 #if AHCI_DEBUG
496 mutex_destroy(&ahci_log_mutex);
497 #endif
498 ddi_soft_state_fini(&ahci_statep);
499
500 return (ret);
501 }
502
503 /*
504 * _info entry point
505 */
506 int
507 _info(struct modinfo *modinfop)
508 {
509 return (mod_info(&modlinkage, modinfop));
510 }
511
512 /*
513 * The attach entry point for dev_ops.
514 */
515 static int
516 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
517 {
518 ahci_ctl_t *ahci_ctlp = NULL;
519 int instance = ddi_get_instance(dip);
520 int status;
521 int attach_state;
522 uint32_t cap_status, ahci_version;
523 uint32_t ghc_control;
524 int intr_types;
525 int i;
526 pci_regspec_t *regs;
527 int regs_length;
528 int rnumber;
529 #if AHCI_DEBUG
530 int speed;
531 #endif
532
533 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
534 NULL);
535
536 switch (cmd) {
537 case DDI_ATTACH:
538 break;
539
540 case DDI_RESUME:
541
542 /*
543 * During DDI_RESUME, the hardware state of the device
544 * (power may have been removed from the device) must be
545 * restored, allow pending requests to continue, and
546 * service new requests.
547 */
548 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
549 mutex_enter(&ahci_ctlp->ahcictl_mutex);
550
551 /*
552 * GHC.AE must be set to 1 before any other AHCI register
553 * is accessed
554 */
555 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
556 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
557 ghc_control |= AHCI_HBA_GHC_AE;
558 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
559 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
560
561 /* Restart watch thread */
562 if (ahci_ctlp->ahcictl_timeout_id == 0)
563 ahci_ctlp->ahcictl_timeout_id = timeout(
564 (void (*)(void *))ahci_watchdog_handler,
565 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
566
567 mutex_exit(&ahci_ctlp->ahcictl_mutex);
568
569 /*
570 * Re-initialize the controller and enable the interrupts and
571 * restart all the ports.
572 *
573 * Note that so far we don't support hot-plug during
574 * suspend/resume.
575 */
576 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
577 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
578 "Failed to initialize the controller "
579 "during DDI_RESUME", NULL);
580 return (DDI_FAILURE);
581 }
582
583 mutex_enter(&ahci_ctlp->ahcictl_mutex);
584 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
585 mutex_exit(&ahci_ctlp->ahcictl_mutex);
586
587 return (DDI_SUCCESS);
588
589 default:
590 return (DDI_FAILURE);
591 }
592
593 attach_state = AHCI_ATTACH_STATE_NONE;
594
595 /* Allocate soft state */
596 status = ddi_soft_state_zalloc(ahci_statep, instance);
597 if (status != DDI_SUCCESS) {
598 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
599 instance);
600 goto err_out;
601 }
602
603 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
604 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
605 ahci_ctlp->ahcictl_dip = dip;
606
607 /* Initialize the cport/port mapping */
608 for (i = 0; i < AHCI_MAX_PORTS; i++) {
609 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
610 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
611 }
612
613 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
614
615 /* Initialize FMA properties */
616 ahci_fm_init(ahci_ctlp);
617
618 attach_state |= AHCI_ATTACH_STATE_FMA;
619
620 /*
621 * Now map the AHCI base address; which includes global
622 * registers and port control registers
623 *
624 * According to the spec, the AHCI Base Address is BAR5,
625 * but BAR0-BAR4 are optional, so we need to check which
626 * rnumber is used for BAR5.
627 */
628
629 /*
630 * search through DDI "reg" property for the AHCI register set
631 */
632 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
633 DDI_PROP_DONTPASS, "reg", (int **)®s,
634 (uint_t *)®s_length) != DDI_PROP_SUCCESS) {
635 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
636 instance);
637 goto err_out;
638 }
639
640 /* AHCI Base Address is located at 0x24 offset */
641 for (rnumber = 0; rnumber < regs_length; ++rnumber) {
642 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
643 == AHCI_PCI_RNUM)
644 break;
645 }
646
647 ddi_prop_free(regs);
648
649 if (rnumber == regs_length) {
650 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
651 instance);
652 goto err_out;
653 }
654
655 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
656
657 status = ddi_regs_map_setup(dip,
658 rnumber,
659 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
660 0,
661 0,
662 &accattr,
663 &ahci_ctlp->ahcictl_ahci_acc_handle);
664 if (status != DDI_SUCCESS) {
665 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
666 instance);
667 goto err_out;
668 }
669
670 attach_state |= AHCI_ATTACH_STATE_REG_MAP;
671
672 /*
673 * GHC.AE must be set to 1 before any other AHCI register
674 * is accessed
675 */
676 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
677 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
678 ghc_control |= AHCI_HBA_GHC_AE;
679 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
680 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
681
682 /* Get the AHCI version information */
683 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
684 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
685
686 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
687 (ahci_version & 0xffff0000) >> 16,
688 ((ahci_version & 0x0000ff00) >> 4 |
689 (ahci_version & 0x000000ff)));
690
691 /* We don't support controllers whose versions are lower than 1.0 */
692 if (!(ahci_version & 0xffff0000)) {
693 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
694 "than version 1.0", instance);
695 goto err_out;
696 }
697
698 /* Get the HBA capabilities information */
699 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
700 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
701
702 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
703 cap_status);
704
705 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
706 if (ahci_version >= 0x00010200) {
707 uint32_t cap2_status;
708
709 /* Get the HBA capabilities extended information */
710 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
711 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
712
713 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
714 "hba capabilities extended = 0x%x", cap2_status);
715 }
716
717 #if AHCI_DEBUG
718 /* Get the interface speed supported by the HBA */
719 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
720 if (speed == 0x01) {
721 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
722 "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
723 } else if (speed == 0x10) {
724 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
725 "hba interface speed support: Gen 2 (3 Gbps)", NULL);
726 } else if (speed == 0x11) {
727 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
728 "hba interface speed support: Gen 3 (6 Gbps)", NULL);
729 }
730 #endif
731
732 /* Get the number of command slots supported by the HBA */
733 ahci_ctlp->ahcictl_num_cmd_slots =
734 ((cap_status & AHCI_HBA_CAP_NCS) >>
735 AHCI_HBA_CAP_NCS_SHIFT) + 1;
736
737 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
738 ahci_ctlp->ahcictl_num_cmd_slots);
739
740 /* Get the bit map which indicates ports implemented by the HBA */
741 ahci_ctlp->ahcictl_ports_implemented =
742 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
743 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
744
745 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
746 ahci_ctlp->ahcictl_ports_implemented);
747
748 /* Max port number implemented */
749 ahci_ctlp->ahcictl_num_ports =
750 ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
751
752 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
753 (cap_status & AHCI_HBA_CAP_NP) + 1);
754
755 /* Get the number of implemented ports by the HBA */
756 ahci_ctlp->ahcictl_num_implemented_ports =
757 ahci_get_num_implemented_ports(
758 ahci_ctlp->ahcictl_ports_implemented);
759
760 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
761 "hba number of implemented ports: %d",
762 ahci_ctlp->ahcictl_num_implemented_ports);
763
764 /* Check whether HBA supports 64bit DMA addressing */
765 if (!(cap_status & AHCI_HBA_CAP_S64A)) {
766 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
767 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
768 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
769 "hba does not support 64-bit addressing", NULL);
770 }
771
772 /* Checking for the support of Port Multiplier */
773 if (cap_status & AHCI_HBA_CAP_SPM) {
774 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
775 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
776 "hba supports port multiplier (CBSS)", NULL);
777
778 /* Support FIS-based switching ? */
779 if (cap_status & AHCI_HBA_CAP_FBSS) {
780 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
781 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
782 "hba supports FIS-based switching (FBSS)", NULL);
783 }
784 }
785
786 /* Checking for Support Command List Override */
787 if (cap_status & AHCI_HBA_CAP_SCLO) {
788 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
789 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
790 "hba supports command list override.", NULL);
791 }
792
793 /* Checking for Asynchronous Notification */
794 if (cap_status & AHCI_HBA_CAP_SSNTF) {
795 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
796 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
797 "hba supports asynchronous notification.", NULL);
798 }
799
800 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
801 != DDI_SUCCESS) {
802 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
803 instance);
804 goto err_out;
805 }
806
807 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
808
809 /*
810 * Check the pci configuration space, and set caps. We also
811 * handle the hardware defect in this function.
812 *
813 * For example, force ATI SB600 to use 32-bit dma addressing
814 * since it doesn't support 64-bit dma though its CAP register
815 * declares it support.
816 */
817 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
818 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
819 instance);
820 goto err_out;
821 }
822
823 /*
824 * Disable the whole controller interrupts before adding
825 * interrupt handlers(s).
826 */
827 ahci_disable_all_intrs(ahci_ctlp);
828
829 /* Get supported interrupt types */
830 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
831 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
832 instance);
833 goto err_out;
834 }
835
836 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
837 "ddi_intr_get_supported_types() returned: 0x%x",
838 intr_types);
839
840 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
841 /*
842 * Try MSI first, but fall back to FIXED if failed
843 */
844 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
845 DDI_SUCCESS) {
846 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
847 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
848 "Using MSI interrupt type", NULL);
849 goto intr_done;
850 }
851
852 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
853 "MSI registration failed, "
854 "trying FIXED interrupts", NULL);
855 }
856
857 if (intr_types & DDI_INTR_TYPE_FIXED) {
858 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
859 DDI_SUCCESS) {
860 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
861 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
862 "Using FIXED interrupt type", NULL);
863 goto intr_done;
864 }
865
866 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
867 "FIXED interrupt registration failed", NULL);
868 }
869
870 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
871
872 goto err_out;
873
874 intr_done:
875
876 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
877
878 /* Initialize the controller mutex */
879 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
880 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
881
882 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
883
884 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
885 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
886 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
887 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
888 }
889
890 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
891 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
892 sizeof (ahci_prdt_item_t));
893
894 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
895 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
896 " ahci_cmd_table_size is 0x%x",
897 ahci_dma_prdt_number, ahci_cmd_table_size);
898
899 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
900 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
901 ahci_dma_prdt_number;
902
903 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
904 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
905 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
906 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
907
908 /*
909 * enable 64bit dma for data buffer for SB600 if
910 * sb600_buf_64bit_dma_disable is B_FALSE
911 */
912 if ((ahci_buf_64bit_dma == B_FALSE) ||
913 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
914 !(sb600_buf_64bit_dma_disable == B_FALSE &&
915 ahci_ctlp->ahcictl_venid == 0x1002 &&
916 ahci_ctlp->ahcictl_devid == 0x4380))) {
917 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
918 0xffffffffull;
919 }
920
921 /*
922 * enable 64bit dma for command buffer for SB600/700/710/800
923 * if sbxxx_commu_64bit_dma_disable is B_FALSE
924 */
925 if ((ahci_commu_64bit_dma == B_FALSE) ||
926 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
927 !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
928 ahci_ctlp->ahcictl_venid == 0x1002 &&
929 (ahci_ctlp->ahcictl_devid == 0x4380 ||
930 ahci_ctlp->ahcictl_devid == 0x4391)))) {
931 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
932 0xffffffffull;
933 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
934 0xffffffffull;
935 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
936 0xffffffffull;
937 }
938
939 /* Allocate the ports structure */
940 status = ahci_alloc_ports_state(ahci_ctlp);
941 if (status != AHCI_SUCCESS) {
942 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
943 instance);
944 goto err_out;
945 }
946
947 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
948
949 /*
950 * Initialize the controller and ports.
951 */
952 status = ahci_initialize_controller(ahci_ctlp);
953 if (status != AHCI_SUCCESS) {
954 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
955 instance);
956 goto err_out;
957 }
958
959 attach_state |= AHCI_ATTACH_STATE_HW_INIT;
960
961 /* Start one thread to check packet timeouts */
962 ahci_ctlp->ahcictl_timeout_id = timeout(
963 (void (*)(void *))ahci_watchdog_handler,
964 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
965
966 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
967
968 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
969 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
970 instance);
971 goto err_out;
972 }
973
974 /* Check all handles at the end of the attach operation. */
975 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
976 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
977 instance);
978 goto err_out;
979 }
980
981 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
982
983 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
984
985 return (DDI_SUCCESS);
986
987 err_out:
988 /* FMA message */
989 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
990 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
991
992 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
993 mutex_enter(&ahci_ctlp->ahcictl_mutex);
994 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
995 ahci_ctlp->ahcictl_timeout_id = 0;
996 mutex_exit(&ahci_ctlp->ahcictl_mutex);
997 }
998
999 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1000 ahci_uninitialize_controller(ahci_ctlp);
1001 }
1002
1003 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1004 ahci_dealloc_ports_state(ahci_ctlp);
1005 }
1006
1007 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1008 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1009 }
1010
1011 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1012 ahci_rem_intrs(ahci_ctlp);
1013 }
1014
1015 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1016 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1017 }
1018
1019 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1020 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1021 }
1022
1023 if (attach_state & AHCI_ATTACH_STATE_FMA) {
1024 ahci_fm_fini(ahci_ctlp);
1025 }
1026
1027 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1028 ddi_soft_state_free(ahci_statep, instance);
1029 }
1030
1031 return (DDI_FAILURE);
1032 }
1033
1034 /*
1035 * The detach entry point for dev_ops.
1036 */
1037 static int
1038 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1039 {
1040 ahci_ctl_t *ahci_ctlp;
1041 int instance;
1042 int ret;
1043
1044 instance = ddi_get_instance(dip);
1045 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1046
1047 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1048
1049 switch (cmd) {
1050 case DDI_DETACH:
1051
1052 /* disable the interrupts for an uninterrupted detach */
1053 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1054 ahci_disable_all_intrs(ahci_ctlp);
1055 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1056
1057 /* unregister from the sata framework. */
1058 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1059 if (ret != AHCI_SUCCESS) {
1060 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1061 ahci_enable_all_intrs(ahci_ctlp);
1062 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1063 return (DDI_FAILURE);
1064 }
1065
1066 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1067
1068 /* stop the watchdog handler */
1069 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1070 ahci_ctlp->ahcictl_timeout_id = 0;
1071
1072 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1073
1074 /* uninitialize the controller */
1075 ahci_uninitialize_controller(ahci_ctlp);
1076
1077 /* remove the interrupts */
1078 ahci_rem_intrs(ahci_ctlp);
1079
1080 /* deallocate the ports structures */
1081 ahci_dealloc_ports_state(ahci_ctlp);
1082
1083 /* destroy mutex */
1084 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1085
1086 /* teardown the pci config */
1087 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1088
1089 /* remove the reg maps. */
1090 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1091
1092 /* release fma resource */
1093 ahci_fm_fini(ahci_ctlp);
1094
1095 /* free the soft state. */
1096 ddi_soft_state_free(ahci_statep, instance);
1097
1098 return (DDI_SUCCESS);
1099
1100 case DDI_SUSPEND:
1101
1102 /*
1103 * The steps associated with suspension must include putting
1104 * the underlying device into a quiescent state so that it
1105 * will not generate interrupts or modify or access memory.
1106 */
1107 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1108 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1109 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1110 return (DDI_SUCCESS);
1111 }
1112
1113 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1114
1115 /* stop the watchdog handler */
1116 if (ahci_ctlp->ahcictl_timeout_id) {
1117 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1118 ahci_ctlp->ahcictl_timeout_id = 0;
1119 }
1120
1121 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1122
1123 /*
1124 * drain the taskq
1125 */
1126 ahci_drain_ports_taskq(ahci_ctlp);
1127
1128 /*
1129 * Disable the interrupts and stop all the ports.
1130 */
1131 ahci_uninitialize_controller(ahci_ctlp);
1132
1133 return (DDI_SUCCESS);
1134
1135 default:
1136 return (DDI_FAILURE);
1137 }
1138 }
1139
1140 /*
1141 * The info entry point for dev_ops.
1142 *
1143 */
1144 static int
1145 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1146 void *arg, void **result)
1147 {
1148 #ifndef __lock_lint
1149 _NOTE(ARGUNUSED(dip))
1150 #endif /* __lock_lint */
1151
1152 ahci_ctl_t *ahci_ctlp;
1153 int instance;
1154 dev_t dev;
1155
1156 dev = (dev_t)arg;
1157 instance = getminor(dev);
1158
1159 switch (infocmd) {
1160 case DDI_INFO_DEVT2DEVINFO:
1161 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1162 if (ahci_ctlp != NULL) {
1163 *result = ahci_ctlp->ahcictl_dip;
1164 return (DDI_SUCCESS);
1165 } else {
1166 *result = NULL;
1167 return (DDI_FAILURE);
1168 }
1169 case DDI_INFO_DEVT2INSTANCE:
1170 *(int *)result = instance;
1171 break;
1172 default:
1173 break;
1174 }
1175
1176 return (DDI_SUCCESS);
1177 }
1178
1179 /*
1180 * Registers the ahci with sata framework.
1181 */
1182 static int
1183 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1184 {
1185 struct sata_hba_tran *sata_hba_tran;
1186
1187 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1188 "ahci_register_sata_hba_tran enter", NULL);
1189
1190 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1191
1192 /* Allocate memory for the sata_hba_tran */
1193 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1194
1195 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1196 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1197 sata_hba_tran->sata_tran_hba_dma_attr =
1198 &ahci_ctlp->ahcictl_buffer_dma_attr;
1199
1200 /* Report the number of implemented ports */
1201 sata_hba_tran->sata_tran_hba_num_cports =
1202 ahci_ctlp->ahcictl_num_implemented_ports;
1203
1204 /* Support ATAPI device */
1205 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1206
1207 /* Get the data transfer capability for PIO command by the HBA */
1208 if (cap_status & AHCI_HBA_CAP_PMD) {
1209 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1210 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1211 "DRQ block data transfer for PIO command protocol", NULL);
1212 }
1213
1214 /*
1215 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1216 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1217 * QUEUED (EXT), and SERVICE commands). Queued operations are
1218 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1219 * QUEUED commands when the HBA and device support native command
1220 * queuing(NCQ).
1221 *
1222 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1223 * CAP register of the HBA indicates NCQ is supported.
1224 *
1225 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1226 * set because the previous register content of PxCI can be re-written
1227 * in the register write.
1228 */
1229 if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1230 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1231 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1232 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1233 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1234 "Command Queuing", NULL);
1235 }
1236
1237 /* Support port multiplier? */
1238 if (cap_status & AHCI_HBA_CAP_SPM) {
1239 sata_hba_tran->sata_tran_hba_features_support |=
1240 SATA_CTLF_PORT_MULTIPLIER;
1241
1242 /* Support FIS-based switching for port multiplier? */
1243 if (cap_status & AHCI_HBA_CAP_FBSS) {
1244 sata_hba_tran->sata_tran_hba_features_support |=
1245 SATA_CTLF_PMULT_FBS;
1246 }
1247 }
1248
1249 /* Report the number of command slots */
1250 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1251
1252 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1253 sata_hba_tran->sata_tran_start = ahci_tran_start;
1254 sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1255 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1256 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1257 #ifdef __lock_lint
1258 sata_hba_tran->sata_tran_selftest = ahci_selftest;
1259 #endif
1260 /*
1261 * When SATA framework adds support for pwrmgt the
1262 * pwrmgt_ops needs to be updated
1263 */
1264 sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1265 sata_hba_tran->sata_tran_ioctl = NULL;
1266
1267 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1268
1269 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1270
1271 /* Attach it to SATA framework */
1272 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1273 != DDI_SUCCESS) {
1274 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1275 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1276 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1277 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1278 return (AHCI_FAILURE);
1279 }
1280
1281 return (AHCI_SUCCESS);
1282 }
1283
1284 /*
1285 * Unregisters the ahci with sata framework.
1286 */
1287 static int
1288 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1289 {
1290 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1291 "ahci_unregister_sata_hba_tran enter", NULL);
1292
1293 /* Detach from the SATA framework. */
1294 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1295 DDI_SUCCESS) {
1296 return (AHCI_FAILURE);
1297 }
1298
1299 /* Deallocate sata_hba_tran. */
1300 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1301 sizeof (sata_hba_tran_t));
1302
1303 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1304 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1305 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1306
1307 return (AHCI_SUCCESS);
1308 }
1309
1310 #define SET_PORTSTR(str, addrp) \
1311 if (AHCI_ADDR_IS_PORT(addrp)) \
1312 (void) sprintf((str), "%d", (addrp)->aa_port); \
1313 else if (AHCI_ADDR_IS_PMULT(addrp)) \
1314 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \
1315 else \
1316 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \
1317 (addrp)->aa_pmport);
1318
1319 /*
1320 * ahci_tran_probe_port is called by SATA framework. It returns port state,
1321 * port status registers and an attached device type via sata_device
1322 * structure.
1323 *
1324 * We return the cached information from a previous hardware probe. The
1325 * actual hardware probing itself was done either from within
1326 * ahci_initialize_controller() during the driver attach or from a phy
1327 * ready change interrupt handler.
1328 */
1329 static int
1330 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1331 {
1332 ahci_ctl_t *ahci_ctlp;
1333 ahci_port_t *ahci_portp;
1334 ahci_addr_t addr, pmult_addr;
1335 uint8_t cport = sd->satadev_addr.cport;
1336 char portstr[10];
1337 uint8_t device_type;
1338 uint32_t port_state;
1339 uint8_t port;
1340 int rval = SATA_SUCCESS, rval_init;
1341
1342 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1343 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1344
1345 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1346
1347 mutex_enter(&ahci_portp->ahciport_mutex);
1348
1349 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1350 ASSERT(AHCI_ADDR_IS_VALID(&addr));
1351 SET_PORTSTR(portstr, &addr);
1352
1353 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1354 "ahci_tran_probe_port enter: port %s", portstr);
1355
1356 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1357 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1358 ahci_portp->ahciport_pmult_info == NULL)) {
1359 /* port mutliplier is removed. */
1360 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1361 "ahci_tran_probe_port: "
1362 "pmult is removed from port %s", portstr);
1363 mutex_exit(&ahci_portp->ahciport_mutex);
1364 return (SATA_FAILURE);
1365 }
1366
1367 /*
1368 * The sata_device may refer to
1369 * 1. A controller port.
1370 * A controller port should be ready here.
1371 * 2. A port multiplier.
1372 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1373 * it and register the port multiplier to the framework.
1374 * SATA_ADDR_PMULT - check the status of all its device ports.
1375 * 3. A port multiplier port.
1376 * If it has not been initialized, initialized it.
1377 *
1378 * A port multiplier or a port multiplier port may require some
1379 * initialization because we cannot do these time-consuming jobs in an
1380 * interrupt context.
1381 */
1382 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1383 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1384 /* Initialize registers on a port multiplier */
1385 rval_init = ahci_initialize_pmult(ahci_ctlp,
1386 ahci_portp, &pmult_addr, sd);
1387 if (rval_init != AHCI_SUCCESS) {
1388 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1389 "ahci_tran_probe_port: "
1390 "pmult initialization failed.", NULL);
1391 mutex_exit(&ahci_portp->ahciport_mutex);
1392 return (SATA_FAILURE);
1393 }
1394 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1395 /* Check pmports hotplug events */
1396 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1397 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1398 SATA_ADDR_DPMPORT)) {
1399 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1400 &addr, sd) != AHCI_SUCCESS) {
1401 rval = SATA_FAILURE;
1402 goto out;
1403 }
1404 }
1405
1406 /* Update port state and device type */
1407 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1408
1409 switch (port_state) {
1410
1411 case SATA_PSTATE_FAILED:
1412 sd->satadev_state = SATA_PSTATE_FAILED;
1413 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1414 "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1415 goto out;
1416
1417 case SATA_PSTATE_SHUTDOWN:
1418 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1419 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1420 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1421 goto out;
1422
1423 case SATA_PSTATE_PWROFF:
1424 sd->satadev_state = SATA_PSTATE_PWROFF;
1425 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1426 "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1427 goto out;
1428
1429 case SATA_PSTATE_PWRON:
1430 sd->satadev_state = SATA_PSTATE_PWRON;
1431 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1432 "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1433 break;
1434
1435 default:
1436 sd->satadev_state = port_state;
1437 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1438 "ahci_tran_probe_port: port %s PORT NORMAL %x",
1439 portstr, port_state);
1440 break;
1441 }
1442
1443 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1444
1445 switch (device_type) {
1446
1447 case SATA_DTYPE_ATADISK:
1448 sd->satadev_type = SATA_DTYPE_ATADISK;
1449 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1450 "ahci_tran_probe_port: port %s DISK found", portstr);
1451 break;
1452
1453 case SATA_DTYPE_ATAPI:
1454 /*
1455 * HBA driver only knows it's an ATAPI device, and don't know
1456 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1457 * type need to be determined by checking IDENTIFY PACKET
1458 * DEVICE data
1459 */
1460 sd->satadev_type = SATA_DTYPE_ATAPI;
1461 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1462 "ahci_tran_probe_port: port %s ATAPI found", portstr);
1463 break;
1464
1465 case SATA_DTYPE_PMULT:
1466 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1467 sd->satadev_type = SATA_DTYPE_PMULT;
1468
1469 /* Update the number of pmports. */
1470 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1471 sd->satadev_add_info = ahci_portp->
1472 ahciport_pmult_info->ahcipmi_num_dev_ports;
1473
1474 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1475 "ahci_tran_probe_port: port %s Port Multiplier found",
1476 portstr);
1477 break;
1478
1479 case SATA_DTYPE_UNKNOWN:
1480 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1481 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1482 "ahci_tran_probe_port: port %s Unknown device found",
1483 portstr);
1484 break;
1485
1486 default:
1487 /* we don't support any other device types */
1488 sd->satadev_type = SATA_DTYPE_NONE;
1489 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1490 "ahci_tran_probe_port: port %s No device found", portstr);
1491 break;
1492 }
1493
1494 out:
1495 /* Register update only fails while probing a pmult/pmport */
1496 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1497 ahci_update_sata_registers(ahci_ctlp, port, sd);
1498 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1499 if (port_state & SATA_STATE_READY)
1500 if (ahci_update_pmult_pscr(ahci_ctlp,
1501 &addr, sd) != AHCI_SUCCESS)
1502 rval = SATA_FAILURE;
1503 }
1504
1505 /* Check handles for the sata registers access */
1506 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1507 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1508 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1509 DDI_SERVICE_UNAFFECTED);
1510 rval = SATA_FAILURE;
1511 }
1512
1513 mutex_exit(&ahci_portp->ahciport_mutex);
1514 return (rval);
1515 }
1516
1517 /*
1518 * There are four operation modes in sata framework:
1519 * SATA_OPMODE_INTERRUPTS
1520 * SATA_OPMODE_POLLING
1521 * SATA_OPMODE_ASYNCH
1522 * SATA_OPMODE_SYNCH
1523 *
1524 * Their combined meanings as following:
1525 *
1526 * SATA_OPMODE_SYNCH
1527 * The command has to be completed before sata_tran_start functions returns.
1528 * Either interrupts or polling could be used - it's up to the driver.
1529 * Mode used currently for internal, sata-module initiated operations.
1530 *
1531 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1532 * It is the same as the one above.
1533 *
1534 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1535 * The command has to be completed before sata_tran_start function returns.
1536 * No interrupt used, polling only. This should be the mode used for scsi
1537 * packets with FLAG_NOINTR.
1538 *
1539 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1540 * The command may be queued (callback function specified). Interrupts could
1541 * be used. It's normal operation mode.
1542 */
1543 /*
1544 * Called by sata framework to transport a sata packet down stream.
1545 */
1546 static int
1547 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1548 {
1549 ahci_ctl_t *ahci_ctlp;
1550 ahci_port_t *ahci_portp;
1551 ahci_addr_t addr;
1552 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1553 uint8_t port;
1554 char portstr[10];
1555
1556 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1557 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1558
1559 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1560 "ahci_tran_start enter: cport %d satapkt 0x%p",
1561 cport, (void *)spkt);
1562
1563 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1564
1565 mutex_enter(&ahci_portp->ahciport_mutex);
1566 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1567 SET_PORTSTR(portstr, &addr);
1568
1569 /* Sanity check */
1570 if (AHCI_ADDR_IS_PMPORT(&addr)) {
1571 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1572 ahci_portp->ahciport_pmult_info == NULL) {
1573
1574 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1575 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1576 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1577 ahci_update_sata_registers(ahci_ctlp, port,
1578 &spkt->satapkt_device);
1579 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1580 "ahci_tran_start returning PORT_ERROR while "
1581 "pmult removed: port: %s", portstr);
1582 mutex_exit(&ahci_portp->ahciport_mutex);
1583 return (SATA_TRAN_PORT_ERROR);
1584 }
1585
1586 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1587 SATA_STATE_READY)) {
1588 if (!ddi_in_panic() ||
1589 ahci_initialize_pmport(ahci_ctlp,
1590 ahci_portp, &addr) != AHCI_SUCCESS) {
1591 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1592 spkt->satapkt_device.satadev_type =
1593 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1594 spkt->satapkt_device.satadev_state =
1595 AHCIPORT_GET_STATE(ahci_portp, &addr);
1596 ahci_update_sata_registers(ahci_ctlp, port,
1597 &spkt->satapkt_device);
1598 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1599 "ahci_tran_start returning PORT_ERROR "
1600 "while sub-link is not initialized "
1601 "at port: %s", portstr);
1602 mutex_exit(&ahci_portp->ahciport_mutex);
1603 return (SATA_TRAN_PORT_ERROR);
1604 }
1605 }
1606 }
1607
1608 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1609 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1610 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1611 /*
1612 * In case the target driver would send the packet before
1613 * sata framework can have the opportunity to process those
1614 * event reports.
1615 */
1616 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1617 spkt->satapkt_device.satadev_state =
1618 ahci_portp->ahciport_port_state;
1619 ahci_update_sata_registers(ahci_ctlp, port,
1620 &spkt->satapkt_device);
1621 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1622 "ahci_tran_start returning PORT_ERROR while "
1623 "port in FAILED/SHUTDOWN/PWROFF state: "
1624 "port: %s", portstr);
1625 mutex_exit(&ahci_portp->ahciport_mutex);
1626 return (SATA_TRAN_PORT_ERROR);
1627 }
1628
1629 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1630 /*
1631 * ahci_intr_phyrdy_change() may have rendered it to
1632 * SATA_DTYPE_NONE.
1633 */
1634 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1635 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1636 spkt->satapkt_device.satadev_state =
1637 ahci_portp->ahciport_port_state;
1638 ahci_update_sata_registers(ahci_ctlp, port,
1639 &spkt->satapkt_device);
1640 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1641 "ahci_tran_start returning PORT_ERROR while "
1642 "no device attached: port: %s", portstr);
1643 mutex_exit(&ahci_portp->ahciport_mutex);
1644 return (SATA_TRAN_PORT_ERROR);
1645 }
1646
1647 /* R/W PMULT command will occupy the whole HBA port */
1648 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1649 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1650 "ahci_tran_start returning BUSY while "
1651 "executing READ/WRITE PORT-MULT command: "
1652 "port: %s", portstr);
1653 spkt->satapkt_reason = SATA_PKT_BUSY;
1654 mutex_exit(&ahci_portp->ahciport_mutex);
1655 return (SATA_TRAN_BUSY);
1656 }
1657
1658 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1659 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1660 "ahci_tran_start returning BUSY while "
1661 "hot-plug in progress: port: %s", portstr);
1662 spkt->satapkt_reason = SATA_PKT_BUSY;
1663 mutex_exit(&ahci_portp->ahciport_mutex);
1664 return (SATA_TRAN_BUSY);
1665 }
1666
1667 /*
1668 * SATA HBA driver should remember that a device was reset and it
1669 * is supposed to reject any packets which do not specify either
1670 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1671 *
1672 * This is to prevent a race condition when a device was arbitrarily
1673 * reset by the HBA driver (and lost it's setting) and a target
1674 * driver sending some commands to a device before the sata framework
1675 * has a chance to restore the device setting (such as cache enable/
1676 * disable or other resettable stuff).
1677 */
1678 /*
1679 * It is unnecessary to use specific flags to indicate
1680 * reset_in_progress for a pmport. While mopping, all command will be
1681 * mopped so that the entire HBA port is being dealt as a single
1682 * object.
1683 */
1684 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1685 ahci_portp->ahciport_reset_in_progress = 0;
1686 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1687 "ahci_tran_start [CLEAR] the "
1688 "reset_in_progress for port: %d", port);
1689 }
1690
1691 if (ahci_portp->ahciport_reset_in_progress &&
1692 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1693 ! ddi_in_panic()) {
1694 spkt->satapkt_reason = SATA_PKT_BUSY;
1695 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1696 "ahci_tran_start returning BUSY while "
1697 "reset in progress: port: %d", port);
1698 mutex_exit(&ahci_portp->ahciport_mutex);
1699 return (SATA_TRAN_BUSY);
1700 }
1701
1702 #ifdef AHCI_DEBUG
1703 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1704 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1705 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1706 spkt, port);
1707 }
1708 #endif
1709
1710 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1711 spkt->satapkt_reason = SATA_PKT_BUSY;
1712 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1713 "ahci_tran_start returning BUSY while "
1714 "mopping in progress: port: %d", port);
1715 mutex_exit(&ahci_portp->ahciport_mutex);
1716 return (SATA_TRAN_BUSY);
1717 }
1718
1719 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1720 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1721 DDI_SERVICE_UNAFFECTED);
1722 mutex_exit(&ahci_portp->ahciport_mutex);
1723 return (SATA_TRAN_BUSY);
1724 }
1725
1726 if (spkt->satapkt_op_mode &
1727 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1728 /*
1729 * If a SYNC command to be executed in interrupt context,
1730 * bounce it back to sata module.
1731 */
1732 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1733 servicing_interrupt()) {
1734 spkt->satapkt_reason = SATA_PKT_BUSY;
1735 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1736 "ahci_tran_start returning BUSY while "
1737 "sending SYNC mode under interrupt context: "
1738 "port : %d", port);
1739 mutex_exit(&ahci_portp->ahciport_mutex);
1740 return (SATA_TRAN_BUSY);
1741 }
1742
1743 /* We need to do the sync start now */
1744 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1745 spkt) == AHCI_FAILURE) {
1746 goto fail_out;
1747 }
1748 } else {
1749 /* Async start, using interrupt */
1750 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1751 == AHCI_FAILURE) {
1752 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1753 goto fail_out;
1754 }
1755 }
1756
1757 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1758 "sata tran accepted: port %s", portstr);
1759
1760 mutex_exit(&ahci_portp->ahciport_mutex);
1761 return (SATA_TRAN_ACCEPTED);
1762
1763 fail_out:
1764 /*
1765 * Failed to deliver packet to the controller.
1766 * Check if it's caused by invalid handles.
1767 */
1768 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1769 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1770 spkt->satapkt_device.satadev_type =
1771 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1772 spkt->satapkt_device.satadev_state =
1773 AHCIPORT_GET_STATE(ahci_portp, &addr);
1774 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1775 mutex_exit(&ahci_portp->ahciport_mutex);
1776 return (SATA_TRAN_PORT_ERROR);
1777 }
1778
1779 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1780 "return QUEUE_FULL: port %d", port);
1781 mutex_exit(&ahci_portp->ahciport_mutex);
1782 return (SATA_TRAN_QUEUE_FULL);
1783 }
1784
1785 /*
1786 * SATA_OPMODE_SYNCH flag is set
1787 *
1788 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1789 * without interrupt, otherwise we can still use the interrupt.
1790 */
1791 static int
1792 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1793 ahci_addr_t *addrp, sata_pkt_t *spkt)
1794 {
1795 int pkt_timeout_ticks;
1796 uint32_t timeout_tags;
1797 int rval;
1798 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1799 uint8_t port = addrp->aa_port;
1800
1801 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1802
1803 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1804 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1805
1806 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1807 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1808 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1809 addrp, spkt)) == AHCI_FAILURE) {
1810 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1811 return (rval);
1812 }
1813
1814 pkt_timeout_ticks =
1815 drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1816
1817 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1818 /* Simulate the interrupt */
1819 mutex_exit(&ahci_portp->ahciport_mutex);
1820 ahci_port_intr(ahci_ctlp, ahci_portp, port);
1821 mutex_enter(&ahci_portp->ahciport_mutex);
1822
1823 if (spkt->satapkt_reason != SATA_PKT_BUSY)
1824 break;
1825
1826 mutex_exit(&ahci_portp->ahciport_mutex);
1827 drv_usecwait(AHCI_1MS_USECS);
1828 mutex_enter(&ahci_portp->ahciport_mutex);
1829
1830 pkt_timeout_ticks -= AHCI_1MS_TICKS;
1831 if (pkt_timeout_ticks < 0) {
1832 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1833 "port %d satapkt 0x%p timed out\n",
1834 instance, port, (void *)spkt);
1835 timeout_tags = (0x1 << rval);
1836 mutex_exit(&ahci_portp->ahciport_mutex);
1837 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1838 port, timeout_tags);
1839 mutex_enter(&ahci_portp->ahciport_mutex);
1840 }
1841 }
1842
1843 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1844 return (AHCI_SUCCESS);
1845
1846 } else {
1847 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1848 addrp, spkt)) == AHCI_FAILURE)
1849 return (rval);
1850
1851 #if AHCI_DEBUG
1852 /*
1853 * Note that the driver always uses the slot 0 to deliver
1854 * REQUEST SENSE or READ LOG EXT command
1855 */
1856 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1857 ASSERT(rval == 0);
1858 #endif
1859
1860 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1861 cv_wait(&ahci_portp->ahciport_cv,
1862 &ahci_portp->ahciport_mutex);
1863
1864 return (AHCI_SUCCESS);
1865 }
1866 }
1867
1868 /*
1869 * Searches for and claims a free command slot.
1870 *
1871 * Returns value:
1872 *
1873 * AHCI_FAILURE returned only if
1874 * 1. No empty slot left
1875 * 2. Non-queued command requested while queued command(s) is outstanding
1876 * 3. Queued command requested while non-queued command(s) is outstanding
1877 * 4. HBA doesn't support multiple-use of command list while already a
1878 * non-queued command is oustanding
1879 * 5. Queued command requested while some queued command(s) has been
1880 * outstanding on a different port multiplier port. (AHCI spec 1.2,
1881 * 9.1.2)
1882 *
1883 * claimed slot number returned if succeeded
1884 *
1885 * NOTE: it will always return slot 0 for following commands to simplify the
1886 * algorithm.
1887 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process
1888 * 2. READ/WRITE PORTMULT command
1889 */
1890 static int
1891 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1892 ahci_addr_t *addrp, int command_type)
1893 {
1894 uint32_t port_cmd_issue;
1895 uint32_t free_slots;
1896 int slot;
1897
1898 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1899
1900 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1901 "ahciport_pending_tags = 0x%x "
1902 "ahciport_pending_ncq_tags = 0x%x",
1903 ahci_portp->ahciport_pending_tags,
1904 ahci_portp->ahciport_pending_ncq_tags);
1905
1906 /*
1907 * According to the AHCI spec, system software is responsible to
1908 * ensure that queued and non-queued commands are not mixed in
1909 * the command list.
1910 */
1911 if (command_type == AHCI_NON_NCQ_CMD) {
1912 /* Non-NCQ command request */
1913 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1914 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1915 "ahci_claim_free_slot: there is still pending "
1916 "queued command(s) in the command list, "
1917 "so no available slot for the non-queued "
1918 "command", NULL);
1919 return (AHCI_FAILURE);
1920 }
1921 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1922 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1923 "ahci_claim_free_slot: there is still pending "
1924 "read/write port-mult command(s) in command list, "
1925 "so no available slot for the non-queued command",
1926 NULL);
1927 return (AHCI_FAILURE);
1928 }
1929 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
1930 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1931 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1932 "ahci_claim_free_slot: HBA cannot support multiple-"
1933 "use of the command list for non-queued commands",
1934 NULL);
1935 return (AHCI_FAILURE);
1936 }
1937 free_slots = (~ahci_portp->ahciport_pending_tags) &
1938 AHCI_SLOT_MASK(ahci_ctlp);
1939 } else if (command_type == AHCI_NCQ_CMD) {
1940 /* NCQ command request */
1941 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1942 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1943 "ahci_claim_free_slot: there is still pending "
1944 "non-queued command(s) in the command list, "
1945 "so no available slot for the queued command",
1946 NULL);
1947 return (AHCI_FAILURE);
1948 }
1949
1950 /*
1951 * NCQ commands cannot be sent to different port multiplier
1952 * ports in Command-Based Switching mode
1953 */
1954 /*
1955 * NOTE: In Command-Based Switching mode, AHCI controller
1956 * usually reports a 'Handshake Error' when multiple NCQ
1957 * commands are outstanding simultaneously.
1958 */
1959 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
1960 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1961 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
1962 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
1963 AHCIPORT_NCQ_PMPORT(ahci_portp) !=
1964 addrp->aa_pmport) {
1965 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1966 "ahci_claim_free_slot: there is still "
1967 "pending queued command(s) in the "
1968 "command list for another Port Multiplier "
1969 "port, so no available slot.", NULL);
1970 return (AHCI_FAILURE);
1971 }
1972 }
1973
1974 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
1975 AHCI_NCQ_SLOT_MASK(ahci_portp);
1976 } else if (command_type == AHCI_ERR_RETRI_CMD) {
1977 /* Error retrieval command request */
1978 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1979 "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
1980 "SENSE or READ LOG EXT command", NULL);
1981 slot = 0;
1982 goto out;
1983 } else if (command_type == AHCI_RDWR_PMULT_CMD) {
1984 /*
1985 * An extra check on PxCI. Sometimes PxCI bits may not be
1986 * cleared during hot-plug or error recovery process.
1987 */
1988 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1989 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
1990
1991 if (port_cmd_issue != 0) {
1992 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1993 "ahci_claim_free_slot: there is still pending "
1994 "command(s) in command list (0x%x/0x%x, PxCI %x),"
1995 "so no available slot for R/W PMULT command.",
1996 NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
1997 NCQ_CMD_IN_PROGRESS(ahci_portp),
1998 port_cmd_issue);
1999 return (AHCI_FAILURE);
2000 }
2001
2002 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2003 "ahci_claim_free_slot: slot 0 is allocated for "
2004 "READ/WRITE PORTMULT command", NULL);
2005 slot = 0;
2006 goto out;
2007 }
2008
2009 slot = ddi_ffs(free_slots) - 1;
2010 if (slot == -1) {
2011 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2012 "ahci_claim_free_slot: no empty slots", NULL);
2013 return (AHCI_FAILURE);
2014 }
2015
2016 /*
2017 * According to the AHCI spec, to allow a simple mechanism for the
2018 * HBA to map command list slots to queue entries, software must
2019 * match the tag number it uses to the slot it is placing the command
2020 * in. For example, if a queued command is placed in slot 5, the tag
2021 * for that command must be 5.
2022 */
2023 if (command_type == AHCI_NCQ_CMD) {
2024 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2025 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2026 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2027 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2028 }
2029 }
2030
2031 ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2032
2033 out:
2034 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2035 "ahci_claim_free_slot: found slot: 0x%x", slot);
2036
2037 return (slot);
2038 }
2039
2040 /*
2041 * Builds the Command Table for the sata packet and delivers it to controller.
2042 *
2043 * Returns:
2044 * slot number if we can obtain a slot successfully
2045 * otherwise, return AHCI_FAILURE
2046 */
2047 static int
2048 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2049 ahci_addr_t *addrp, sata_pkt_t *spkt)
2050 {
2051 int cmd_slot;
2052 sata_cmd_t *scmd;
2053 ahci_fis_h2d_register_t *h2d_register_fisp;
2054 ahci_cmd_table_t *cmd_table;
2055 ahci_cmd_header_t *cmd_header;
2056 int ncookies;
2057 int i;
2058 int command_type = AHCI_NON_NCQ_CMD;
2059 int ncq_qdepth;
2060 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2061 uint8_t port, pmport;
2062 #if AHCI_DEBUG
2063 uint32_t *ptr;
2064 uint8_t *ptr2;
2065 #endif
2066
2067 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2068
2069 port = addrp->aa_port;
2070 pmport = addrp->aa_pmport;
2071
2072 spkt->satapkt_reason = SATA_PKT_BUSY;
2073
2074 scmd = &spkt->satapkt_cmd;
2075
2076 /* Check if the command is a NCQ command */
2077 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2078 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2079 command_type = AHCI_NCQ_CMD;
2080
2081 /*
2082 * When NCQ is support, system software must determine the
2083 * maximum tag allowed by the device and the HBA, and it
2084 * must use a value not beyond of the lower bound of the two.
2085 *
2086 * Sata module is going to calculate the qdepth and send
2087 * down to HBA driver via sata_cmd.
2088 */
2089 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2090
2091 /*
2092 * At the moment, the driver doesn't support the dynamic
2093 * setting of the maximum ncq depth, and the value can be
2094 * set either during the attach or after hot-plug insertion.
2095 */
2096 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2097 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2098 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2099 "ahci_deliver_satapkt: port %d the max tags for "
2100 "NCQ command is %d", port, ncq_qdepth);
2101 } else {
2102 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2103 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2104 " port %d the max tag for NCQ command is "
2105 "requested to change from %d to %d, at the"
2106 " moment the driver doesn't support the "
2107 "dynamic change so it's going to "
2108 "still use the previous tag value",
2109 instance, port,
2110 ahci_portp->ahciport_max_ncq_tags,
2111 ncq_qdepth);
2112 }
2113 }
2114 }
2115
2116 /* Check if the command is an error retrieval command */
2117 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2118 command_type = AHCI_ERR_RETRI_CMD;
2119
2120 /* Check if the command is an read/write pmult command */
2121 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2122 command_type = AHCI_RDWR_PMULT_CMD;
2123
2124 /* Check if there is an empty command slot */
2125 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2126 addrp, command_type);
2127 if (cmd_slot == AHCI_FAILURE) {
2128 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2129 return (AHCI_FAILURE);
2130 }
2131
2132 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2133 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2134 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2135 cmd_slot, port, (void *)spkt);
2136
2137 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2138 bzero((void *)cmd_table, ahci_cmd_table_size);
2139
2140 /* For data transfer operations, it is the H2D Register FIS */
2141 h2d_register_fisp =
2142 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2143
2144 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2145
2146 /*
2147 * PMP field only make sense when target is a port multiplier or a
2148 * device behind a port multiplier. Otherwise should set it to 0.
2149 */
2150 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2151 SET_FIS_PMP(h2d_register_fisp, pmport);
2152
2153 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2154 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2155 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2156 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2157
2158 switch (scmd->satacmd_addr_type) {
2159
2160 case 0:
2161 /*
2162 * satacmd_addr_type will be 0 for the commands below:
2163 * ATAPI command
2164 * SATAC_IDLE_IM
2165 * SATAC_STANDBY_IM
2166 * SATAC_DOWNLOAD_MICROCODE
2167 * SATAC_FLUSH_CACHE
2168 * SATAC_SET_FEATURES
2169 * SATAC_SMART
2170 * SATAC_ID_PACKET_DEVICE
2171 * SATAC_ID_DEVICE
2172 * SATAC_READ_PORTMULT
2173 * SATAC_WRITE_PORTMULT
2174 */
2175 /* FALLTHRU */
2176
2177 case ATA_ADDR_LBA:
2178 /* FALLTHRU */
2179
2180 case ATA_ADDR_LBA28:
2181 /* LBA[7:0] */
2182 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2183
2184 /* LBA[15:8] */
2185 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2186
2187 /* LBA[23:16] */
2188 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2189
2190 /* LBA [27:24] (also called dev_head) */
2191 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2192
2193 break;
2194
2195 case ATA_ADDR_LBA48:
2196 /* LBA[7:0] */
2197 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2198
2199 /* LBA[15:8] */
2200 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2201
2202 /* LBA[23:16] */
2203 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2204
2205 /* LBA [31:24] */
2206 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2207 scmd->satacmd_lba_low_msb);
2208
2209 /* LBA [39:32] */
2210 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2211 scmd->satacmd_lba_mid_msb);
2212
2213 /* LBA [47:40] */
2214 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2215 scmd->satacmd_lba_high_msb);
2216
2217 /* Set dev_head */
2218 SET_FIS_DEV_HEAD(h2d_register_fisp,
2219 scmd->satacmd_device_reg);
2220
2221 /* Set the extended sector count and features */
2222 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2223 scmd->satacmd_sec_count_msb);
2224 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2225 scmd->satacmd_features_reg_ext);
2226 break;
2227 }
2228
2229 /*
2230 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2231 * filled into features field, and sector count 8:15 is filled into
2232 * features (exp) field. The hba driver doesn't need to anything
2233 * special with regard to this, since sata framework has already
2234 * done so.
2235 *
2236 * However the driver needs to make sure TAG is filled into sector
2237 * field.
2238 */
2239 if (command_type == AHCI_NCQ_CMD) {
2240 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2241 (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2242 }
2243
2244 ncookies = scmd->satacmd_num_dma_cookies;
2245 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2246 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2247 ncookies, ahci_dma_prdt_number);
2248
2249 ASSERT(ncookies <= ahci_dma_prdt_number);
2250 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2251
2252 /* *** now fill the scatter gather list ******* */
2253 for (i = 0; i < ncookies; i++) {
2254 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2255 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2256 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2257 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2258 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2259 scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2260 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2261 scmd->satacmd_dma_cookie_list[i].dmac_size;
2262 }
2263
2264 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2265 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2266 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2267
2268 /* The ACMD field is filled in for ATAPI command */
2269 if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2270 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2271 SATA_ATAPI_MAX_CDB_LEN);
2272 }
2273
2274 /* Set Command Header in Command List */
2275 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2276 BZERO_DESCR_INFO(cmd_header);
2277 BZERO_PRD_BYTE_COUNT(cmd_header);
2278
2279 /* Set the number of entries in the PRD table */
2280 SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2281
2282 /* Set the length of the command in the CFIS area */
2283 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2284
2285 /*
2286 * PMP field only make sense when target is a port multiplier or a
2287 * device behind a port multiplier. Otherwise should set it to 0.
2288 */
2289 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2290 SET_PORT_MULTI_PORT(cmd_header, pmport);
2291
2292 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2293 "sata_data_direction = 0x%x",
2294 scmd->satacmd_flags.sata_data_direction);
2295
2296 /* Set A bit if it is an ATAPI command */
2297 if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2298 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2299
2300 /* Set W bit if data is going to the device */
2301 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2302 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2303
2304 /*
2305 * Set the prefetchable bit - this bit is only valid if the PRDTL
2306 * field is non-zero or the ATAPI 'A' bit is set in the command
2307 * header. This bit cannot be set when using native command
2308 * queuing commands or when using FIS-based switching with a Port
2309 * multiplier.
2310 */
2311 if (command_type != AHCI_NCQ_CMD)
2312 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2313
2314 /*
2315 * Now remember the sata packet in ahciport_slot_pkts[].
2316 * Error retrieval command and r/w port multiplier command will
2317 * be stored specifically for each port.
2318 */
2319 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2320 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2321 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2322
2323 /*
2324 * Keep the timeout value
2325 */
2326 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2327
2328 /*
2329 * If the intial timout is less than 1 tick, then make it longer by
2330 * 1 tick to avoid immediate timeout
2331 */
2332 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2333 ahci_watchdog_timeout)
2334 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2335 ahci_watchdog_timeout;
2336
2337 #if AHCI_DEBUG
2338 if (ahci_debug_flags & AHCIDBG_ATACMD &&
2339 scmd->satacmd_cmd_reg != SATAC_PACKET ||
2340 ahci_debug_flags & AHCIDBG_ATAPICMD &&
2341 scmd->satacmd_cmd_reg == SATAC_PACKET) {
2342
2343 /* Dump the command header and table */
2344 ahci_log(ahci_ctlp, CE_WARN, "\n");
2345 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2346 "0x%p cmd_reg 0x%x port %d", spkt,
2347 scmd->satacmd_cmd_reg, port);
2348 ptr = (uint32_t *)cmd_header;
2349 ahci_log(ahci_ctlp, CE_WARN,
2350 " Command Header:%8x %8x %8x %8x",
2351 ptr[0], ptr[1], ptr[2], ptr[3]);
2352
2353 /* Dump the H2D register FIS */
2354 ptr = (uint32_t *)h2d_register_fisp;
2355 ahci_log(ahci_ctlp, CE_WARN,
2356 " Command FIS: %8x %8x %8x %8x",
2357 ptr[0], ptr[1], ptr[2], ptr[3]);
2358
2359 /* Dump the ACMD register FIS */
2360 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2361 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2362 if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2363 ahci_log(ahci_ctlp, CE_WARN,
2364 " ATAPI command: %2x %2x %2x %2x "
2365 "%2x %2x %2x %2x",
2366 ptr2[8 * i], ptr2[8 * i + 1],
2367 ptr2[8 * i + 2], ptr2[8 * i + 3],
2368 ptr2[8 * i + 4], ptr2[8 * i + 5],
2369 ptr2[8 * i + 6], ptr2[8 * i + 7]);
2370
2371 /* Dump the PRDT */
2372 for (i = 0; i < ncookies; i++) {
2373 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2374 ahci_log(ahci_ctlp, CE_WARN,
2375 " Cookie %d: %8x %8x %8x %8x",
2376 i, ptr[0], ptr[1], ptr[2], ptr[3]);
2377 }
2378 }
2379 #endif
2380
2381 (void) ddi_dma_sync(
2382 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2383 0,
2384 ahci_cmd_table_size,
2385 DDI_DMA_SYNC_FORDEV);
2386
2387 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2388 cmd_slot * sizeof (ahci_cmd_header_t),
2389 sizeof (ahci_cmd_header_t),
2390 DDI_DMA_SYNC_FORDEV);
2391
2392 if ((ahci_check_dma_handle(ahci_portp->
2393 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2394 ahci_check_dma_handle(ahci_portp->
2395 ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2396 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2397 DDI_SERVICE_UNAFFECTED);
2398 return (AHCI_FAILURE);
2399 }
2400
2401 /* Set the corresponding bit in the PxSACT.DS for queued command */
2402 if (command_type == AHCI_NCQ_CMD) {
2403 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2404 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2405 (0x1 << cmd_slot));
2406 }
2407
2408 /* Indicate to the HBA that a command is active. */
2409 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2410 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2411 (0x1 << cmd_slot));
2412
2413 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2414 "exit: port %d", port);
2415
2416 /* Make sure the command is started by the PxSACT/PxCI */
2417 if (ahci_check_acc_handle(ahci_ctlp->
2418 ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2419 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2420 DDI_SERVICE_UNAFFECTED);
2421 return (AHCI_FAILURE);
2422 }
2423
2424 return (cmd_slot);
2425 }
2426
2427 /*
2428 * Called by the sata framework to abort the previously sent packet(s).
2429 *
2430 * Reset device to abort commands.
2431 */
2432 static int
2433 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2434 {
2435 ahci_ctl_t *ahci_ctlp;
2436 ahci_port_t *ahci_portp;
2437 uint32_t slot_status = 0;
2438 uint32_t aborted_tags = 0;
2439 uint32_t finished_tags = 0;
2440 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2441 uint8_t port;
2442 int tmp_slot;
2443 int instance = ddi_get_instance(dip);
2444
2445 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2446 port = ahci_ctlp->ahcictl_cport_to_port[cport];
2447
2448 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2449 "ahci_tran_abort enter: port %d", port);
2450
2451 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2452 mutex_enter(&ahci_portp->ahciport_mutex);
2453
2454 /*
2455 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2456 * commands are being mopped, therefore there is nothing else to do
2457 */
2458 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2459 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2460 "ahci_tran_abort: port %d is in "
2461 "mopping process, so just return directly ", port);
2462 mutex_exit(&ahci_portp->ahciport_mutex);
2463 return (SATA_SUCCESS);
2464 }
2465
2466 /*
2467 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2468 * command is being executed so no other commands is outstanding,
2469 * nothing to do.
2470 */
2471 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2472 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2473 "ahci_tran_abort: port %d is reading/writing "
2474 "port multiplier, so just return directly ", port);
2475 mutex_exit(&ahci_portp->ahciport_mutex);
2476 return (SATA_SUCCESS);
2477 }
2478
2479 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2480 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2481 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2482 /*
2483 * In case the targer driver would send the request before
2484 * sata framework can have the opportunity to process those
2485 * event reports.
2486 */
2487 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2488 spkt->satapkt_device.satadev_state =
2489 ahci_portp->ahciport_port_state;
2490 ahci_update_sata_registers(ahci_ctlp, port,
2491 &spkt->satapkt_device);
2492 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2493 "ahci_tran_abort returning SATA_FAILURE while "
2494 "port in FAILED/SHUTDOWN/PWROFF state: "
2495 "port: %d", port);
2496 mutex_exit(&ahci_portp->ahciport_mutex);
2497 return (SATA_FAILURE);
2498 }
2499
2500 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2501 /*
2502 * ahci_intr_phyrdy_change() may have rendered it to
2503 * AHCI_PORT_TYPE_NODEV.
2504 */
2505 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2506 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2507 spkt->satapkt_device.satadev_state =
2508 ahci_portp->ahciport_port_state;
2509 ahci_update_sata_registers(ahci_ctlp, port,
2510 &spkt->satapkt_device);
2511 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2512 "ahci_tran_abort returning SATA_FAILURE while "
2513 "no device attached: port: %d", port);
2514 mutex_exit(&ahci_portp->ahciport_mutex);
2515 return (SATA_FAILURE);
2516 }
2517
2518 if (flag == SATA_ABORT_ALL_PACKETS) {
2519 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2520 aborted_tags = ahci_portp->ahciport_pending_tags;
2521 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2522 aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2523
2524 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2525 instance, port);
2526 } else {
2527 aborted_tags = 0xffffffff;
2528 /*
2529 * Aborting one specific packet, first search the
2530 * ahciport_slot_pkts[] list for matching spkt.
2531 */
2532 for (tmp_slot = 0;
2533 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2534 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2535 aborted_tags = (0x1 << tmp_slot);
2536 break;
2537 }
2538 }
2539
2540 if (aborted_tags == 0xffffffff) {
2541 /* request packet is not on the pending list */
2542 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2543 "Cannot find the aborting pkt 0x%p on the "
2544 "pending list", (void *)spkt);
2545 ahci_update_sata_registers(ahci_ctlp, port,
2546 &spkt->satapkt_device);
2547 mutex_exit(&ahci_portp->ahciport_mutex);
2548 return (SATA_FAILURE);
2549 }
2550 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2551 instance, port, (void *)spkt);
2552 }
2553
2554 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2555 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2556 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2557 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2558 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2559 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2560
2561 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2562 ahci_portp->ahciport_mop_in_progress++;
2563
2564 /*
2565 * To abort the packet(s), first we are trying to clear PxCMD.ST
2566 * to stop the port, and if the port can be stopped
2567 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2568 * then we just send back the aborted packet(s) with ABORTED flag
2569 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2570 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2571 * perform a COMRESET.
2572 */
2573 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2574 ahci_portp, port, NULL, NULL);
2575
2576 /*
2577 * Compute which have finished and which need to be retried.
2578 *
2579 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2580 * minus the slot_status. The aborted_tags has to be deducted by
2581 * finished_tags since we can't possibly abort a tag which had finished
2582 * already.
2583 */
2584 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2585 finished_tags = ahci_portp->ahciport_pending_tags &
2586 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2587 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2588 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2589 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2590
2591 aborted_tags &= ~finished_tags;
2592
2593 ahci_mop_commands(ahci_ctlp,
2594 ahci_portp,
2595 slot_status,
2596 0, /* failed tags */
2597 0, /* timeout tags */
2598 aborted_tags,
2599 0); /* reset tags */
2600
2601 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2602 mutex_exit(&ahci_portp->ahciport_mutex);
2603
2604 return (SATA_SUCCESS);
2605 }
2606
2607 /*
2608 * Used to do device reset and reject all the pending packets on a device
2609 * during the reset operation.
2610 *
2611 * NOTE: ONLY called by ahci_tran_reset_dport
2612 */
2613 static int
2614 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2615 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2616 {
2617 uint32_t slot_status = 0;
2618 uint32_t reset_tags = 0;
2619 uint32_t finished_tags = 0;
2620 uint8_t port = addrp->aa_port;
2621 sata_device_t sdevice;
2622 int ret;
2623
2624 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2625
2626 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2627 "ahci_reset_device_reject_pkts on port: %d", port);
2628
2629 /*
2630 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2631 * commands are being mopped, therefore there is nothing else to do
2632 */
2633 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2634 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2635 "ahci_reset_device_reject_pkts: port %d is in "
2636 "mopping process, so return directly ", port);
2637 return (SATA_SUCCESS);
2638 }
2639
2640 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2641 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2642 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2643 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2644 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2645 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2646 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2647 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2648 }
2649
2650 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2651 != AHCI_SUCCESS) {
2652 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2653 "Try to do a port reset after software "
2654 "reset failed", port);
2655 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2656 if (ret != AHCI_SUCCESS) {
2657 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2658 "ahci_reset_device_reject_pkts: port %d "
2659 "failed", port);
2660 return (SATA_FAILURE);
2661 }
2662 }
2663 /* Set the reset in progress flag */
2664 ahci_portp->ahciport_reset_in_progress = 1;
2665
2666 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2667 ahci_portp->ahciport_mop_in_progress++;
2668
2669 /* Indicate to the framework that a reset has happened */
2670 bzero((void *)&sdevice, sizeof (sata_device_t));
2671 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2672 sdevice.satadev_addr.pmport = 0;
2673 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2674 sdevice.satadev_state = SATA_DSTATE_RESET |
2675 SATA_DSTATE_PWR_ACTIVE;
2676 mutex_exit(&ahci_portp->ahciport_mutex);
2677 sata_hba_event_notify(
2678 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2679 &sdevice,
2680 SATA_EVNT_DEVICE_RESET);
2681 mutex_enter(&ahci_portp->ahciport_mutex);
2682
2683 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2684 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2685
2686 /* Next try to mop the pending commands */
2687 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2688 finished_tags = ahci_portp->ahciport_pending_tags &
2689 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2690 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2691 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2692 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2693
2694 reset_tags &= ~finished_tags;
2695
2696 ahci_mop_commands(ahci_ctlp,
2697 ahci_portp,
2698 slot_status,
2699 0, /* failed tags */
2700 0, /* timeout tags */
2701 0, /* aborted tags */
2702 reset_tags); /* reset tags */
2703
2704 return (SATA_SUCCESS);
2705 }
2706
2707 /*
2708 * Used to do device reset and reject all the pending packets on a device
2709 * during the reset operation.
2710 *
2711 * NOTE: ONLY called by ahci_tran_reset_dport
2712 */
2713 static int
2714 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2715 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2716 {
2717 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2718 uint8_t port = addrp->aa_port;
2719 uint8_t pmport = addrp->aa_pmport;
2720 sata_device_t sdevice;
2721
2722 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2723
2724 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2725 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2726
2727 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2728 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2729 "ahci_reset_pmdevice_reject_pkts: port %d is in "
2730 "mopping process, so return directly ", port);
2731 return (SATA_SUCCESS);
2732 }
2733
2734 /* Checking for outstanding commands */
2735 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2736 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2737 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2738 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2739 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2740 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2741 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2742 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2743 }
2744
2745 /* Issue SOFTWARE reset command. */
2746 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2747 != AHCI_SUCCESS) {
2748 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2749 "Try to do a port reset after software "
2750 "reset failed", port);
2751 return (SATA_FAILURE);
2752 }
2753
2754 /* Set the reset in progress flag */
2755 ahci_portp->ahciport_reset_in_progress = 1;
2756
2757 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2758 ahci_portp->ahciport_mop_in_progress++;
2759
2760 /* Indicate to the framework that a reset has happened */
2761 bzero((void *)&sdevice, sizeof (sata_device_t));
2762 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2763 sdevice.satadev_addr.pmport = pmport;
2764 if (AHCI_ADDR_IS_PMULT(addrp))
2765 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2766 else
2767 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2768 sdevice.satadev_state = SATA_DSTATE_RESET |
2769 SATA_DSTATE_PWR_ACTIVE;
2770 mutex_exit(&ahci_portp->ahciport_mutex);
2771 sata_hba_event_notify(
2772 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2773 &sdevice,
2774 SATA_EVNT_DEVICE_RESET);
2775 mutex_enter(&ahci_portp->ahciport_mutex);
2776
2777 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2778 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2779 port, pmport);
2780
2781 /* Next try to mop the pending commands */
2782 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2783 finished_tags = ahci_portp->ahciport_pending_tags &
2784 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2785 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2786 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2787 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2788 reset_tags &= ~finished_tags;
2789
2790 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2791 "reset_tags = %x, finished_tags = %x, slot_status = %x",
2792 reset_tags, finished_tags, slot_status);
2793
2794 /*
2795 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2796 * try to reset a single device behind a port multiplier will
2797 * terminate all the commands on that HBA port. We need mop these
2798 * commands as well.
2799 */
2800 ahci_mop_commands(ahci_ctlp,
2801 ahci_portp,
2802 slot_status,
2803 0, /* failed tags */
2804 0, /* timeout tags */
2805 0, /* aborted tags */
2806 reset_tags); /* reset tags */
2807
2808 return (SATA_SUCCESS);
2809 }
2810
2811 /*
2812 * Used to do port reset and reject all the pending packets on a port during
2813 * the reset operation.
2814 */
2815 static int
2816 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2817 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2818 {
2819 uint32_t slot_status = 0;
2820 uint32_t reset_tags = 0;
2821 uint32_t finished_tags = 0;
2822 uint8_t port = addrp->aa_port;
2823
2824 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2825
2826 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2827 "ahci_reset_port_reject_pkts at port: %d", port);
2828
2829 /*
2830 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2831 * commands are being mopped, therefore there is nothing else to do
2832 */
2833 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2834 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2835 "ahci_reset_port_reject_pkts: port %d is in "
2836 "mopping process, so return directly ", port);
2837 return (SATA_SUCCESS);
2838 }
2839
2840 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2841 ahci_portp->ahciport_mop_in_progress++;
2842
2843 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2844 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2845 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2846 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2847 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2848 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2849 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2850 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2851 }
2852
2853 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2854 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2855 NULL) != AHCI_SUCCESS) {
2856
2857 /* Clear mop flag */
2858 ahci_portp->ahciport_mop_in_progress--;
2859 if (ahci_portp->ahciport_mop_in_progress == 0)
2860 ahci_portp->ahciport_flags &=
2861 ~AHCI_PORT_FLAG_MOPPING;
2862 return (SATA_FAILURE);
2863 }
2864
2865 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2866 finished_tags = ahci_portp->ahciport_pending_tags &
2867 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2868 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2869 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2870 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2871
2872 reset_tags &= ~finished_tags;
2873
2874 ahci_mop_commands(ahci_ctlp,
2875 ahci_portp,
2876 slot_status,
2877 0, /* failed tags */
2878 0, /* timeout tags */
2879 0, /* aborted tags */
2880 reset_tags); /* reset tags */
2881
2882 return (SATA_SUCCESS);
2883 }
2884
2885 /*
2886 * Used to do hba reset and reject all the pending packets on all ports
2887 * during the reset operation.
2888 */
2889 static int
2890 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2891 {
2892 ahci_port_t *ahci_portp;
2893 uint32_t slot_status[AHCI_MAX_PORTS];
2894 uint32_t reset_tags[AHCI_MAX_PORTS];
2895 uint32_t finished_tags[AHCI_MAX_PORTS];
2896 int port;
2897 int ret = SATA_SUCCESS;
2898
2899 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2900 "ahci_reset_hba_reject_pkts enter", NULL);
2901
2902 bzero(slot_status, sizeof (slot_status));
2903 bzero(reset_tags, sizeof (reset_tags));
2904 bzero(finished_tags, sizeof (finished_tags));
2905
2906 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2907 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2908 continue;
2909 }
2910
2911 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2912
2913 mutex_enter(&ahci_portp->ahciport_mutex);
2914 ahci_portp->ahciport_reset_in_progress = 1;
2915 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2916 slot_status[port] = ddi_get32(
2917 ahci_ctlp->ahcictl_ahci_acc_handle,
2918 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2919 reset_tags[port] = slot_status[port] &
2920 AHCI_SLOT_MASK(ahci_ctlp);
2921 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2922 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2923 port, reset_tags[port],
2924 ahci_portp->ahciport_pending_tags);
2925 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2926 slot_status[port] = ddi_get32(
2927 ahci_ctlp->ahcictl_ahci_acc_handle,
2928 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2929 reset_tags[port] = slot_status[port] &
2930 AHCI_NCQ_SLOT_MASK(ahci_portp);
2931 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2932 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2933 port, reset_tags[port],
2934 ahci_portp->ahciport_pending_tags);
2935 }
2936 mutex_exit(&ahci_portp->ahciport_mutex);
2937 }
2938
2939 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
2940 ret = SATA_FAILURE;
2941 }
2942
2943 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2944 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2945 continue;
2946 }
2947
2948 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2949
2950 mutex_enter(&ahci_portp->ahciport_mutex);
2951 /*
2952 * To prevent recursive enter to ahci_mop_commands, we need
2953 * check AHCI_PORT_FLAG_MOPPING flag.
2954 */
2955 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2956 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2957 "ahci_reset_hba_reject_pkts: port %d is in "
2958 "mopping process, so return directly ", port);
2959 mutex_exit(&ahci_portp->ahciport_mutex);
2960 continue;
2961 }
2962
2963 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2964 ahci_portp->ahciport_mop_in_progress++;
2965
2966 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2967 finished_tags[port] =
2968 ahci_portp->ahciport_pending_tags &
2969 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
2970 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2971 finished_tags[port] =
2972 ahci_portp->ahciport_pending_ncq_tags &
2973 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
2974
2975 reset_tags[port] &= ~finished_tags[port];
2976
2977 ahci_mop_commands(ahci_ctlp,
2978 ahci_portp,
2979 slot_status[port],
2980 0, /* failed tags */
2981 0, /* timeout tags */
2982 0, /* aborted tags */
2983 reset_tags[port]); /* reset tags */
2984 mutex_exit(&ahci_portp->ahciport_mutex);
2985 }
2986 out:
2987 return (ret);
2988 }
2989
2990 /*
2991 * Called by sata framework to reset a port(s) or device.
2992 */
2993 static int
2994 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
2995 {
2996 ahci_ctl_t *ahci_ctlp;
2997 ahci_port_t *ahci_portp;
2998 ahci_addr_t addr;
2999 uint8_t cport = sd->satadev_addr.cport;
3000 uint8_t pmport = sd->satadev_addr.pmport;
3001 uint8_t port;
3002 int ret = SATA_SUCCESS;
3003 int instance = ddi_get_instance(dip);
3004
3005 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3006 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3007 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3008
3009 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3010
3011 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3012 "ahci_tran_reset_dport enter: cport %d", cport);
3013
3014 switch (sd->satadev_addr.qual) {
3015 case SATA_ADDR_PMPORT:
3016 /*
3017 * If we want to issue a COMRESET on a pmport, we need to
3018 * reject the outstanding commands on that pmport. According
3019 * to AHCI spec, PxCI register could only be cleared by
3020 * clearing PxCMD.ST, which will halt the controller port - as
3021 * well as other pmports.
3022 *
3023 * Therefore we directly reset the controller port for
3024 * simplicity. ahci_tran_probe_port() will handle reset stuff
3025 * like initializing the given pmport.
3026 */
3027 /* FALLTHRU */
3028 case SATA_ADDR_CPORT:
3029 /* Port reset */
3030 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3031 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3032 "port %d reset port", instance, port);
3033
3034 mutex_enter(&ahci_portp->ahciport_mutex);
3035 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3036 mutex_exit(&ahci_portp->ahciport_mutex);
3037
3038 break;
3039
3040 case SATA_ADDR_DPMPORT:
3041 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3042 "port %d:%d reset device", instance, port, pmport);
3043 /* FALLTHRU */
3044 case SATA_ADDR_DCPORT:
3045 /* Device reset */
3046 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3047 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3048 "port %d reset device", instance, port);
3049
3050 mutex_enter(&ahci_portp->ahciport_mutex);
3051 /*
3052 * software reset request must be sent to SATA_PMULT_HOSTPORT
3053 * if target is a port multiplier:
3054 */
3055 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3056 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3057 AHCI_ADDR_SET_PMULT(&addr, port);
3058
3059 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3060 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3061 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3062 /*
3063 * In case the targer driver would send the request
3064 * before sata framework can have the opportunity to
3065 * process those event reports.
3066 */
3067 sd->satadev_state = ahci_portp->ahciport_port_state;
3068 ahci_update_sata_registers(ahci_ctlp, port, sd);
3069 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3070 "ahci_tran_reset_dport returning SATA_FAILURE "
3071 "while port in FAILED/SHUTDOWN/PWROFF state: "
3072 "port: %d", port);
3073 mutex_exit(&ahci_portp->ahciport_mutex);
3074 ret = SATA_FAILURE;
3075 break;
3076 }
3077
3078 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3079 SATA_DTYPE_NONE) {
3080 /*
3081 * ahci_intr_phyrdy_change() may have rendered it to
3082 * AHCI_PORT_TYPE_NODEV.
3083 */
3084 sd->satadev_type = SATA_DTYPE_NONE;
3085 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3086 &addr);
3087 ahci_update_sata_registers(ahci_ctlp, port, sd);
3088 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3089 "ahci_tran_reset_dport returning SATA_FAILURE "
3090 "while no device attached: port: %d", port);
3091 mutex_exit(&ahci_portp->ahciport_mutex);
3092 ret = SATA_FAILURE;
3093 break;
3094 }
3095
3096 if (AHCI_ADDR_IS_PORT(&addr)) {
3097 ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3098 ahci_portp, &addr);
3099 } else {
3100 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3101 ahci_portp, &addr);
3102 }
3103
3104 mutex_exit(&ahci_portp->ahciport_mutex);
3105 break;
3106
3107 case SATA_ADDR_CNTRL:
3108 /* Reset the whole controller */
3109 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3110 "reset the whole hba", instance);
3111 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3112 break;
3113
3114 default:
3115 ret = SATA_FAILURE;
3116 }
3117
3118 return (ret);
3119 }
3120
3121 /*
3122 * Called by sata framework to activate a port as part of hotplug.
3123 * (cfgadm -c connect satax/y)
3124 * Support port multiplier.
3125 */
3126 static int
3127 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3128 {
3129 ahci_ctl_t *ahci_ctlp;
3130 ahci_port_t *ahci_portp;
3131 ahci_addr_t addr;
3132 uint8_t cport = satadev->satadev_addr.cport;
3133 uint8_t pmport = satadev->satadev_addr.pmport;
3134 uint8_t port;
3135 int instance = ddi_get_instance(dip);
3136
3137 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3138 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3139
3140 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3141 "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3142
3143 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3144
3145 mutex_enter(&ahci_portp->ahciport_mutex);
3146 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3147 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3148
3149 if (AHCI_ADDR_IS_PORT(&addr)) {
3150 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3151 instance, port);
3152
3153 /* Enable the interrupts on the port */
3154 ahci_enable_port_intrs(ahci_ctlp, port);
3155
3156 /*
3157 * Reset the port so that the PHY communication would be
3158 * re-established. But this reset is an internal operation
3159 * and the sata module doesn't need to know about it.
3160 * Moreover, the port with a device attached will be started
3161 * too.
3162 */
3163 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3164 ahci_portp, port,
3165 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3166 NULL);
3167
3168 /*
3169 * Need to check the link status and device status of the port
3170 * and consider raising power if the port was in D3 state
3171 */
3172 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3173 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3174 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3175 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3176 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3177 instance, port, pmport);
3178 /* AHCI_ADDR_PMPORT */
3179 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3180 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3181 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3182 }
3183
3184 satadev->satadev_state = ahci_portp->ahciport_port_state;
3185
3186 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3187
3188 mutex_exit(&ahci_portp->ahciport_mutex);
3189 return (SATA_SUCCESS);
3190 }
3191
3192 /*
3193 * Called by sata framework to deactivate a port as part of hotplug.
3194 * (cfgadm -c disconnect satax/y)
3195 * Support port multiplier.
3196 */
3197 static int
3198 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3199 {
3200 ahci_ctl_t *ahci_ctlp;
3201 ahci_port_t *ahci_portp;
3202 ahci_addr_t addr;
3203 uint8_t cport = satadev->satadev_addr.cport;
3204 uint8_t pmport = satadev->satadev_addr.pmport;
3205 uint8_t port;
3206 uint32_t port_scontrol;
3207 int instance = ddi_get_instance(dip);
3208
3209 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3210 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3211
3212 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3213 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3214
3215 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3216 mutex_enter(&ahci_portp->ahciport_mutex);
3217 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3218 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3219
3220 if (AHCI_ADDR_IS_PORT(&addr)) {
3221 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3222 instance, port);
3223
3224 /* Disable the interrupts on the port */
3225 ahci_disable_port_intrs(ahci_ctlp, port);
3226
3227 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3228
3229 /* First to abort all the pending commands */
3230 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3231
3232 /* Then stop the port */
3233 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3234 ahci_portp, port);
3235 }
3236
3237 /* Next put the PHY offline */
3238 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3239 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3240 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3241 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3242 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3243 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3244 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3245 instance, port, pmport);
3246
3247 ahci_disable_port_intrs(ahci_ctlp, port);
3248 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3249 != SATA_DTYPE_NONE)
3250 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3251
3252 /* Re-enable the interrupts for the other pmports */
3253 ahci_enable_port_intrs(ahci_ctlp, port);
3254 }
3255
3256 /* Update port state */
3257 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3258 satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3259
3260 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3261
3262 mutex_exit(&ahci_portp->ahciport_mutex);
3263 return (SATA_SUCCESS);
3264 }
3265
3266 /*
3267 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3268 * when a device is unplugged or a port is deactivated.
3269 */
3270 static void
3271 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3272 ahci_port_t *ahci_portp, uint8_t port)
3273 {
3274 uint32_t slot_status = 0;
3275 uint32_t abort_tags = 0;
3276
3277 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3278
3279 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3280 "ahci_reject_all_abort_pkts at port: %d", port);
3281
3282 /* Read/write port multiplier command takes highest priority */
3283 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3284 slot_status = 0x1;
3285 abort_tags = 0x1;
3286 goto out;
3287 }
3288
3289 /*
3290 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3291 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3292 * to get the error data, if yes when the device is removed, the
3293 * command needs to be aborted too.
3294 */
3295 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3296 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3297 slot_status = 0x1;
3298 abort_tags = 0x1;
3299 goto out;
3300 } else {
3301 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3302 "ahci_reject_all_abort_pkts return directly "
3303 "port %d no needs to reject any outstanding "
3304 "commands", port);
3305 return;
3306 }
3307 }
3308
3309 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3310 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3311 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3312 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3313 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3314 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3315 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3316 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3317 }
3318
3319 out:
3320 /* No need to do mop when there is no outstanding commands */
3321 if (slot_status != 0) {
3322 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3323 ahci_portp->ahciport_mop_in_progress++;
3324
3325 ahci_mop_commands(ahci_ctlp,
3326 ahci_portp,
3327 slot_status,
3328 0, /* failed tags */
3329 0, /* timeout tags */
3330 abort_tags, /* aborting tags */
3331 0); /* reset tags */
3332 }
3333 }
3334
3335 #if defined(__lock_lint)
3336 static int
3337 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3338 {
3339 return (SATA_SUCCESS);
3340 }
3341 #endif
3342
3343 /*
3344 * Initialize fma capabilities and register with IO fault services.
3345 */
3346 static void
3347 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3348 {
3349 /*
3350 * Need to change iblock to priority for new MSI intr
3351 */
3352 ddi_iblock_cookie_t fm_ibc;
3353
3354 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3355 ahci_ctlp->ahcictl_dip,
3356 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3357 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3358 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3359
3360 /* Only register with IO Fault Services if we have some capability */
3361 if (ahci_ctlp->ahcictl_fm_cap) {
3362 /* Adjust access and dma attributes for FMA */
3363 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3364 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3365 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3366 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3367 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3368
3369 /*
3370 * Register capabilities with IO Fault Services.
3371 * ahcictl_fm_cap will be updated to indicate
3372 * capabilities actually supported (not requested.)
3373 */
3374 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3375 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3376
3377 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3378 cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3379 ddi_get_instance(ahci_ctlp->ahcictl_dip));
3380 return;
3381 }
3382 /*
3383 * Initialize pci ereport capabilities if ereport
3384 * capable (should always be.)
3385 */
3386 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3387 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3388 pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3389 }
3390
3391 /*
3392 * Register error callback if error callback capable.
3393 */
3394 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3395 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3396 ahci_fm_error_cb, (void *) ahci_ctlp);
3397 }
3398
3399 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3400 "ahci_fm_fini: fma enabled.", NULL);
3401 }
3402 }
3403
3404 /*
3405 * Releases fma capabilities and un-registers with IO fault services.
3406 */
3407 static void
3408 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3409 {
3410 /* Only unregister FMA capabilities if registered */
3411 if (ahci_ctlp->ahcictl_fm_cap) {
3412 /*
3413 * Un-register error callback if error callback capable.
3414 */
3415 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3416 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3417 }
3418
3419 /*
3420 * Release any resources allocated by pci_ereport_setup()
3421 */
3422 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3423 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3424 pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3425 }
3426
3427 /* Unregister from IO Fault Services */
3428 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3429
3430 /* Adjust access and dma attributes for FMA */
3431 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3432 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3433 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3434 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3435 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3436
3437 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3438 "ahci_fm_fini: fma disabled.", NULL);
3439 }
3440 }
3441
3442 /*ARGSUSED*/
3443 static int
3444 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3445 {
3446 /*
3447 * as the driver can always deal with an error in any dma or
3448 * access handle, we can just return the fme_status value.
3449 */
3450 pci_ereport_post(dip, err, NULL);
3451 return (err->fme_status);
3452 }
3453
3454 int
3455 ahci_check_acc_handle(ddi_acc_handle_t handle)
3456 {
3457 ddi_fm_error_t de;
3458
3459 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3460 return (de.fme_status);
3461 }
3462
3463 int
3464 ahci_check_dma_handle(ddi_dma_handle_t handle)
3465 {
3466 ddi_fm_error_t de;
3467
3468 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3469 return (de.fme_status);
3470 }
3471
3472 /*
3473 * Generate an ereport
3474 */
3475 void
3476 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3477 {
3478 uint64_t ena;
3479 char buf[FM_MAX_CLASS];
3480
3481 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3482 ena = fm_ena_generate(0, FM_ENA_FMT1);
3483 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3484 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3485 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3486 FM_EREPORT_VERSION, NULL);
3487 }
3488 }
3489
3490 /*
3491 * Check if all handles are correctly allocated.
3492 */
3493 static int
3494 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3495 {
3496 int port;
3497
3498 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3499 return (DDI_FAILURE);
3500 }
3501
3502 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3503 ahci_port_t *ahci_portp;
3504
3505 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3506 continue;
3507
3508 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3509
3510 mutex_enter(&ahci_portp->ahciport_mutex);
3511 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3512 mutex_exit(&ahci_portp->ahciport_mutex);
3513 return (DDI_FAILURE);
3514 }
3515 mutex_exit(&ahci_portp->ahciport_mutex);
3516 }
3517
3518 return (DDI_SUCCESS);
3519 }
3520
3521 /*
3522 * Check the access handles for the controller. Note that
3523 * ahcictl_pci_conf_handle is only used in attach process.
3524 */
3525 static int
3526 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3527 {
3528 if ((ahci_check_acc_handle(ahci_ctlp->
3529 ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3530 (ahci_check_acc_handle(ahci_ctlp->
3531 ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3532 return (DDI_FAILURE);
3533 }
3534 return (DDI_SUCCESS);
3535 }
3536
3537 /*
3538 * Check the DMA handles and the access handles of a controller port.
3539 */
3540 static int
3541 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3542 {
3543 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3544 int slot;
3545
3546 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3547
3548 if ((ahci_check_dma_handle(ahci_portp->
3549 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3550 (ahci_check_dma_handle(ahci_portp->
3551 ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3552 (ahci_check_acc_handle(ahci_portp->
3553 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3554 (ahci_check_acc_handle(ahci_portp->
3555 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3556 return (DDI_FAILURE);
3557 }
3558 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3559 if (ahci_check_slot_handle(ahci_portp, slot)
3560 != DDI_SUCCESS) {
3561 return (DDI_FAILURE);
3562 }
3563 }
3564 return (DDI_SUCCESS);
3565 }
3566
3567 /*
3568 * Check the DMA handles and the access handles of a cmd table slot.
3569 */
3570 static int
3571 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3572 {
3573 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3574
3575 if ((ahci_check_acc_handle(ahci_portp->
3576 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3577 (ahci_check_dma_handle(ahci_portp->
3578 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3579 return (DDI_FAILURE);
3580 }
3581 return (DDI_SUCCESS);
3582 }
3583
3584 /*
3585 * Allocate the ports structure, only called by ahci_attach
3586 */
3587 static int
3588 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3589 {
3590 int port, cport = 0;
3591
3592 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3593 "ahci_alloc_ports_state enter", NULL);
3594
3595 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3596
3597 /* Allocate structures only for the implemented ports */
3598 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3599 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3600 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3601 "hba port %d not implemented", port);
3602 continue;
3603 }
3604
3605 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3606 ahci_ctlp->ahcictl_port_to_cport[port] =
3607 (uint8_t)cport++;
3608
3609 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3610 goto err_out;
3611 }
3612 }
3613
3614 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3615 return (AHCI_SUCCESS);
3616
3617 err_out:
3618 for (port--; port >= 0; port--) {
3619 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3620 ahci_dealloc_port_state(ahci_ctlp, port);
3621 }
3622 }
3623
3624 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3625 return (AHCI_FAILURE);
3626 }
3627
3628 /*
3629 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3630 */
3631 static void
3632 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3633 {
3634 int port;
3635
3636 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3637 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3638 /* if this port is implemented by the HBA */
3639 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3640 ahci_dealloc_port_state(ahci_ctlp, port);
3641 }
3642 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3643 }
3644
3645 /*
3646 * Drain the taskq.
3647 */
3648 static void
3649 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3650 {
3651 ahci_port_t *ahci_portp;
3652 int port;
3653
3654 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3655 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3656 continue;
3657 }
3658
3659 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3660
3661 mutex_enter(&ahci_portp->ahciport_mutex);
3662 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3663 mutex_exit(&ahci_portp->ahciport_mutex);
3664 }
3665 }
3666
3667 /*
3668 * Initialize the controller and all ports. And then try to start the ports
3669 * if there are devices attached.
3670 *
3671 * This routine can be called from three seperate cases: DDI_ATTACH,
3672 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3673 * other two cases; device signature probing are attempted only during
3674 * DDI_ATTACH case.
3675 */
3676 static int
3677 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3678 {
3679 ahci_port_t *ahci_portp;
3680 ahci_addr_t addr;
3681 int port;
3682
3683 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3684 "ahci_initialize_controller enter", NULL);
3685
3686 /* Disable the whole controller interrupts */
3687 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3688 ahci_disable_all_intrs(ahci_ctlp);
3689 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3690
3691 /* Initialize the implemented ports and structures */
3692 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3693 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3694 continue;
3695 }
3696
3697 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3698 mutex_enter(&ahci_portp->ahciport_mutex);
3699
3700 /*
3701 * Ensure that the controller is not in the running state
3702 * by checking every implemented port's PxCMD register
3703 */
3704 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3705
3706 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3707 != AHCI_SUCCESS) {
3708 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3709 "ahci_initialize_controller: failed to "
3710 "initialize port %d", port);
3711 /*
3712 * Set the port state to SATA_PSTATE_FAILED if
3713 * failed to initialize it.
3714 */
3715 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3716 }
3717
3718 mutex_exit(&ahci_portp->ahciport_mutex);
3719 }
3720
3721 /* Enable the whole controller interrupts */
3722 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3723 ahci_enable_all_intrs(ahci_ctlp);
3724 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3725
3726 return (AHCI_SUCCESS);
3727 }
3728
3729 /*
3730 * Reverse of ahci_initialize_controller()
3731 *
3732 * We only need to stop the ports and disable the interrupt.
3733 */
3734 static void
3735 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3736 {
3737 ahci_port_t *ahci_portp;
3738 int port;
3739
3740 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3741 "ahci_uninitialize_controller enter", NULL);
3742
3743 /* disable all the interrupts. */
3744 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3745 ahci_disable_all_intrs(ahci_ctlp);
3746 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3747
3748 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3749 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3750 continue;
3751 }
3752
3753 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3754
3755 /* Stop the port by clearing PxCMD.ST */
3756 mutex_enter(&ahci_portp->ahciport_mutex);
3757
3758 /*
3759 * Here we must disable the port interrupt because
3760 * ahci_disable_all_intrs only clear GHC.IE, and IS
3761 * register will be still set if PxIE is enabled.
3762 * When ahci shares one IRQ with other drivers, the
3763 * intr handler may claim the intr mistakenly.
3764 */
3765 ahci_disable_port_intrs(ahci_ctlp, port);
3766 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3767 ahci_portp, port);
3768 mutex_exit(&ahci_portp->ahciport_mutex);
3769 }
3770 }
3771
3772 /*
3773 * ahci_alloc_pmult()
3774 * 1. Setting HBA port registers which are necessary for a port multiplier.
3775 * (Set PxCMD.PMA while PxCMD.ST is '0')
3776 * 2. Allocate ahci_pmult_info structure.
3777 *
3778 * NOTE: Must stop port before the function is called.
3779 */
3780 static void
3781 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3782 {
3783 uint32_t port_cmd_status;
3784 uint8_t port = ahci_portp->ahciport_port_num;
3785
3786 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3787
3788 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3789 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3790
3791 /* The port must have been stopped before. */
3792 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3793
3794 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3795 /* set PMA bit */
3796 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3797 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3798 port_cmd_status|AHCI_CMD_STATUS_PMA);
3799
3800 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3801 "ahci_alloc_pmult: "
3802 "PxCMD.PMA bit set at port %d.", port);
3803 }
3804
3805 /* Allocate port multiplier information structure */
3806 if (ahci_portp->ahciport_pmult_info == NULL) {
3807 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3808 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3809 }
3810
3811 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3812 }
3813
3814 /*
3815 * ahci_dealloc_pmult()
3816 * 1. Clearing related registers when a port multiplier is detached.
3817 * (Clear PxCMD.PMA while PxCMD.ST is '0')
3818 * 2. Deallocate ahci_pmult_info structure.
3819 *
3820 * NOTE: Must stop port before the function is called.
3821 */
3822 static void
3823 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3824 {
3825 uint32_t port_cmd_status;
3826 uint8_t port = ahci_portp->ahciport_port_num;
3827
3828 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3829
3830 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3831 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3832
3833 if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3834 /* Clear PMA bit */
3835 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3836 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3837 (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3838
3839 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3840 "ahci_dealloc_pmult: "
3841 "PxCMD.PMA bit cleared at port %d.", port);
3842 }
3843
3844 /* Release port multiplier information structure */
3845 if (ahci_portp->ahciport_pmult_info != NULL) {
3846 kmem_free(ahci_portp->ahciport_pmult_info,
3847 sizeof (ahci_pmult_info_t));
3848 ahci_portp->ahciport_pmult_info = NULL;
3849 }
3850 }
3851
3852 /*
3853 * Staggered Spin-up.
3854 */
3855 static void
3856 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3857 {
3858 uint32_t cap_status;
3859 uint32_t port_cmd_status;
3860
3861 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3862
3863 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3864 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3865
3866 /* Check for staggered spin-up support */
3867 if (!(cap_status & AHCI_HBA_CAP_SSS))
3868 return;
3869
3870 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3871 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3872
3873 /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3874 if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3875 return;
3876
3877 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3878
3879 /* Set PxCMD.SUD */
3880 port_cmd_status |= AHCI_CMD_STATUS_SUD;
3881 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3882 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3883 port_cmd_status);
3884 }
3885
3886 /*
3887 * The routine is to initialize a port. First put the port in NotRunning
3888 * state, then enable port interrupt and clear Serror register. And under
3889 * AHCI_ATTACH case, find device signature and then try to start the port.
3890 *
3891 * Called by
3892 * 1. ahci_initialize_controller
3893 * 2. ahci_intr_phyrdy_change (hotplug)
3894 */
3895 static int
3896 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3897 ahci_addr_t *addrp)
3898 {
3899 uint32_t port_sstatus, port_task_file, port_cmd_status;
3900 uint8_t port = addrp->aa_port;
3901 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */
3902 int ret;
3903
3904 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3905
3906 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */
3907 ASSERT(AHCI_ADDR_IS_PORT(addrp));
3908
3909 /*
3910 * At the time being, only probe ports/devices and get the types of
3911 * attached devices during DDI_ATTACH. In fact, the device can be
3912 * changed during power state changes, but at the time being, we
3913 * don't support the situation.
3914 */
3915 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
3916 resuming = B_FALSE;
3917 } else {
3918 /* check for DDI_RESUME case */
3919 mutex_exit(&ahci_portp->ahciport_mutex);
3920 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3921 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
3922 resuming = B_FALSE;
3923 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3924 mutex_enter(&ahci_portp->ahciport_mutex);
3925 }
3926
3927 if (resuming) {
3928 /*
3929 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
3930 * and PxFBU registers in case these registers were cleared
3931 * during the suspend.
3932 */
3933 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
3934 "ahci_initialize_port: port %d "
3935 "set PxCLB, PxCLBU, PxFB and PxFBU "
3936 "during resume", port);
3937
3938 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
3939 AHCI_SUCCESS)
3940 return (AHCI_FAILURE);
3941 }
3942
3943 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3944 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3945
3946 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3947 "ahci_initialize_port: port %d ", port);
3948
3949 /*
3950 * Check whether the port is in NotRunning state, if not,
3951 * put the port in NotRunning state
3952 */
3953 if (port_cmd_status &
3954 (AHCI_CMD_STATUS_ST |
3955 AHCI_CMD_STATUS_CR |
3956 AHCI_CMD_STATUS_FRE |
3957 AHCI_CMD_STATUS_FR)) {
3958 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3959 ahci_portp, port);
3960 }
3961
3962 /* Make sure the drive is spun-up */
3963 ahci_staggered_spin_up(ahci_ctlp, port);
3964
3965 /* Disable interrupt */
3966 ahci_disable_port_intrs(ahci_ctlp, port);
3967
3968 /* Device is unknown at first */
3969 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
3970
3971 /* Disable the interface power management */
3972 ahci_disable_interface_pm(ahci_ctlp, port);
3973
3974 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3975 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
3976 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3977 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
3978
3979 /* Check physcial link status */
3980 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
3981 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
3982
3983 /* Check interface status */
3984 port_task_file & AHCI_TFD_STS_BSY ||
3985 port_task_file & AHCI_TFD_STS_DRQ ||
3986
3987 /* Check whether port reset must be executed */
3988 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
3989
3990 /* Always reset port on RESUME */
3991 resuming != B_FALSE) {
3992
3993 /* Something went wrong, we need do some reset things */
3994 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
3995
3996 /* Does port reset succeed on HBA port? */
3997 if (ret != AHCI_SUCCESS) {
3998 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
3999 "ahci_initialize_port:"
4000 "port reset failed at port %d", port);
4001 return (AHCI_FAILURE);
4002 }
4003
4004 /* Is port failed? */
4005 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4006 SATA_PSTATE_FAILED) {
4007 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4008 "ahci_initialize_port: port %d state 0x%x",
4009 port, ahci_portp->ahciport_port_state);
4010 return (AHCI_FAILURE);
4011 }
4012 }
4013
4014 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4015 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4016
4017 /*
4018 * Try to get the device signature if the port is not empty.
4019 */
4020 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4021 SATA_DTYPE_NONE)
4022 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4023
4024 /* Return directly if no device connected */
4025 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4026 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4027 "No device connected to port %d", port);
4028 goto out;
4029 }
4030
4031 /* If this is a port multiplier, we need do some initialization */
4032 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4033 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4034 "Port multiplier found at port %d", port);
4035 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4036 }
4037
4038 /* Try to start the port */
4039 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4040 != AHCI_SUCCESS) {
4041 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4042 "failed to start port %d", port);
4043 return (AHCI_FAILURE);
4044 }
4045 out:
4046 /* Enable port interrupts */
4047 ahci_enable_port_intrs(ahci_ctlp, port);
4048
4049 return (AHCI_SUCCESS);
4050 }
4051
4052 /*
4053 * Handle hardware defect, and check the capabilities. For example,
4054 * power management capabilty and MSI capability.
4055 */
4056 static int
4057 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4058 {
4059 ushort_t caps_ptr, cap_count, cap;
4060 #if AHCI_DEBUG
4061 ushort_t pmcap, pmcsr;
4062 ushort_t msimc;
4063 #endif
4064 uint8_t revision;
4065
4066 ahci_ctlp->ahcictl_venid =
4067 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4068 PCI_CONF_VENID);
4069
4070 ahci_ctlp->ahcictl_devid =
4071 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4072 PCI_CONF_DEVID);
4073
4074 /*
4075 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4076 * controllers with 0x00 revision id work on 4-byte aligned buffer,
4077 * which is a bug and was fixed after 0x00 revision id controllers.
4078 *
4079 * Moreover, VT8251 cannot use multiple command slots in the command
4080 * list for non-queued commands because the previous register content
4081 * of PxCI can be re-written in the register write, so a flag will be
4082 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4083 *
4084 * For VT8251, software reset also has the same defect as the below
4085 * AMD/ATI chipset. That is, software reset will get failed if 0xf
4086 * is filled in pmport field. Therefore, another software reset need
4087 * to be done with 0 filled in pmport field.
4088 */
4089 if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4090 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4091 PCI_CONF_REVID);
4092 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4093 "revision id = 0x%x", revision);
4094 if (revision == 0x00) {
4095 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4096 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4097 "change ddi_attr_align to 0x4", NULL);
4098 }
4099
4100 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4101 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4102 "VT8251 cannot use multiple command lists for "
4103 "non-queued commands", NULL);
4104
4105 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4106 }
4107
4108 /*
4109 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4110 * DMA addressing for communication memory descriptors though S64A bit
4111 * of CAP register declares it supports. Even though 64-bit DMA for
4112 * data buffer works on ASUS M2A-VM with newer BIOS, three other
4113 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4114 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4115 *
4116 * Due to certain hardware issue, the chipset must do port reset during
4117 * initialization, otherwise, when retrieving device signature,
4118 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4119 * need to set.
4120 *
4121 * For this chipset software reset will get failure if the pmport of
4122 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4123 * multiplier is connected to the port. In order to fix the issue,
4124 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4125 * reset got failure, the driver will try to do another software reset
4126 * with pmport 0.
4127 */
4128 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4129 ahci_ctlp->ahcictl_devid == 0x4380) {
4130 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4131 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4132 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4133 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4134
4135 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4136 "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4137 "communication memory descriptors though CAP indicates "
4138 "support, so force it to use 32-bit DMA", NULL);
4139 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4140 "ATI SB600 need to do a port reset during initialization",
4141 NULL);
4142 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4143 "ATI SB600 will get software reset failure if pmport "
4144 "is set 0xf and no port multiplier is attached", NULL);
4145 }
4146
4147 /*
4148 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4149 * vendor ID and device ID (0x1002,0x4391).
4150 *
4151 * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4152 * DMA addressing for communication memory descriptors though S64A bit
4153 * of CAP register declares the support. However, it does support
4154 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4155 * set for this controller.
4156 *
4157 * SB710 has the same initialization issue as SB600, so it also need
4158 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4159 *
4160 * SB700 also has the same issue about software reset, and thus
4161 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4162 */
4163 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4164 ahci_ctlp->ahcictl_devid == 0x4391) {
4165 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4166 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4167 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4168
4169 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4170 "ATI SB700/750 cannot do 64-bit DMA for communication "
4171 "memory descriptors though CAP indicates support, "
4172 "so force it to use 32-bit DMA", NULL);
4173 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4174 "ATI SB710 need to do a port reset during initialization",
4175 NULL);
4176 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4177 "ATI SB700 will get software reset failure if pmport "
4178 "is set 0xf and no port multiplier is attached", NULL);
4179 }
4180
4181 /*
4182 * Check if capabilities list is supported and if so,
4183 * get initial capabilities pointer and clear bits 0,1.
4184 */
4185 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4186 PCI_CONF_STAT) & PCI_STAT_CAP) {
4187 caps_ptr = P2ALIGN(pci_config_get8(
4188 ahci_ctlp->ahcictl_pci_conf_handle,
4189 PCI_CONF_CAP_PTR), 4);
4190 } else {
4191 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4192 }
4193
4194 /*
4195 * Walk capabilities if supported.
4196 */
4197 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4198
4199 /*
4200 * Check that we haven't exceeded the maximum number of
4201 * capabilities and that the pointer is in a valid range.
4202 */
4203 if (++cap_count > PCI_CAP_MAX_PTR) {
4204 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4205 "too many device capabilities", NULL);
4206 return (AHCI_FAILURE);
4207 }
4208 if (caps_ptr < PCI_CAP_PTR_OFF) {
4209 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4210 "capabilities pointer 0x%x out of range",
4211 caps_ptr);
4212 return (AHCI_FAILURE);
4213 }
4214
4215 /*
4216 * Get next capability and check that it is valid.
4217 * For now, we only support power management.
4218 */
4219 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4220 caps_ptr);
4221 switch (cap) {
4222 case PCI_CAP_ID_PM:
4223
4224 /* power management supported */
4225 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4226
4227 /* Save PMCSR offset */
4228 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4229
4230 #if AHCI_DEBUG
4231 pmcap = pci_config_get16(
4232 ahci_ctlp->ahcictl_pci_conf_handle,
4233 caps_ptr + PCI_PMCAP);
4234 pmcsr = pci_config_get16(
4235 ahci_ctlp->ahcictl_pci_conf_handle,
4236 ahci_ctlp->ahcictl_pmcsr_offset);
4237 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4238 "Power Management capability found PCI_PMCAP "
4239 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4240 if ((pmcap & 0x3) == 0x3)
4241 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4242 "PCI Power Management Interface "
4243 "spec 1.2 compliant", NULL);
4244 #endif
4245 break;
4246
4247 case PCI_CAP_ID_MSI:
4248 #if AHCI_DEBUG
4249 msimc = pci_config_get16(
4250 ahci_ctlp->ahcictl_pci_conf_handle,
4251 caps_ptr + PCI_MSI_CTRL);
4252 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4253 "Message Signaled Interrupt capability found "
4254 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4255 #endif
4256 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4257 "MSI capability found", NULL);
4258 break;
4259
4260 case PCI_CAP_ID_PCIX:
4261 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4262 "PCI-X capability found", NULL);
4263 break;
4264
4265 case PCI_CAP_ID_PCI_E:
4266 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4267 "PCI Express capability found", NULL);
4268 break;
4269
4270 case PCI_CAP_ID_MSI_X:
4271 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4272 "MSI-X capability found", NULL);
4273 break;
4274
4275 case PCI_CAP_ID_SATA:
4276 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4277 "SATA capability found", NULL);
4278 break;
4279
4280 case PCI_CAP_ID_VS:
4281 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4282 "Vendor Specific capability found", NULL);
4283 break;
4284
4285 default:
4286 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4287 "unrecognized capability 0x%x", cap);
4288 break;
4289 }
4290
4291 /*
4292 * Get next capabilities pointer and clear bits 0,1.
4293 */
4294 caps_ptr = P2ALIGN(pci_config_get8(
4295 ahci_ctlp->ahcictl_pci_conf_handle,
4296 (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4297 }
4298
4299 return (AHCI_SUCCESS);
4300 }
4301
4302 /*
4303 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4304 * PORTMULT command. SYNC & POLLING mode is used.
4305 */
4306 static int
4307 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4308 uint8_t regn, uint32_t *pregv, uint8_t type)
4309 {
4310 ahci_port_t *ahci_portp;
4311 ahci_addr_t pmult_addr;
4312 sata_pkt_t *spkt;
4313 sata_cmd_t *scmd;
4314 sata_device_t sata_device;
4315 uint8_t port = addrp->aa_port;
4316 uint8_t pmport = addrp->aa_pmport;
4317 uint8_t cport;
4318 uint32_t intr_mask;
4319 int rval;
4320 char portstr[10];
4321
4322 SET_PORTSTR(portstr, addrp);
4323 cport = ahci_ctlp->ahcictl_port_to_cport[port];
4324 ahci_portp = ahci_ctlp->ahcictl_ports[port];
4325
4326 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4327 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4328
4329 /* Check the existence of the port multiplier */
4330 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4331 return (AHCI_FAILURE);
4332
4333 /* Request a READ/WRITE PORTMULT sata packet. */
4334 bzero(&sata_device, sizeof (sata_device_t));
4335 sata_device.satadev_addr.cport = cport;
4336 sata_device.satadev_addr.pmport = pmport;
4337 sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4338 sata_device.satadev_rev = SATA_DEVICE_REV;
4339
4340 /*
4341 * Make sure no command is outstanding here. All R/W PMULT requests
4342 * come from
4343 *
4344 * 1. ahci_attach()
4345 * The port should be empty.
4346 *
4347 * 2. ahci_tran_probe_port()
4348 * Any request from SATA framework (via ahci_tran_start) should be
4349 * rejected if R/W PMULT command is outstanding.
4350 *
4351 * If we are doing mopping, do not check those flags because no
4352 * command will be actually outstanding.
4353 *
4354 * If the port has been occupied by any other commands, the probe
4355 * function will return a SATA_RETRY. SATA framework will retry
4356 * later.
4357 */
4358 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4359 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4360 "R/W PMULT failed: R/W PMULT in progress at port %d.",
4361 port, ahci_portp->ahciport_flags);
4362 return (AHCI_FAILURE);
4363 }
4364
4365 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4366 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4367 NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4368 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4369 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4370 "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4371 port, ahci_portp->ahciport_flags);
4372 return (AHCI_FAILURE);
4373 }
4374
4375 /*
4376 * The port multiplier is gone. This may happen when
4377 * 1. Cutting off the power of an enclosure. The device lose the power
4378 * before port multiplier.
4379 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4380 *
4381 * The issued command should be aborted and the following command
4382 * should not be continued.
4383 */
4384 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4385 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4386 "READ/WRITE PMULT failed: "
4387 "port-mult is removed from port %d", port);
4388 return (AHCI_FAILURE);
4389 }
4390
4391 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4392
4393 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4394 &sata_device, regn, *pregv, type);
4395
4396 /*
4397 * READ/WRITE PORTMULT command is intended to sent to the control port
4398 * of the port multiplier.
4399 */
4400 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4401
4402 ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4403
4404 /* No interrupt here. Store the interrupt enable mask. */
4405 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4406 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4407 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4408 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4409
4410 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4411
4412 if (rval == AHCI_SUCCESS &&
4413 spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4414 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4415 scmd = &spkt->satapkt_cmd;
4416 *pregv = scmd->satacmd_lba_high_lsb << 24 |
4417 scmd->satacmd_lba_mid_lsb << 16 |
4418 scmd->satacmd_lba_low_lsb << 8 |
4419 scmd->satacmd_sec_count_lsb;
4420 }
4421 } else {
4422 /* Failed or not completed. */
4423 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4424 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4425 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4426 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4427 rval = AHCI_FAILURE;
4428 }
4429 out:
4430 /* Restore the interrupt mask */
4431 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4432 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4433
4434 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4435 ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4436 sata_free_rdwr_pmult_pkt(spkt);
4437 return (rval);
4438 }
4439
4440 static int
4441 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4442 uint8_t regn, uint32_t *pregv)
4443 {
4444 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4445 SATA_RDWR_PMULT_PKT_TYPE_READ);
4446 }
4447
4448 static int
4449 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4450 uint8_t regn, uint32_t regv)
4451 {
4452 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v,
4453 SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4454 }
4455
4456 #define READ_PMULT(addrp, r, pv, out) \
4457 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \
4458 goto out;
4459
4460 #define WRITE_PMULT(addrp, r, v, out) \
4461 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \
4462 goto out;
4463
4464 /*
4465 * Update sata registers on port multiplier, including GSCR/PSCR registers.
4466 * ahci_update_pmult_gscr()
4467 * ahci_update_pmult_pscr()
4468 */
4469 static int
4470 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4471 sata_pmult_gscr_t *sg)
4472 {
4473 ASSERT(MUTEX_HELD(
4474 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4475
4476 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4477 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4478 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4479 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4480
4481 return (AHCI_SUCCESS);
4482
4483 err: /* R/W PMULT error */
4484 return (AHCI_FAILURE);
4485 }
4486
4487 static int
4488 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4489 sata_device_t *sd)
4490 {
4491 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4492 ASSERT(MUTEX_HELD(
4493 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4494
4495 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4496 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4497 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4498 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4499
4500 return (AHCI_SUCCESS);
4501
4502 err: /* R/W PMULT error */
4503 return (AHCI_FAILURE);
4504 }
4505
4506 /*
4507 * ahci_initialize_pmult()
4508 *
4509 * Initialize a port multiplier, including
4510 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4511 * 2. Redefine MASK register. (SATA Chap 16.?)
4512 */
4513 static int
4514 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4515 ahci_addr_t *addrp, sata_device_t *sd)
4516 {
4517 sata_pmult_gscr_t sg;
4518 uint32_t gscr64;
4519 uint8_t port = addrp->aa_port;
4520
4521 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4522
4523 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4524 "[Initialize] Port-multiplier at port %d.", port);
4525
4526 /*
4527 * Enable features of port multiplier. Currently only
4528 * Asynchronous Notification is enabled.
4529 */
4530 /* Check gscr64 for supported features. */
4531 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4532
4533 if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4534 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4535 "port %d: Port Multiplier supports "
4536 "Asynchronous Notification.", port);
4537
4538 /* Write to gscr96 to enabled features */
4539 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4540 SATA_PMULT_CAP_SNOTIF, err);
4541
4542 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4543 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4544 AHCI_SNOTIF_CLEAR_ALL);
4545 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4546 "port %d: PMult PxSNTF cleared.", port);
4547
4548 }
4549
4550 /*
4551 * Now we need to update gscr33 register to enable hot-plug interrupt
4552 * for sub devices behind port multiplier.
4553 */
4554 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4555 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4556 "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4557
4558 /*
4559 * Fetch the number of device ports of the port multiplier
4560 */
4561 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4562 return (AHCI_FAILURE);
4563
4564 /* Register the port multiplier to SATA Framework. */
4565 mutex_exit(&ahci_portp->ahciport_mutex);
4566 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4567 mutex_enter(&ahci_portp->ahciport_mutex);
4568
4569 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4570 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4571
4572 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4573 "port %d: pmult sub-port number updated to %x.", port,
4574 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4575
4576 /* Till now port-mult is successfully initialized */
4577 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4578 return (AHCI_SUCCESS);
4579
4580 err: /* R/W PMULT error */
4581 return (AHCI_FAILURE);
4582 }
4583
4584 /*
4585 * Initialize a port multiplier port. According to spec, firstly we need
4586 * issue a COMRESET, then a software reset to get its signature.
4587 *
4588 * NOTE: This function should only be called in ahci_probe_pmport()
4589 */
4590 static int
4591 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4592 ahci_addr_t *addrp)
4593 {
4594 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4595 uint8_t port = addrp->aa_port;
4596 uint8_t pmport = addrp->aa_pmport;
4597 int ret = AHCI_FAILURE;
4598
4599 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4600 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4601
4602 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4603 "ahci_initialize_pmport: port %d:%d", port, pmport);
4604
4605 /* Check HBA port state */
4606 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4607 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4608 "ahci_initialize_pmport:"
4609 "port %d:%d Port Multiplier is failed.",
4610 port, pmport);
4611 return (AHCI_FAILURE);
4612 }
4613
4614 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4615 return (AHCI_FAILURE);
4616 }
4617 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4618
4619 /* Checking for outstanding commands */
4620 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4621 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4622 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4623 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4624 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4625 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4626 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4627 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4628 }
4629
4630 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4631 ahci_portp->ahciport_mop_in_progress++;
4632
4633 /* Clear status */
4634 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4635
4636 /* Firstly assume an unknown device */
4637 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4638
4639 ahci_disable_port_intrs(ahci_ctlp, port);
4640
4641 /* port reset is necessary for port multiplier port */
4642 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4643 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4644 "ahci_initialize_pmport:"
4645 "port reset failed at port %d:%d",
4646 port, pmport);
4647 goto out;
4648 }
4649
4650 /* Is port failed? */
4651 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4652 SATA_PSTATE_FAILED) {
4653 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4654 "ahci_initialize_pmport: port %d:%d failed. "
4655 "state = 0x%x", port, pmport,
4656 ahci_portp->ahciport_port_state);
4657 goto out;
4658 }
4659
4660 /* Is there any device attached? */
4661 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4662 == SATA_DTYPE_NONE) {
4663 /* Do not waste time on an empty port */
4664 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4665 "ahci_initialize_pmport: No device is found "
4666 "at port %d:%d", port, pmport);
4667 ret = AHCI_SUCCESS;
4668 goto out;
4669 }
4670
4671 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4672 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4673 "port %d:%d is ready now.", port, pmport);
4674
4675 /*
4676 * Till now we can assure a device attached to that HBA port and work
4677 * correctly. Now try to get the device signature. This is an optional
4678 * step. If failed, unknown device is assumed, then SATA module will
4679 * continue to use IDENTIFY DEVICE to get the information of the
4680 * device.
4681 */
4682 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4683
4684 ret = AHCI_SUCCESS;
4685
4686 out:
4687 /* Next try to mop the pending commands */
4688 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4689 finished_tags = ahci_portp->ahciport_pending_tags &
4690 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4691 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4692 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4693 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4694 reset_tags &= ~finished_tags;
4695
4696 ahci_mop_commands(ahci_ctlp,
4697 ahci_portp,
4698 slot_status,
4699 0, /* failed tags */
4700 0, /* timeout tags */
4701 0, /* aborted tags */
4702 reset_tags); /* reset tags */
4703
4704 /* Clear PxSNTF register if supported. */
4705 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4706 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4707 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4708 AHCI_SNOTIF_CLEAR_ALL);
4709 }
4710
4711 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4712 ahci_enable_port_intrs(ahci_ctlp, port);
4713 return (ret);
4714 }
4715
4716 /*
4717 * ahci_probe_pmult()
4718 *
4719 * This function will be called to probe a port multiplier, which will
4720 * handle hotplug events on port multiplier ports.
4721 *
4722 * NOTE: Only called from ahci_tran_probe_port()
4723 */
4724 static int
4725 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4726 ahci_addr_t *addrp)
4727 {
4728 sata_device_t sdevice;
4729 ahci_addr_t pmport_addr;
4730 uint32_t gscr32, port_hotplug_tags;
4731 uint32_t pmport_sstatus;
4732 int dev_exists_now = 0, dev_existed_previously = 0;
4733 uint8_t port = addrp->aa_port;
4734 int npmport;
4735
4736 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4737
4738 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4739 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4740 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4741
4742 do {
4743 npmport = ddi_ffs(port_hotplug_tags) - 1;
4744 if (npmport == -1)
4745 /* no pending hot plug events. */
4746 return (AHCI_SUCCESS);
4747
4748 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4749 "hot-plug event at port %d:%d", port, npmport);
4750
4751 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4752
4753 /* Check previous device at that port */
4754 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4755 != SATA_DTYPE_NONE)
4756 dev_existed_previously = 1;
4757
4758 /* PxSStatus tells the presence of device. */
4759 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4760 &pmport_sstatus, err);
4761
4762 if (SSTATUS_GET_DET(pmport_sstatus) ==
4763 SSTATUS_DET_DEVPRE_PHYCOM)
4764 dev_exists_now = 1;
4765
4766 /*
4767 * Clear PxSERR is critical. The transition from 0 to 1 will
4768 * emit a FIS which generates an asynchronous notification
4769 * event at controller. If we fail to clear the PxSERR, the
4770 * Async Notif events will no longer be activated on this
4771 * pmport.
4772 */
4773 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4774 AHCI_SERROR_CLEAR_ALL, err);
4775
4776 bzero((void *)&sdevice, sizeof (sata_device_t));
4777 sdevice.satadev_addr.cport = ahci_ctlp->
4778 ahcictl_port_to_cport[port];
4779 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4780 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4781 sdevice.satadev_state = SATA_PSTATE_PWRON;
4782
4783 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4784 "[Existence] %d -> %d", dev_existed_previously,
4785 dev_exists_now);
4786
4787 if (dev_exists_now) {
4788 if (dev_existed_previously) {
4789 /* Link (may) not change: Exist -> Exist * */
4790 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4791 "ahci_probe_pmult: port %d:%d "
4792 "device link lost/established",
4793 port, npmport);
4794
4795 mutex_exit(&ahci_portp->ahciport_mutex);
4796 sata_hba_event_notify(
4797 ahci_ctlp->ahcictl_sata_hba_tran->
4798 sata_tran_hba_dip,
4799 &sdevice,
4800 SATA_EVNT_LINK_LOST|
4801 SATA_EVNT_LINK_ESTABLISHED);
4802 mutex_enter(&ahci_portp->ahciport_mutex);
4803 } else {
4804 /* Link change: None -> Exist */
4805 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4806 "ahci_probe_pmult: port %d:%d "
4807 "device link established", port, npmport);
4808
4809 /* Clear port state */
4810 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4811 SATA_STATE_UNKNOWN);
4812 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4813 "ahci_probe_pmult: port %d "
4814 "ahciport_port_state [Cleared].", port);
4815
4816 mutex_exit(&ahci_portp->ahciport_mutex);
4817 sata_hba_event_notify(
4818 ahci_ctlp->ahcictl_sata_hba_tran->
4819 sata_tran_hba_dip,
4820 &sdevice,
4821 SATA_EVNT_LINK_ESTABLISHED);
4822 mutex_enter(&ahci_portp->ahciport_mutex);
4823 }
4824 } else { /* No device exists now */
4825 if (dev_existed_previously) {
4826
4827 /* Link change: Exist -> None */
4828 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4829 "ahci_probe_pmult: port %d:%d "
4830 "device link lost", port, npmport);
4831
4832 /* An existing device is lost. */
4833 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4834 SATA_STATE_UNKNOWN);
4835 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4836 SATA_DTYPE_NONE);
4837
4838 mutex_exit(&ahci_portp->ahciport_mutex);
4839 sata_hba_event_notify(
4840 ahci_ctlp->ahcictl_sata_hba_tran->
4841 sata_tran_hba_dip,
4842 &sdevice,
4843 SATA_EVNT_LINK_LOST);
4844 mutex_enter(&ahci_portp->ahciport_mutex);
4845 }
4846 }
4847
4848 CLEAR_BIT(port_hotplug_tags, npmport);
4849 } while (port_hotplug_tags != 0);
4850
4851 return (AHCI_SUCCESS);
4852
4853 err: /* R/W PMULT error */
4854 return (AHCI_FAILURE);
4855 }
4856
4857 /*
4858 * Probe and initialize a port multiplier port.
4859 * A port multiplier port could only be initilaizer here.
4860 */
4861 static int
4862 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4863 ahci_addr_t *addrp, sata_device_t *sd)
4864 {
4865 uint32_t port_state;
4866 uint8_t port = addrp->aa_port;
4867 ahci_addr_t addr_pmult;
4868
4869 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4870
4871 /*
4872 * Check the parent - port multiplier first.
4873 */
4874
4875 /*
4876 * Parent port multiplier might have been removed. This event will be
4877 * ignored and failure.
4878 */
4879 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4880 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4881 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4882 "ahci_tran_probe_port: "
4883 "parent device removed, ignore event.", NULL);
4884
4885 return (AHCI_FAILURE);
4886 }
4887
4888 /* The port is ready? */
4889 port_state = ahci_portp->ahciport_port_state;
4890 if (!(port_state & SATA_STATE_READY)) {
4891 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4892 "ahci_tran_probe_port: "
4893 "parent port-mult is NOT ready.", NULL);
4894
4895 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4896 ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4897 AHCI_SUCCESS) {
4898 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4899 "ahci_tran_probe_port: "
4900 "restart port-mult failed.", NULL);
4901 return (AHCI_FAILURE);
4902 }
4903 }
4904
4905 /*
4906 * If port-mult is restarted due to some reason, we need
4907 * re-initialized the PMult.
4908 */
4909 if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
4910 /* Initialize registers on a port multiplier */
4911 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
4912 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
4913 &addr_pmult, sd) != AHCI_SUCCESS)
4914 return (AHCI_FAILURE);
4915 }
4916
4917 /*
4918 * Then we check the port-mult port
4919 */
4920 /* Is this pmport initialized? */
4921 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
4922 if (!(port_state & SATA_STATE_READY)) {
4923
4924 /* ahci_initialize_pmport() will set READY state */
4925 if (ahci_initialize_pmport(ahci_ctlp,
4926 ahci_portp, addrp) != AHCI_SUCCESS)
4927 return (AHCI_FAILURE);
4928 }
4929
4930 return (AHCI_SUCCESS);
4931 }
4932
4933 /*
4934 * AHCI device reset ...; a single device on one of the ports is reset,
4935 * but the HBA and physical communication remain intact. This is the
4936 * least intrusive.
4937 *
4938 * When issuing a software reset sequence, there should not be other
4939 * commands in the command list, so we will first clear and then re-set
4940 * PxCMD.ST to clear PxCI. And before issuing the software reset,
4941 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
4942 * cleared unless command list override (PxCMD.CLO) is supported.
4943 */
4944 static int
4945 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4946 ahci_addr_t *addrp)
4947 {
4948 ahci_fis_h2d_register_t *h2d_register_fisp;
4949 ahci_cmd_table_t *cmd_table;
4950 ahci_cmd_header_t *cmd_header;
4951 uint32_t port_cmd_status, port_cmd_issue, port_task_file;
4952 int slot, loop_count;
4953 uint8_t port = addrp->aa_port;
4954 uint8_t pmport = addrp->aa_pmport;
4955 int rval = AHCI_FAILURE;
4956
4957 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4958
4959 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
4960 "port %d:%d device software resetting (FIS)", port, pmport);
4961
4962 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
4963 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
4964 port) != AHCI_SUCCESS) {
4965 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4966 "ahci_software_reset: cannot stop HBA port %d.", port);
4967 goto out;
4968 }
4969
4970 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
4971 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4972 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4973
4974 if (port_task_file & AHCI_TFD_STS_BSY ||
4975 port_task_file & AHCI_TFD_STS_DRQ) {
4976 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
4977 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4978 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
4979 "cannot issue a software reset.", port_task_file);
4980 goto out;
4981 }
4982
4983 /*
4984 * If HBA Support CLO, as Command List Override (CAP.SCLO is
4985 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
4986 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
4987 */
4988 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4989 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
4990
4991 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4992 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4993 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4994 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
4995 port_cmd_status|AHCI_CMD_STATUS_CLO);
4996
4997 /* Waiting till PxCMD.SCLO bit is cleared */
4998 loop_count = 0;
4999 do {
5000 /* Wait for 10 millisec */
5001 drv_usecwait(AHCI_10MS_USECS);
5002
5003 /* We are effectively timing out after 1 sec. */
5004 if (loop_count++ > 100) {
5005 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5006 "SCLO time out. port %d is busy.", port);
5007 goto out;
5008 }
5009
5010 port_cmd_status =
5011 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5012 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5013 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5014
5015 /* Re-check */
5016 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5017 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5018 if (port_task_file & AHCI_TFD_STS_BSY ||
5019 port_task_file & AHCI_TFD_STS_DRQ) {
5020 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5021 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5022 port_task_file);
5023 goto out;
5024 }
5025 }
5026
5027 /* Then start port */
5028 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5029 != AHCI_SUCCESS) {
5030 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5031 "ahci_software_reset: cannot start AHCI port %d.", port);
5032 goto out;
5033 }
5034
5035 /*
5036 * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5037 * ahci_port.ahciport_pending_ncq_tags may fail
5038 * ahci_claim_free_slot(). Actually according to spec, by clearing
5039 * PxCMD.ST there is no command outstanding while executing software
5040 * reseting. Hence we directly use slot 0 instead of
5041 * ahci_claim_free_slot().
5042 */
5043 slot = 0;
5044
5045 /* Now send the first H2D Register FIS with SRST set to 1 */
5046 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5047 bzero((void *)cmd_table, ahci_cmd_table_size);
5048
5049 h2d_register_fisp =
5050 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5051
5052 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5053 SET_FIS_PMP(h2d_register_fisp, pmport);
5054 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5055
5056 /* Set Command Header in Command List */
5057 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5058 BZERO_DESCR_INFO(cmd_header);
5059 BZERO_PRD_BYTE_COUNT(cmd_header);
5060 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5061 SET_PORT_MULTI_PORT(cmd_header, pmport);
5062
5063 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5064 SET_RESET(cmd_header, 1);
5065 SET_WRITE(cmd_header, 1);
5066
5067 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5068 0,
5069 ahci_cmd_table_size,
5070 DDI_DMA_SYNC_FORDEV);
5071
5072 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5073 slot * sizeof (ahci_cmd_header_t),
5074 sizeof (ahci_cmd_header_t),
5075 DDI_DMA_SYNC_FORDEV);
5076
5077 /* Indicate to the HBA that a command is active. */
5078 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5079 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5080 (0x1 << slot));
5081
5082 loop_count = 0;
5083
5084 /* Loop till the first command is finished */
5085 do {
5086 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5087 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5088
5089 /* We are effectively timing out after 1 sec. */
5090 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5091 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5092 "the first SRST FIS is timed out, "
5093 "loop_count = %d", loop_count);
5094 goto out;
5095 }
5096 /* Wait for 10 millisec */
5097 drv_usecwait(AHCI_10MS_USECS);
5098 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5099
5100 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5101 "ahci_software_reset: 1st loop count: %d, "
5102 "port_cmd_issue = 0x%x, slot = 0x%x",
5103 loop_count, port_cmd_issue, slot);
5104
5105 /* According to ATA spec, we need wait at least 5 microsecs here. */
5106 drv_usecwait(AHCI_1MS_USECS);
5107
5108 /* Now send the second H2D Register FIS with SRST cleard to zero */
5109 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5110 bzero((void *)cmd_table, ahci_cmd_table_size);
5111
5112 h2d_register_fisp =
5113 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5114
5115 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5116 SET_FIS_PMP(h2d_register_fisp, pmport);
5117
5118 /* Set Command Header in Command List */
5119 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5120 BZERO_DESCR_INFO(cmd_header);
5121 BZERO_PRD_BYTE_COUNT(cmd_header);
5122 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5123 SET_PORT_MULTI_PORT(cmd_header, pmport);
5124
5125 SET_WRITE(cmd_header, 1);
5126
5127 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5128 0,
5129 ahci_cmd_table_size,
5130 DDI_DMA_SYNC_FORDEV);
5131
5132 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5133 slot * sizeof (ahci_cmd_header_t),
5134 sizeof (ahci_cmd_header_t),
5135 DDI_DMA_SYNC_FORDEV);
5136
5137 /* Indicate to the HBA that a command is active. */
5138 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5139 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5140 (0x1 << slot));
5141
5142 loop_count = 0;
5143
5144 /* Loop till the second command is finished */
5145 do {
5146 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5147 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5148
5149 /* We are effectively timing out after 1 sec. */
5150 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5151 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5152 "the second SRST FIS is timed out, "
5153 "loop_count = %d", loop_count);
5154 goto out;
5155 }
5156
5157 /* Wait for 10 millisec */
5158 drv_usecwait(AHCI_10MS_USECS);
5159 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5160
5161 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5162 "ahci_software_reset: 2nd loop count: %d, "
5163 "port_cmd_issue = 0x%x, slot = 0x%x",
5164 loop_count, port_cmd_issue, slot);
5165
5166 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5167 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5168 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5169 DDI_SERVICE_UNAFFECTED);
5170 goto out;
5171 }
5172
5173 rval = AHCI_SUCCESS;
5174 out:
5175 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5176 "ahci_software_reset: %s at port %d:%d",
5177 rval == AHCI_SUCCESS ? "succeed" : "failed",
5178 port, pmport);
5179
5180 return (rval);
5181 }
5182
5183 /*
5184 * AHCI port reset ...; the physical communication between the HBA and device
5185 * on a port are disabled. This is more intrusive.
5186 *
5187 * When an HBA or port reset occurs, Phy communication is going to
5188 * be re-established with the device through a COMRESET followed by the
5189 * normal out-of-band communication sequence defined in Serial ATA. At
5190 * the end of reset, the device, if working properly, will send a D2H
5191 * Register FIS, which contains the device signature. When the HBA receives
5192 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5193 * the PxSIG register with the signature.
5194 *
5195 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5196 * function is called. If not, it is assumed the interface is in hung
5197 * condition.
5198 */
5199 static int
5200 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5201 ahci_addr_t *addrp)
5202 {
5203 ahci_addr_t pmult_addr;
5204 uint32_t port_cmd_status;
5205 uint32_t port_scontrol, port_sstatus;
5206 uint32_t port_task_file;
5207 uint32_t port_state;
5208 uint8_t port = addrp->aa_port;
5209
5210 int loop_count;
5211 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5212
5213 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5214
5215 /* Target is a port multiplier port? */
5216 if (AHCI_ADDR_IS_PMPORT(addrp))
5217 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5218
5219 /* Otherwise it must be an HBA port. */
5220 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5221
5222 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5223 "Port %d port resetting...", port);
5224 ahci_portp->ahciport_port_state = 0;
5225
5226 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5227 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5228
5229 /*
5230 * According to the spec, SUD bit should be set here,
5231 * but JMicron JMB363 doesn't follow it, so print
5232 * a debug message.
5233 */
5234 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5235 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5236 "ahci_port_reset: port %d SUD bit not set", port);
5237
5238 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5239 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5240 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5241
5242 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5243 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5244 port_scontrol);
5245
5246 /* Enable PxCMD.FRE to read device */
5247 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5248 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5249 port_cmd_status|AHCI_CMD_STATUS_FRE);
5250
5251 /*
5252 * The port enters P:StartComm state, and the HBA tells the link layer
5253 * to start communication, which involves sending COMRESET to the
5254 * device. And the HBA resets PxTFD.STS to 7Fh.
5255 *
5256 * Give time for COMRESET to percolate, according to the AHCI
5257 * spec, software shall wait at least 1 millisecond before
5258 * clearing PxSCTL.DET
5259 */
5260 drv_usecwait(AHCI_1MS_USECS * 2);
5261
5262 /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5263 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5264 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5265 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5266 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5267 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5268 port_scontrol);
5269
5270 /*
5271 * When a COMINIT is received from the device, then the port enters
5272 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5273 * PxSSTS.DET to 1h to indicate a device is detected but communication
5274 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5275 * a COMINIT has been received.
5276 */
5277 /*
5278 * The DET field is valid only if IPM field indicates
5279 * that the interface is in active state.
5280 */
5281 loop_count = 0;
5282 for (;;) {
5283 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5284 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5285
5286 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5287 /*
5288 * If the interface is not active, the DET field
5289 * is considered not accurate. So we want to
5290 * continue looping.
5291 */
5292 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5293 }
5294
5295 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5296 break;
5297
5298 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5299 /*
5300 * We are effectively timing out after 0.1 sec.
5301 */
5302 break;
5303 }
5304
5305 /* Wait for 10 millisec */
5306 drv_usecwait(AHCI_10MS_USECS);
5307 }
5308
5309 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5310 "ahci_port_reset: 1st loop count: %d, "
5311 "port_sstatus = 0x%x port %d",
5312 loop_count, port_sstatus, port);
5313
5314 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5315 /*
5316 * Either the port is not active or there
5317 * is no device present.
5318 */
5319 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5320 return (AHCI_SUCCESS);
5321 }
5322
5323 /* Clear port serror register for the port */
5324 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5325 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5326 AHCI_SERROR_CLEAR_ALL);
5327
5328 /*
5329 * Devices should return a FIS contains its signature to HBA after
5330 * COMINIT signal. Check whether a D2H Register FIS is received by
5331 * polling PxTFD.STS.
5332 */
5333 loop_count = 0;
5334 for (;;) {
5335 port_task_file =
5336 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5337 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5338
5339 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5340 AHCI_TFD_STS_ERR)) == 0)
5341 break;
5342
5343 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5344 /*
5345 * We are effectively timing out after 11 sec.
5346 */
5347 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5348 "the device hardware has been initialized and "
5349 "the power-up diagnostics failed",
5350 instance, port);
5351
5352 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5353 "port %d: some or all of BSY, DRQ and ERR in "
5354 "PxTFD.STS are not clear. We need another "
5355 "software reset.", port);
5356
5357 /* Clear port serror register for the port */
5358 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5359 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5360 AHCI_SERROR_CLEAR_ALL);
5361
5362 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5363
5364 /* Try another software reset. */
5365 if (ahci_software_reset(ahci_ctlp, ahci_portp,
5366 &pmult_addr) != AHCI_SUCCESS) {
5367 AHCIPORT_SET_STATE(ahci_portp, addrp,
5368 SATA_PSTATE_FAILED);
5369 return (AHCI_FAILURE);
5370 }
5371 break;
5372 }
5373
5374 /* Wait for 10 millisec */
5375 drv_usecwait(AHCI_10MS_USECS);
5376 }
5377
5378 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5379 "ahci_port_reset: 2nd loop count: %d, "
5380 "port_task_file = 0x%x port %d",
5381 loop_count, port_task_file, port);
5382
5383 /* Clear port serror register for the port */
5384 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5385 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5386 AHCI_SERROR_CLEAR_ALL);
5387
5388 /* Set port as ready */
5389 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5390 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5391
5392 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5393 "ahci_port_reset: succeed at port %d.", port);
5394 return (AHCI_SUCCESS);
5395 }
5396
5397 /*
5398 * COMRESET on a port multiplier port.
5399 *
5400 * NOTE: Only called in ahci_port_reset()
5401 */
5402 static int
5403 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5404 ahci_addr_t *addrp)
5405 {
5406 uint32_t port_scontrol, port_sstatus, port_serror;
5407 uint32_t port_cmd_status, port_intr_status;
5408 uint32_t port_state;
5409 uint8_t port = addrp->aa_port;
5410 uint8_t pmport = addrp->aa_pmport;
5411 int loop_count;
5412 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5413
5414 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5415 "port %d:%d: pmport resetting", port, pmport);
5416
5417 /* Initialize pmport state */
5418 AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5419
5420 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5421 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5422 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5423
5424 /* PxCMD.FRE should be set before. */
5425 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5426 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5427 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5428 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5429 return (AHCI_FAILURE);
5430
5431 /*
5432 * Give time for COMRESET to percolate, according to the AHCI
5433 * spec, software shall wait at least 1 millisecond before
5434 * clearing PxSCTL.DET
5435 */
5436 drv_usecwait(AHCI_1MS_USECS*2);
5437
5438 /*
5439 * Fetch the SCONTROL again and rewrite the DET part with 0
5440 * This will generate an Asychronous Notification events.
5441 */
5442 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5443 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5444 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5445
5446 /*
5447 * The port enters P:StartComm state, and HBA tells link layer to
5448 * start communication, which involves sending COMRESET to device.
5449 * And the HBA resets PxTFD.STS to 7Fh.
5450 *
5451 * When a COMINIT is received from the device, then the port enters
5452 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5453 * PxSSTS.DET to 1h to indicate a device is detected but communication
5454 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5455 * a COMINIT has been received.
5456 */
5457 /*
5458 * The DET field is valid only if IPM field indicates
5459 * that the interface is in active state.
5460 */
5461 loop_count = 0;
5462 do {
5463 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5464
5465 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5466 /*
5467 * If the interface is not active, the DET field
5468 * is considered not accurate. So we want to
5469 * continue looping.
5470 */
5471 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5472 }
5473
5474 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5475 /*
5476 * We are effectively timing out after 0.1 sec.
5477 */
5478 break;
5479 }
5480
5481 /* Wait for 10 millisec */
5482 drv_usecwait(AHCI_10MS_USECS);
5483
5484 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5485
5486 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5487 "ahci_pmport_reset: 1st loop count: %d, "
5488 "port_sstatus = 0x%x port %d:%d",
5489 loop_count, port_sstatus, port, pmport);
5490
5491 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5492 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5493 /*
5494 * Either the port is not active or there
5495 * is no device present.
5496 */
5497 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5498 "ahci_pmport_reset: "
5499 "no device attached to port %d:%d",
5500 port, pmport);
5501 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5502 return (AHCI_SUCCESS);
5503 }
5504
5505 /* Now we can make sure there is a device connected to the port */
5506 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5507 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5508
5509 if (!(port_serror & (1 << 26))) {
5510 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5511 "COMINIT signal from the device not received port %d:%d",
5512 instance, port, pmport);
5513
5514 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5515 return (AHCI_FAILURE);
5516 }
5517
5518 /*
5519 * After clear PxSERR register, we will receive a D2H FIS.
5520 * Normally this FIS will cause a IPMS error according to AHCI spec
5521 * v1.2 because there is no command outstanding for it. So we need
5522 * to ignore this error.
5523 */
5524 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5525 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5526
5527 /* Now we need to check the D2H FIS by checking IPMS error. */
5528 loop_count = 0;
5529 do {
5530 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5531 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5532
5533 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5534 /*
5535 * No D2H FIS received. This is possible according
5536 * to SATA 2.6 spec.
5537 */
5538 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5539 "PxIS.IPMS is not set, we need another "
5540 "software reset.", port, pmport);
5541
5542 break;
5543 }
5544
5545 /* Wait for 10 millisec */
5546 mutex_exit(&ahci_portp->ahciport_mutex);
5547 delay(AHCI_10MS_TICKS);
5548 mutex_enter(&ahci_portp->ahciport_mutex);
5549
5550 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5551
5552 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5553 "ahci_pmport_reset: 2st loop count: %d, "
5554 "port_sstatus = 0x%x port %d:%d",
5555 loop_count, port_sstatus, port, pmport);
5556
5557 /* Clear IPMS */
5558 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5559 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5560 AHCI_INTR_STATUS_IPMS);
5561 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5562
5563 /* This pmport is now ready for ahci_tran_start() */
5564 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5565 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5566
5567 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5568 "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5569 return (AHCI_SUCCESS);
5570
5571 err: /* R/W PMULT error */
5572 /* IPMS flags might be set before. */
5573 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5574 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5575 "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5576
5577 return (AHCI_FAILURE);
5578 }
5579
5580 /*
5581 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5582 * This is the most intrusive.
5583 *
5584 * When an HBA reset occurs, Phy communication will be re-established with
5585 * the device through a COMRESET followed by the normal out-of-band
5586 * communication sequence defined in Serial ATA. At the end of reset, the
5587 * device, if working properly, will send a D2H Register FIS, which contains
5588 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5589 * and PxTFD.ERR register fields, and updates the PxSIG register with the
5590 * signature.
5591 *
5592 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5593 */
5594 static int
5595 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5596 {
5597 ahci_port_t *ahci_portp;
5598 uint32_t ghc_control;
5599 uint8_t port;
5600 int loop_count;
5601 int rval = AHCI_SUCCESS;
5602
5603 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5604 NULL);
5605
5606 mutex_enter(&ahci_ctlp->ahcictl_mutex);
5607
5608 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5609 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5610
5611 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5612 ghc_control |= AHCI_HBA_GHC_HR;
5613 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5614 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5615
5616 /*
5617 * Wait until HBA Reset complete or timeout
5618 */
5619 loop_count = 0;
5620 do {
5621 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5622 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5623
5624 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5625 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5626 "ahci hba reset is timing out, "
5627 "ghc_control = 0x%x", ghc_control);
5628 /* We are effectively timing out after 1 sec. */
5629 break;
5630 }
5631
5632 /* Wait for 10 millisec */
5633 drv_usecwait(AHCI_10MS_USECS);
5634 } while (ghc_control & AHCI_HBA_GHC_HR);
5635
5636 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5637 "ahci_hba_reset: 1st loop count: %d, "
5638 "ghc_control = 0x%x", loop_count, ghc_control);
5639
5640 if (ghc_control & AHCI_HBA_GHC_HR) {
5641 /* The hba is not reset for some reasons */
5642 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5643 "hba reset failed: HBA in a hung or locked state", NULL);
5644 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5645 return (AHCI_FAILURE);
5646 }
5647
5648 /*
5649 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5650 */
5651 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5652 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5653 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5654 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5655 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5656
5657 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5658
5659 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5660 /* Only check implemented ports */
5661 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5662 continue;
5663 }
5664
5665 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5666 mutex_enter(&ahci_portp->ahciport_mutex);
5667
5668 /* Make sure the drive is spun-up */
5669 ahci_staggered_spin_up(ahci_ctlp, port);
5670
5671 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5672 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5673 AHCI_SUCCESS) {
5674 rval = AHCI_FAILURE;
5675 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5676 "ahci_hba_reset: port %d failed", port);
5677 /*
5678 * Set the port state to SATA_PSTATE_FAILED if
5679 * failed to initialize it.
5680 */
5681 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5682 }
5683
5684 mutex_exit(&ahci_portp->ahciport_mutex);
5685 }
5686
5687 return (rval);
5688 }
5689
5690 /*
5691 * This routine is only called from AHCI_ATTACH or phyrdy change
5692 * case. It first calls software reset, then stop the port and try to
5693 * read PxSIG register to find the type of device attached to the port.
5694 *
5695 * The caller should make sure a valid device exists on specified port and
5696 * physical communication has been established so that the signature could
5697 * be retrieved by software reset.
5698 *
5699 * NOTE: The port interrupts should be disabled before the function is called.
5700 */
5701 static void
5702 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5703 ahci_addr_t *addrp)
5704 {
5705 ahci_addr_t dev_addr;
5706 uint32_t signature;
5707 uint8_t port = addrp->aa_port;
5708 uint8_t pmport = addrp->aa_pmport;
5709 int rval;
5710
5711 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5712 ASSERT(AHCI_ADDR_IS_VALID(addrp));
5713
5714 /*
5715 * If the HBA doesn't support port multiplier, then the driver
5716 * doesn't need to bother to check port multiplier device.
5717 *
5718 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5719 * connected to Silicon Image 4723, to which the two sata drives
5720 * attached can be set with RAID1, RAID0 or Spanning mode.
5721 *
5722 * We found software reset will get failure if port multiplier address
5723 * 0xf is used by software reset, so just ignore the check since
5724 * ICH7 doesn't support port multiplier device at all.
5725 */
5726 if (AHCI_ADDR_IS_PORT(addrp) &&
5727 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5728 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5729 "ahci_find_dev_signature enter: port %d", port);
5730
5731 /*
5732 * NOTE: when the ahci address is a HBA port, we do not know
5733 * it is a device or a port multiplier that attached. we need
5734 * try a software reset at port multiplier address (0xf
5735 * pmport)
5736 */
5737 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5738 } else {
5739 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5740 "ahci_find_dev_signature enter: port %d:%d",
5741 port, pmport);
5742 dev_addr = *addrp;
5743 }
5744
5745 /* Assume it is unknown. */
5746 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5747
5748 /* Issue a software reset to get the signature */
5749 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5750 if (rval != AHCI_SUCCESS) {
5751
5752 /*
5753 * Try to do software reset again with pmport set with 0 if
5754 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5755 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5756 */
5757 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5758 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5759 dev_addr.aa_pmport = 0;
5760 rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5761 &dev_addr);
5762 }
5763
5764 if (rval != AHCI_SUCCESS) {
5765 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5766 "ahci_find_dev_signature: software reset failed "
5767 "at port %d:%d, cannot get signature.",
5768 port, pmport);
5769
5770 AHCIPORT_SET_STATE(ahci_portp, addrp,
5771 SATA_PSTATE_FAILED);
5772 return;
5773 }
5774 }
5775
5776 /*
5777 * ahci_software_reset has started the port, so we need manually stop
5778 * the port again.
5779 */
5780 if (AHCI_ADDR_IS_PORT(addrp)) {
5781 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5782 ahci_portp, port) != AHCI_SUCCESS) {
5783 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5784 "ahci_find_dev_signature: cannot stop port %d.",
5785 port);
5786 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5787 return;
5788 }
5789 }
5790
5791 /* Now we can make sure that a valid signature is received. */
5792 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5793 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5794
5795 if (AHCI_ADDR_IS_PMPORT(addrp)) {
5796 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5797 "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5798 signature, port, pmport);
5799 } else {
5800 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5801 "ahci_find_dev_signature: signature = 0x%x at port %d",
5802 signature, port);
5803 }
5804
5805 /* NOTE: Only support ATAPI device at controller port. */
5806 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5807 signature = SATA_DTYPE_UNKNOWN;
5808
5809 switch (signature) {
5810
5811 case AHCI_SIGNATURE_DISK:
5812 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5813 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5814 "Disk is found at port: %d", port);
5815 break;
5816
5817 case AHCI_SIGNATURE_ATAPI:
5818 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5819 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5820 "ATAPI device is found at port: %d", port);
5821 break;
5822
5823 case AHCI_SIGNATURE_PORT_MULTIPLIER:
5824 /* Port Multiplier cannot recursively attached. */
5825 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5826 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5827 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5828 "Port Multiplier is found at port: %d", port);
5829 break;
5830
5831 default:
5832 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5833 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5834 "Unknown device is found at port: %d", port);
5835 }
5836 }
5837
5838 /*
5839 * According to the spec, to reliably detect hot plug removals, software
5840 * must disable interface power management. Software should perform the
5841 * following initialization on a port after a device is attached:
5842 * Set PxSCTL.IPM to 3h to disable interface state transitions
5843 * Set PxCMD.ALPE to '0' to disable aggressive power management
5844 * Disable device initiated interface power management by SET FEATURE
5845 *
5846 * We can ignore the last item because by default the feature is disabled
5847 */
5848 static void
5849 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5850 {
5851 uint32_t port_scontrol, port_cmd_status;
5852
5853 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5854 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5855 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5856 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5857 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5858
5859 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5860 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5861 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5862 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5863 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5864 }
5865
5866 /*
5867 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5868 * to 1, then set it firstly.
5869 *
5870 * Each port contains two major DMA engines. One DMA engine walks through
5871 * the command list, and is controlled by PxCMD.ST. The second DMA engine
5872 * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5873 *
5874 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5875 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5876 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5877 *
5878 * Software shall not set PxCMD.ST to '1' unless a functional device is
5879 * present on the port(as determined by PxTFD.STS.BSY = '0',
5880 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5881 */
5882 static int
5883 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5884 {
5885 uint32_t port_cmd_status;
5886
5887 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5888
5889 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5890
5891 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5892 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5893 "the state for port %d is 0x%x",
5894 port, ahci_portp->ahciport_port_state);
5895 return (AHCI_FAILURE);
5896 }
5897
5898 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5899 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5900 "no device is attached at port %d", port);
5901 return (AHCI_FAILURE);
5902 }
5903
5904 /* First to set PxCMD.FRE before setting PxCMD.ST. */
5905 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5906 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5907
5908 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
5909 port_cmd_status |= AHCI_CMD_STATUS_FRE;
5910 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5911 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5912 port_cmd_status);
5913 }
5914
5915 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5916 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5917
5918 port_cmd_status |= AHCI_CMD_STATUS_ST;
5919
5920 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5921 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5922 port_cmd_status);
5923
5924 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
5925
5926 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
5927 "PxCMD.ST set to '1' at port %d", port);
5928
5929 return (AHCI_SUCCESS);
5930 }
5931
5932 /*
5933 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
5934 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
5935 * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
5936 */
5937 static int
5938 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
5939 {
5940 uint8_t port = ahci_portp->ahciport_port_num;
5941 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5942 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5943
5944 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5945
5946 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
5947 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
5948 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5949 port) != AHCI_SUCCESS)
5950 return (AHCI_FAILURE);
5951
5952 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5953 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5954 }
5955
5956 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
5957 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
5958 int loop_count = 0;
5959
5960 /* Clear PxCMD.FRE */
5961 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
5962 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5963 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5964 port_cmd_status);
5965
5966 /* Wait until PxCMD.FR is cleared */
5967 for (;;) {
5968 port_cmd_status =
5969 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5970 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5971
5972 if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
5973 break;
5974
5975 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
5976 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
5977 "ahci_setup_port_base_addresses: cannot "
5978 "clear PxCMD.FR for port %d.", port);
5979
5980 /*
5981 * We are effectively timing out after 0.5 sec.
5982 * This value is specified in AHCI spec.
5983 */
5984 return (AHCI_FAILURE);
5985 }
5986
5987 /* Wait for 1 millisec */
5988 drv_usecwait(AHCI_1MS_USECS);
5989 }
5990 }
5991
5992 /* Step 3: Config Port Command List Base Address */
5993 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5994 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
5995 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
5996
5997 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5998 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
5999 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
6000
6001 /* Step 4: Config Port Received FIS Base Address */
6002 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6003 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
6004 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6005
6006 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6007 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6008 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6009
6010 return (AHCI_SUCCESS);
6011 }
6012
6013 /*
6014 * Allocate the ahci_port_t including Received FIS and Command List.
6015 * The argument - port is the physical port number, and not logical
6016 * port number seen by the SATA framework.
6017 */
6018 static int
6019 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6020 {
6021 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6022 ahci_port_t *ahci_portp;
6023 char taskq_name[64] = "event_handle_taskq";
6024
6025 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6026
6027 ahci_portp =
6028 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6029
6030 ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6031 ahci_portp->ahciport_port_num = port;
6032
6033 /* Initialize the port condition variable */
6034 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6035
6036 /* Initialize the port mutex */
6037 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6038 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6039
6040 mutex_enter(&ahci_portp->ahciport_mutex);
6041
6042 /*
6043 * Allocate memory for received FIS structure and
6044 * command list for this port
6045 */
6046 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6047 goto err_case1;
6048 }
6049
6050 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6051 goto err_case2;
6052 }
6053
6054 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6055 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6056 AHCI_SUCCESS) {
6057 goto err_case3;
6058 }
6059
6060 (void) snprintf(taskq_name + strlen(taskq_name),
6061 sizeof (taskq_name) - strlen(taskq_name),
6062 "_port%d", port);
6063
6064 /* Create the taskq for the port */
6065 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6066 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6067 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6068 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6069 goto err_case3;
6070 }
6071
6072 /* Allocate the argument for the taskq */
6073 ahci_portp->ahciport_event_args =
6074 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6075
6076 ahci_portp->ahciport_event_args->ahciea_addrp =
6077 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6078
6079 if (ahci_portp->ahciport_event_args == NULL)
6080 goto err_case4;
6081
6082 /* Initialize the done queue */
6083 ahci_portp->ahciport_doneq = NULL;
6084 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6085 ahci_portp->ahciport_doneq_len = 0;
6086
6087 mutex_exit(&ahci_portp->ahciport_mutex);
6088
6089 return (AHCI_SUCCESS);
6090
6091 err_case4:
6092 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6093
6094 err_case3:
6095 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6096
6097 err_case2:
6098 ahci_dealloc_rcvd_fis(ahci_portp);
6099
6100 err_case1:
6101 mutex_exit(&ahci_portp->ahciport_mutex);
6102 mutex_destroy(&ahci_portp->ahciport_mutex);
6103 cv_destroy(&ahci_portp->ahciport_cv);
6104
6105 kmem_free(ahci_portp, sizeof (ahci_port_t));
6106
6107 return (AHCI_FAILURE);
6108 }
6109
6110 /*
6111 * Reverse of ahci_alloc_port_state().
6112 */
6113 static void
6114 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6115 {
6116 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6117
6118 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6119 ASSERT(ahci_portp != NULL);
6120
6121 mutex_enter(&ahci_portp->ahciport_mutex);
6122 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6123 sizeof (ahci_addr_t));
6124 ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6125 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6126 ahci_portp->ahciport_event_args = NULL;
6127 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6128 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6129 ahci_dealloc_rcvd_fis(ahci_portp);
6130 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6131 mutex_exit(&ahci_portp->ahciport_mutex);
6132
6133 mutex_destroy(&ahci_portp->ahciport_mutex);
6134 cv_destroy(&ahci_portp->ahciport_cv);
6135
6136 kmem_free(ahci_portp, sizeof (ahci_port_t));
6137
6138 ahci_ctlp->ahcictl_ports[port] = NULL;
6139 }
6140
6141 /*
6142 * Allocates memory for the Received FIS Structure
6143 */
6144 static int
6145 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6146 {
6147 size_t rcvd_fis_size;
6148 size_t ret_len;
6149 uint_t cookie_count;
6150
6151 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6152
6153 rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6154
6155 /* allocate rcvd FIS dma handle. */
6156 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6157 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6158 DDI_DMA_SLEEP,
6159 NULL,
6160 &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6161 DDI_SUCCESS) {
6162 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6163 "rcvd FIS dma handle alloc failed", NULL);
6164
6165 return (AHCI_FAILURE);
6166 }
6167
6168 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6169 rcvd_fis_size,
6170 &accattr,
6171 DDI_DMA_CONSISTENT,
6172 DDI_DMA_SLEEP,
6173 NULL,
6174 (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6175 &ret_len,
6176 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6177
6178 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6179 "rcvd FIS dma mem alloc fail", NULL);
6180 /* error.. free the dma handle. */
6181 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6182 return (AHCI_FAILURE);
6183 }
6184
6185 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6186 NULL,
6187 (caddr_t)ahci_portp->ahciport_rcvd_fis,
6188 rcvd_fis_size,
6189 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6190 DDI_DMA_SLEEP,
6191 NULL,
6192 &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6193 &cookie_count) != DDI_DMA_MAPPED) {
6194
6195 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6196 "rcvd FIS dma handle bind fail", NULL);
6197 /* error.. free the dma handle & free the memory. */
6198 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6199 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6200 return (AHCI_FAILURE);
6201 }
6202
6203 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6204
6205 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6206 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6207 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6208 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6209
6210 return (AHCI_SUCCESS);
6211 }
6212
6213 /*
6214 * Deallocates the Received FIS Structure
6215 */
6216 static void
6217 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6218 {
6219 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6220
6221 /* Unbind the cmd list dma handle first. */
6222 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6223
6224 /* Then free the underlying memory. */
6225 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6226
6227 /* Now free the handle itself. */
6228 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6229 }
6230
6231 /*
6232 * Allocates memory for the Command List, which contains up to 32 entries.
6233 * Each entry contains a command header, which is a 32-byte structure that
6234 * includes the pointer to the command table.
6235 */
6236 static int
6237 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6238 {
6239 size_t cmd_list_size;
6240 size_t ret_len;
6241 uint_t cookie_count;
6242
6243 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6244
6245 cmd_list_size =
6246 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6247
6248 /* allocate cmd list dma handle. */
6249 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6250 &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6251 DDI_DMA_SLEEP,
6252 NULL,
6253 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6254
6255 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6256 "cmd list dma handle alloc failed", NULL);
6257 return (AHCI_FAILURE);
6258 }
6259
6260 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6261 cmd_list_size,
6262 &accattr,
6263 DDI_DMA_CONSISTENT,
6264 DDI_DMA_SLEEP,
6265 NULL,
6266 (caddr_t *)&ahci_portp->ahciport_cmd_list,
6267 &ret_len,
6268 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6269
6270 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6271 "cmd list dma mem alloc fail", NULL);
6272 /* error.. free the dma handle. */
6273 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6274 return (AHCI_FAILURE);
6275 }
6276
6277 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6278 NULL,
6279 (caddr_t)ahci_portp->ahciport_cmd_list,
6280 cmd_list_size,
6281 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6282 DDI_DMA_SLEEP,
6283 NULL,
6284 &ahci_portp->ahciport_cmd_list_dma_cookie,
6285 &cookie_count) != DDI_DMA_MAPPED) {
6286
6287 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6288 "cmd list dma handle bind fail", NULL);
6289 /* error.. free the dma handle & free the memory. */
6290 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6291 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6292 return (AHCI_FAILURE);
6293 }
6294
6295 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6296
6297 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6298 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6299
6300 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6301 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6302
6303 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6304 goto err_out;
6305 }
6306
6307 return (AHCI_SUCCESS);
6308
6309 err_out:
6310 /* Unbind the cmd list dma handle first. */
6311 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6312
6313 /* Then free the underlying memory. */
6314 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6315
6316 /* Now free the handle itself. */
6317 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6318
6319 return (AHCI_FAILURE);
6320 }
6321
6322 /*
6323 * Deallocates the Command List
6324 */
6325 static void
6326 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6327 {
6328 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6329
6330 /* First dealloc command table */
6331 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6332
6333 /* Unbind the cmd list dma handle first. */
6334 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6335
6336 /* Then free the underlying memory. */
6337 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6338
6339 /* Now free the handle itself. */
6340 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6341 }
6342
6343 /*
6344 * Allocates memory for all Command Tables, which contains Command FIS,
6345 * ATAPI Command and Physical Region Descriptor Table.
6346 */
6347 static int
6348 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6349 {
6350 size_t ret_len;
6351 ddi_dma_cookie_t cmd_table_dma_cookie;
6352 uint_t cookie_count;
6353 int slot;
6354
6355 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6356
6357 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6358 "ahci_alloc_cmd_tables: port %d enter",
6359 ahci_portp->ahciport_port_num);
6360
6361 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6362 /* Allocate cmd table dma handle. */
6363 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6364 &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6365 DDI_DMA_SLEEP,
6366 NULL,
6367 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6368 DDI_SUCCESS) {
6369
6370 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6371 "cmd table dma handle alloc failed", NULL);
6372
6373 goto err_out;
6374 }
6375
6376 if (ddi_dma_mem_alloc(
6377 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6378 ahci_cmd_table_size,
6379 &accattr,
6380 DDI_DMA_CONSISTENT,
6381 DDI_DMA_SLEEP,
6382 NULL,
6383 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6384 &ret_len,
6385 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6386 NULL) {
6387
6388 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6389 "cmd table dma mem alloc fail", NULL);
6390
6391 /* error.. free the dma handle. */
6392 ddi_dma_free_handle(
6393 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6394 goto err_out;
6395 }
6396
6397 if (ddi_dma_addr_bind_handle(
6398 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6399 NULL,
6400 (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6401 ahci_cmd_table_size,
6402 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6403 DDI_DMA_SLEEP,
6404 NULL,
6405 &cmd_table_dma_cookie,
6406 &cookie_count) != DDI_DMA_MAPPED) {
6407
6408 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6409 "cmd table dma handle bind fail", NULL);
6410 /* error.. free the dma handle & free the memory. */
6411 ddi_dma_mem_free(
6412 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6413 ddi_dma_free_handle(
6414 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6415 goto err_out;
6416 }
6417
6418 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6419 ahci_cmd_table_size);
6420
6421 /* Config Port Command Table Base Address */
6422 SET_COMMAND_TABLE_BASE_ADDR(
6423 (&ahci_portp->ahciport_cmd_list[slot]),
6424 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6425
6426 #ifndef __lock_lint
6427 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6428 (&ahci_portp->ahciport_cmd_list[slot]),
6429 cmd_table_dma_cookie.dmac_laddress >> 32);
6430 #endif
6431 }
6432
6433 return (AHCI_SUCCESS);
6434 err_out:
6435
6436 for (slot--; slot >= 0; slot--) {
6437 /* Unbind the cmd table dma handle first */
6438 (void) ddi_dma_unbind_handle(
6439 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6440
6441 /* Then free the underlying memory */
6442 ddi_dma_mem_free(
6443 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6444
6445 /* Now free the handle itself */
6446 ddi_dma_free_handle(
6447 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6448 }
6449
6450 return (AHCI_FAILURE);
6451 }
6452
6453 /*
6454 * Deallocates memory for all Command Tables.
6455 */
6456 static void
6457 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6458 {
6459 int slot;
6460
6461 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6462
6463 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6464 "ahci_dealloc_cmd_tables: %d enter",
6465 ahci_portp->ahciport_port_num);
6466
6467 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6468 /* Unbind the cmd table dma handle first. */
6469 (void) ddi_dma_unbind_handle(
6470 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6471
6472 /* Then free the underlying memory. */
6473 ddi_dma_mem_free(
6474 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6475
6476 /* Now free the handle itself. */
6477 ddi_dma_free_handle(
6478 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6479 }
6480 }
6481
6482 /*
6483 * Update SATA registers at controller ports
6484 */
6485 static void
6486 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6487 sata_device_t *sd)
6488 {
6489 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6490
6491 sd->satadev_scr.sstatus =
6492 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6493 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6494 sd->satadev_scr.serror =
6495 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6496 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6497 sd->satadev_scr.scontrol =
6498 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6499 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6500 sd->satadev_scr.sactive =
6501 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6502 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6503 }
6504
6505 /*
6506 * For poll mode, ahci_port_intr will be called to emulate the interrupt
6507 */
6508 static void
6509 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6510 {
6511 uint32_t port_intr_status;
6512 uint32_t port_intr_enable;
6513
6514 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6515 "ahci_port_intr enter: port %d", port);
6516
6517 mutex_enter(&ahci_portp->ahciport_mutex);
6518 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6519 /* For SATA_OPMODE_POLLING commands */
6520 port_intr_enable =
6521 (AHCI_INTR_STATUS_DHRS |
6522 AHCI_INTR_STATUS_PSS |
6523 AHCI_INTR_STATUS_SDBS |
6524 AHCI_INTR_STATUS_UFS |
6525 AHCI_INTR_STATUS_PCS |
6526 AHCI_INTR_STATUS_PRCS |
6527 AHCI_INTR_STATUS_OFS |
6528 AHCI_INTR_STATUS_INFS |
6529 AHCI_INTR_STATUS_IFS |
6530 AHCI_INTR_STATUS_HBDS |
6531 AHCI_INTR_STATUS_HBFS |
6532 AHCI_INTR_STATUS_TFES);
6533 } else {
6534 /*
6535 * port_intr_enable indicates that the corresponding interrrupt
6536 * reporting is enabled.
6537 */
6538 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6539 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6540 }
6541
6542 /* IPMS error in port reset should be ignored according AHCI spec. */
6543 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6544 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6545 mutex_exit(&ahci_portp->ahciport_mutex);
6546
6547 /*
6548 * port_intr_stats indicates that the corresponding interrupt
6549 * condition is active.
6550 */
6551 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6552 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6553
6554 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6555 "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6556 "port_intr_enable = 0x%x",
6557 port, port_intr_status, port_intr_enable);
6558
6559 port_intr_status &= port_intr_enable;
6560
6561 /*
6562 * Pending interrupt events are indicated by the PxIS register.
6563 * Make sure we don't miss any event.
6564 */
6565 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6566 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6567 DDI_SERVICE_UNAFFECTED);
6568 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6569 DDI_FME_VERSION);
6570 return;
6571 }
6572
6573 /* First clear the port interrupts status */
6574 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6575 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6576 port_intr_status);
6577
6578 /* Check the completed non-queued commands */
6579 if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6580 AHCI_INTR_STATUS_PSS)) {
6581 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6582 ahci_portp, port);
6583 }
6584
6585 /* Check the completed queued commands */
6586 if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6587 (void) ahci_intr_set_device_bits(ahci_ctlp,
6588 ahci_portp, port);
6589 }
6590
6591 /* Check the port connect change status interrupt bit */
6592 if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6593 (void) ahci_intr_port_connect_change(ahci_ctlp,
6594 ahci_portp, port);
6595 }
6596
6597 /* Check the device mechanical presence status interrupt bit */
6598 if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6599 (void) ahci_intr_device_mechanical_presence_status(
6600 ahci_ctlp, ahci_portp, port);
6601 }
6602
6603 /* Check the PhyRdy change status interrupt bit */
6604 if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6605 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6606 port);
6607 }
6608
6609 /*
6610 * Check the non-fatal error interrupt bits, there are four
6611 * kinds of non-fatal errors at the time being:
6612 *
6613 * PxIS.UFS - Unknown FIS Error
6614 * PxIS.OFS - Overflow Error
6615 * PxIS.INFS - Interface Non-Fatal Error
6616 * PxIS.IPMS - Incorrect Port Multiplier Status Error
6617 *
6618 * For these non-fatal errors, the HBA can continue to operate,
6619 * so the driver just log the error messages.
6620 */
6621 if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6622 AHCI_INTR_STATUS_OFS |
6623 AHCI_INTR_STATUS_IPMS |
6624 AHCI_INTR_STATUS_INFS)) {
6625 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6626 port, port_intr_status);
6627 }
6628
6629 /*
6630 * Check the fatal error interrupt bits, there are four kinds
6631 * of fatal errors for AHCI controllers:
6632 *
6633 * PxIS.HBFS - Host Bus Fatal Error
6634 * PxIS.HBDS - Host Bus Data Error
6635 * PxIS.IFS - Interface Fatal Error
6636 * PxIS.TFES - Task File Error
6637 *
6638 * The fatal error means the HBA can not recover from it by
6639 * itself, and it will try to abort the transfer, and the software
6640 * must intervene to restart the port.
6641 */
6642 if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6643 AHCI_INTR_STATUS_HBDS |
6644 AHCI_INTR_STATUS_HBFS |
6645 AHCI_INTR_STATUS_TFES))
6646 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6647 port, port_intr_status);
6648
6649 /* Check the cold port detect interrupt bit */
6650 if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6651 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6652 }
6653
6654 /* Second clear the corresponding bit in IS.IPS */
6655 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6656 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6657
6658 /* Try to recover at the end of the interrupt handler. */
6659 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6660 DDI_FM_OK) {
6661 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6662 DDI_SERVICE_UNAFFECTED);
6663 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6664 DDI_FME_VERSION);
6665 }
6666 }
6667
6668 /*
6669 * Interrupt service handler
6670 */
6671 static uint_t
6672 ahci_intr(caddr_t arg1, caddr_t arg2)
6673 {
6674 #ifndef __lock_lint
6675 _NOTE(ARGUNUSED(arg2))
6676 #endif
6677 /* LINTED */
6678 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6679 ahci_port_t *ahci_portp;
6680 int32_t global_intr_status;
6681 uint8_t port;
6682
6683 /*
6684 * global_intr_status indicates that the corresponding port has
6685 * an interrupt pending.
6686 */
6687 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6688 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6689
6690 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6691 /* The interrupt is not ours */
6692 return (DDI_INTR_UNCLAIMED);
6693 }
6694
6695 /*
6696 * Check the handle after reading global_intr_status - we don't want
6697 * to miss any port with pending interrupts.
6698 */
6699 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6700 DDI_FM_OK) {
6701 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6702 DDI_SERVICE_UNAFFECTED);
6703 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6704 DDI_FME_VERSION);
6705 return (DDI_INTR_UNCLAIMED);
6706 }
6707
6708 /* Loop for all the ports */
6709 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6710 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6711 continue;
6712 }
6713 if (!((0x1 << port) & global_intr_status)) {
6714 continue;
6715 }
6716
6717 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6718
6719 /* Call ahci_port_intr */
6720 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6721 }
6722
6723 return (DDI_INTR_CLAIMED);
6724 }
6725
6726 /*
6727 * For non-queued commands, when the corresponding bit in the PxCI register
6728 * is cleared, it means the command is completed successfully. And according
6729 * to the HBA state machine, there are three conditions which possibly will
6730 * try to clear the PxCI register bit.
6731 * 1. Receive one D2H Register FIS which is with 'I' bit set
6732 * 2. Update PIO Setup FIS
6733 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6734 *
6735 * Process completed non-queued commands when the interrupt status bit -
6736 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6737 *
6738 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6739 * with the 'I' bit set. And the following commands will send thus
6740 * FIS with 'I' bit set upon the successful completion:
6741 * 1. Non-data commands
6742 * 2. DMA data-in command
6743 * 3. DMA data-out command
6744 * 4. PIO data-out command
6745 * 5. PACKET non-data commands
6746 * 6. PACKET PIO data-in command
6747 * 7. PACKET PIO data-out command
6748 * 8. PACKET DMA data-in command
6749 * 9. PACKET DMA data-out command
6750 *
6751 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6752 * with the 'I' bit set. And the following commands will send this
6753 * FIS upon the successful completion:
6754 * 1. PIO data-in command
6755 */
6756 static int
6757 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6758 ahci_port_t *ahci_portp, uint8_t port)
6759 {
6760 uint32_t port_cmd_issue = 0;
6761 uint32_t finished_tags;
6762 int finished_slot;
6763 sata_pkt_t *satapkt;
6764 ahci_fis_d2h_register_t *rcvd_fisp;
6765 #if AHCI_DEBUG
6766 ahci_cmd_header_t *cmd_header;
6767 uint32_t cmd_dmacount;
6768 #endif
6769
6770 mutex_enter(&ahci_portp->ahciport_mutex);
6771
6772 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6773 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6774 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6775 /*
6776 * Spurious interrupt. Nothing to be done.
6777 */
6778 mutex_exit(&ahci_portp->ahciport_mutex);
6779 return (AHCI_SUCCESS);
6780 }
6781
6782 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6783 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6784
6785 /* If the PxCI corrupts, don't complete the commmands. */
6786 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6787 != DDI_FM_OK) {
6788 mutex_exit(&ahci_portp->ahciport_mutex);
6789 return (AHCI_FAILURE);
6790 }
6791
6792 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6793 /* Slot 0 is always used during error recovery */
6794 finished_tags = 0x1 & ~port_cmd_issue;
6795 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6796 "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6797 "retrieval is finished, and finished_tags = 0x%x",
6798 port, finished_tags);
6799 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6800 finished_tags = 0x1 & ~port_cmd_issue;
6801 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6802 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6803 "port multiplier is finished, and finished_tags = 0x%x",
6804 port, finished_tags);
6805
6806 } else {
6807
6808 finished_tags = ahci_portp->ahciport_pending_tags &
6809 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6810 }
6811
6812 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6813 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6814 "port_cmd_issue = 0x%x finished_tags = 0x%x",
6815 ahci_portp->ahciport_pending_tags, port_cmd_issue,
6816 finished_tags);
6817
6818 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6819 (finished_tags == 0x1)) {
6820 satapkt = ahci_portp->ahciport_err_retri_pkt;
6821 ASSERT(satapkt != NULL);
6822
6823 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6824 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6825 "with SATA_PKT_COMPLETED", (void *)satapkt);
6826
6827 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6828 goto out;
6829 }
6830
6831 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6832 (finished_tags == 0x1)) {
6833 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6834 ASSERT(satapkt != NULL);
6835
6836 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6837 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6838 "with SATA_PKT_COMPLETED", (void *)satapkt);
6839
6840 /* READ PORTMULT need copy out FIS content. */
6841 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6842 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6843 ahcirf_d2h_register_fis);
6844 satapkt->satapkt_cmd.satacmd_status_reg =
6845 GET_RFIS_STATUS(rcvd_fisp);
6846 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6847 }
6848
6849 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6850 goto out;
6851 }
6852
6853 while (finished_tags) {
6854 finished_slot = ddi_ffs(finished_tags) - 1;
6855 if (finished_slot == -1) {
6856 goto out;
6857 }
6858
6859 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6860 ASSERT(satapkt != NULL);
6861 #if AHCI_DEBUG
6862 /*
6863 * For non-native queued commands, the PRD byte count field
6864 * shall contain an accurate count of the number of bytes
6865 * transferred for the command before the PxCI bit is cleared
6866 * to '0' for the command.
6867 *
6868 * The purpose of this field is to let software know how many
6869 * bytes transferred for a given operation in order to
6870 * determine if underflow occurred. When issuing native command
6871 * queuing commands, this field should not be used and is not
6872 * required to be valid since in this case underflow is always
6873 * illegal.
6874 *
6875 * For data reads, the HBA will update its PRD byte count with
6876 * the total number of bytes received from the last FIS, and
6877 * may be able to continue normally. For data writes, the
6878 * device will detect an error, and HBA most likely will get
6879 * a fatal error.
6880 *
6881 * Therefore, here just put code to debug part. And please
6882 * refer to the comment above ahci_intr_fatal_error for the
6883 * definition of underflow error.
6884 */
6885 cmd_dmacount =
6886 ahci_portp->ahciport_prd_bytecounts[finished_slot];
6887 if (cmd_dmacount) {
6888 cmd_header =
6889 &ahci_portp->ahciport_cmd_list[finished_slot];
6890 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6891 "ahci_intr_cmd_cmplt: port %d, "
6892 "PRD Byte Count = 0x%x, "
6893 "ahciport_prd_bytecounts = 0x%x", port,
6894 cmd_header->ahcich_prd_byte_count,
6895 cmd_dmacount);
6896
6897 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6898 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6899 "ahci_intr_cmd_cmplt: port %d, "
6900 "an underflow occurred", port);
6901 }
6902 }
6903 #endif
6904
6905 /*
6906 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6907 * feature, sata_special_regs flag will be set, and the
6908 * driver should copy the status and the other corresponding
6909 * register values in the D2H Register FIS received (It's
6910 * working on Non-data protocol) from the device back to
6911 * the sata_cmd.
6912 *
6913 * For every AHCI port, there is only one Received FIS
6914 * structure, which contains the FISes received from the
6915 * device, So we're trying to copy the content of D2H
6916 * Register FIS in the Received FIS structure back to
6917 * the sata_cmd.
6918 */
6919 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6920 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6921 ahcirf_d2h_register_fis);
6922 satapkt->satapkt_cmd.satacmd_status_reg =
6923 GET_RFIS_STATUS(rcvd_fisp);
6924 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6925 }
6926
6927 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6928 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6929 "with SATA_PKT_COMPLETED", (void *)satapkt);
6930
6931 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
6932 CLEAR_BIT(finished_tags, finished_slot);
6933 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
6934
6935 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6936 }
6937 out:
6938 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
6939 "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
6940 ahci_portp->ahciport_pending_tags);
6941
6942 ahci_flush_doneq(ahci_portp);
6943
6944 mutex_exit(&ahci_portp->ahciport_mutex);
6945
6946 return (AHCI_SUCCESS);
6947 }
6948
6949 /*
6950 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
6951 * with the 'I' bit set and has been copied into system memory. It will
6952 * be sent under the following situations:
6953 *
6954 * 1. NCQ command is completed
6955 *
6956 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
6957 * via the Set Device Bits FIS. When such event is generated, the software
6958 * needs to read PxSACT register and compares the current value to the
6959 * list of commands previously issue by software. ahciport_pending_ncq_tags
6960 * keeps the tags of previously issued commands.
6961 *
6962 * 2. Asynchronous Notification
6963 *
6964 * Asynchronous Notification is a feature in SATA spec 2.6.
6965 *
6966 * 1) ATAPI device will send a signal to the host when media is inserted or
6967 * removed and avoids polling the device for media changes. The signal
6968 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
6969 * set to '1'. At the moment, it's not supported yet.
6970 *
6971 * 2) Port multiplier will send a signal to the host when a hot plug event
6972 * has occured on a port multiplier port. It is used when command based
6973 * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
6974 */
6975 static int
6976 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
6977 ahci_port_t *ahci_portp, uint8_t port)
6978 {
6979 ahci_addr_t addr;
6980
6981 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
6982 "ahci_intr_set_device_bits enter: port %d", port);
6983
6984 /* Initialize HBA port address */
6985 AHCI_ADDR_SET_PORT(&addr, port);
6986
6987 /* NCQ plug handler */
6988 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
6989
6990 /* Check port multiplier's asynchronous notification events */
6991 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
6992 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
6993 ahci_portp, port);
6994 }
6995
6996 /* ATAPI events is not supported yet */
6997
6998 return (AHCI_SUCCESS);
6999 }
7000 /*
7001 * NCQ interrupt handler. Called upon a NCQ command is completed.
7002 * Only be called from ahci_intr_set_device_bits().
7003 */
7004 static int
7005 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7006 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7007 {
7008 uint32_t port_sactive;
7009 uint32_t port_cmd_issue;
7010 uint32_t issued_tags;
7011 int issued_slot;
7012 uint32_t finished_tags;
7013 int finished_slot;
7014 uint8_t port = addrp->aa_port;
7015 sata_pkt_t *satapkt;
7016
7017 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7018 "ahci_intr_set_device_bits enter: port %d", port);
7019
7020 mutex_enter(&ahci_portp->ahciport_mutex);
7021 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7022 mutex_exit(&ahci_portp->ahciport_mutex);
7023 return (AHCI_SUCCESS);
7024 }
7025
7026 /*
7027 * First the handler got which commands are finished by checking
7028 * PxSACT register
7029 */
7030 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7031 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7032
7033 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7034 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7035
7036 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7037 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7038 "port_sactive = 0x%x", port,
7039 ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7040
7041 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7042 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7043
7044 /*
7045 * For NCQ commands, the software can determine which command has
7046 * already been transmitted to the device by checking PxCI register.
7047 */
7048 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7049 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7050
7051 issued_tags = ahci_portp->ahciport_pending_tags &
7052 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7053
7054 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7055 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7056 != DDI_FM_OK) {
7057 mutex_exit(&ahci_portp->ahciport_mutex);
7058 return (AHCI_FAILURE);
7059 }
7060
7061 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7062 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7063 "port_cmd_issue = 0x%x", port,
7064 ahci_portp->ahciport_pending_tags, port_cmd_issue);
7065
7066 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7067 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7068
7069 /*
7070 * Clear ahciport_pending_tags bit when the corresponding command
7071 * is already sent down to the device.
7072 */
7073 while (issued_tags) {
7074 issued_slot = ddi_ffs(issued_tags) - 1;
7075 if (issued_slot == -1) {
7076 goto next;
7077 }
7078 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7079 CLEAR_BIT(issued_tags, issued_slot);
7080 }
7081
7082 next:
7083 while (finished_tags) {
7084 finished_slot = ddi_ffs(finished_tags) - 1;
7085 if (finished_slot == -1) {
7086 goto out;
7087 }
7088
7089 /* The command is certainly transmitted to the device */
7090 ASSERT(!(ahci_portp->ahciport_pending_tags &
7091 (0x1 << finished_slot)));
7092
7093 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7094 ASSERT(satapkt != NULL);
7095
7096 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7097 "ahci_intr_set_device_bits: sending up pkt 0x%p "
7098 "with SATA_PKT_COMPLETED", (void *)satapkt);
7099
7100 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7101 CLEAR_BIT(finished_tags, finished_slot);
7102 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7103
7104 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7105 }
7106 out:
7107 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7108 "ahci_intr_set_device_bits: port %d "
7109 "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7110 port, ahci_portp->ahciport_pending_ncq_tags,
7111 ahci_portp->ahciport_pending_tags);
7112
7113 ahci_flush_doneq(ahci_portp);
7114
7115 mutex_exit(&ahci_portp->ahciport_mutex);
7116
7117 return (AHCI_SUCCESS);
7118 }
7119
7120 /*
7121 * Port multiplier asynchronous notification event handler. Called upon a
7122 * device is hot plugged/pulled.
7123 *
7124 * The async-notification event will only be recorded by ahcipmi_snotif_tags
7125 * here and will be handled by ahci_probe_pmult().
7126 *
7127 * NOTE: called only from ahci_port_intr().
7128 */
7129 static int
7130 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7131 uint8_t port)
7132 {
7133 sata_device_t sdevice;
7134
7135 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7136 "ahci_intr_pmult_sntf_events enter: port %d ", port);
7137
7138 /* no hot-plug while attaching process */
7139 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7140 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7141 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7142 return (AHCI_SUCCESS);
7143 }
7144 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7145
7146 mutex_enter(&ahci_portp->ahciport_mutex);
7147 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7148 mutex_exit(&ahci_portp->ahciport_mutex);
7149 return (AHCI_SUCCESS);
7150 }
7151
7152 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7153
7154 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7155 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7156 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7157 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7158 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7159 AHCI_SNOTIF_CLEAR_ALL);
7160
7161 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7162 mutex_exit(&ahci_portp->ahciport_mutex);
7163 return (AHCI_SUCCESS);
7164 }
7165
7166 /* Port Multiplier sub-device hot-plug handler */
7167 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7168 mutex_exit(&ahci_portp->ahciport_mutex);
7169 return (AHCI_SUCCESS);
7170 }
7171
7172 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7173 /* Not allowed to re-enter. */
7174 mutex_exit(&ahci_portp->ahciport_mutex);
7175 return (AHCI_SUCCESS);
7176 }
7177
7178 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7179
7180 /*
7181 * NOTE:
7182 * Even if Asynchronous Notification is supported (and enabled) by
7183 * both controller and the port multiplier, the content of PxSNTF
7184 * register is always set to 0x8000 by async notification event. We
7185 * need to check GSCR[32] on the port multiplier to find out the
7186 * owner of this event.
7187 * This is not accord with SATA spec 2.6 and needs further
7188 * clarification.
7189 */
7190 /* hot-plug will not reported while reseting. */
7191 if (ahci_portp->ahciport_reset_in_progress == 1) {
7192 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7193 "port %d snotif event ignored", port);
7194 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7195 mutex_exit(&ahci_portp->ahciport_mutex);
7196 return (AHCI_SUCCESS);
7197 }
7198
7199 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7200 "PxSNTF is set to 0x%x by port multiplier",
7201 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7202
7203 /*
7204 * Now we need do some necessary operation and inform SATA framework
7205 * that link/device events has happened.
7206 */
7207 bzero((void *)&sdevice, sizeof (sata_device_t));
7208 sdevice.satadev_addr.cport = ahci_ctlp->
7209 ahcictl_port_to_cport[port];
7210 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7211 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7212 sdevice.satadev_state = SATA_PSTATE_PWRON;
7213
7214 /* Just reject packets, do not stop that port. */
7215 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7216
7217 mutex_exit(&ahci_portp->ahciport_mutex);
7218 sata_hba_event_notify(
7219 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7220 &sdevice,
7221 SATA_EVNT_PMULT_LINK_CHANGED);
7222 mutex_enter(&ahci_portp->ahciport_mutex);
7223
7224 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7225 mutex_exit(&ahci_portp->ahciport_mutex);
7226
7227 return (AHCI_SUCCESS);
7228 }
7229
7230 /*
7231 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7232 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7233 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7234 * indicates a COMINIT signal was received.
7235 *
7236 * Hot plug insertion is detected by reception of a COMINIT signal from the
7237 * device. On reception of unsolicited COMINIT, the HBA shall generate a
7238 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7239 * begin the normal communication negotiation sequence as outlined in the
7240 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7241 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7242 * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7243 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7244 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7245 * to handle hot plug insertion. In this interrupt handler, just do nothing
7246 * but print some log message and clear the bit.
7247 */
7248 static int
7249 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7250 ahci_port_t *ahci_portp, uint8_t port)
7251 {
7252 #if AHCI_DEBUG
7253 uint32_t port_serror;
7254 #endif
7255
7256 mutex_enter(&ahci_portp->ahciport_mutex);
7257
7258 #if AHCI_DEBUG
7259 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7260 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7261
7262 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7263 "ahci_intr_port_connect_change: port %d, "
7264 "port_serror = 0x%x", port, port_serror);
7265 #endif
7266
7267 /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7268 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7269 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7270 SERROR_EXCHANGED_ERR);
7271
7272 mutex_exit(&ahci_portp->ahciport_mutex);
7273
7274 return (AHCI_SUCCESS);
7275 }
7276
7277 /*
7278 * Hot Plug Operation for platforms that support Mechanical Presence
7279 * Switches.
7280 *
7281 * When set, it indicates that a mechanical presence switch attached to this
7282 * port has been opened or closed, which may lead to a change in the connection
7283 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7284 * are set to '1'.
7285 *
7286 * At the moment, this interrupt is not needed and disabled and we just log
7287 * the debug message.
7288 */
7289 static int
7290 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7291 ahci_port_t *ahci_portp, uint8_t port)
7292 {
7293 uint32_t cap_status, port_cmd_status;
7294
7295 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7296 "ahci_intr_device_mechanical_presence_status enter, "
7297 "port %d", port);
7298
7299 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7300 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7301
7302 mutex_enter(&ahci_portp->ahciport_mutex);
7303 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7304 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7305
7306 if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7307 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7308 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7309 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7310 "the interrupt: cap_status = 0x%x, "
7311 "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7312 mutex_exit(&ahci_portp->ahciport_mutex);
7313
7314 return (AHCI_SUCCESS);
7315 }
7316
7317 #if AHCI_DEBUG
7318 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7319 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7320 "The mechanical presence switch is open: "
7321 "port %d, port_cmd_status = 0x%x",
7322 port, port_cmd_status);
7323 } else {
7324 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7325 "The mechanical presence switch is close: "
7326 "port %d, port_cmd_status = 0x%x",
7327 port, port_cmd_status);
7328 }
7329 #endif
7330
7331 mutex_exit(&ahci_portp->ahciport_mutex);
7332
7333 return (AHCI_SUCCESS);
7334 }
7335
7336 /*
7337 * Native Hot Plug Support.
7338 *
7339 * When set, it indicates that the internal PHYRDY signal changed state.
7340 * This bit reflects the state of PxSERR.DIAG.N.
7341 *
7342 * There are three kinds of conditions to generate this interrupt event:
7343 * 1. a device is inserted
7344 * 2. a device is disconnected
7345 * 3. when the link enters/exits a Partial or Slumber interface power
7346 * management state
7347 *
7348 * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7349 * bit may be set due to the link entering the Partial or Slumber power
7350 * management state, rather than due to a hot plug insertion or removal
7351 * event. So far, the interface power management is disabled, so the
7352 * driver can reliably get removal detection notification via the
7353 * PxSERR.DIAG.N bit.
7354 */
7355 static int
7356 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7357 ahci_port_t *ahci_portp, uint8_t port)
7358 {
7359 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7360 sata_device_t sdevice;
7361 int dev_exists_now = 0;
7362 int dev_existed_previously = 0;
7363 ahci_addr_t port_addr;
7364
7365 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7366 "ahci_intr_phyrdy_change enter, port %d", port);
7367
7368 /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7369 mutex_enter(&ahci_portp->ahciport_mutex);
7370 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7371 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7372 SERROR_PHY_RDY_CHG);
7373 mutex_exit(&ahci_portp->ahciport_mutex);
7374
7375 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7376 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7377 (ahci_portp == NULL)) {
7378 /* The whole controller setup is not yet done. */
7379 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7380 return (AHCI_SUCCESS);
7381 }
7382 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7383
7384 mutex_enter(&ahci_portp->ahciport_mutex);
7385
7386 /* SStatus tells the presence of device. */
7387 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7388 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7389
7390 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7391 dev_exists_now = 1;
7392 }
7393
7394 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7395 dev_existed_previously = 1;
7396 }
7397
7398 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7399 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7400 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7401 "ahci_intr_phyrdy_change: port %d "
7402 "AHCI_PORT_FLAG_NODEV is cleared", port);
7403 if (dev_exists_now == 0)
7404 dev_existed_previously = 1;
7405 }
7406
7407 bzero((void *)&sdevice, sizeof (sata_device_t));
7408 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7409 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7410 sdevice.satadev_addr.pmport = 0;
7411 sdevice.satadev_state = SATA_PSTATE_PWRON;
7412 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7413
7414 AHCI_ADDR_SET_PORT(&port_addr, port);
7415
7416 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7417 if (dev_exists_now) {
7418 if (dev_existed_previously) { /* 1 -> 1 */
7419 /* Things are fine now. The loss was temporary. */
7420 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7421 "ahci_intr_phyrdy_change port %d "
7422 "device link lost/established", port);
7423
7424 mutex_exit(&ahci_portp->ahciport_mutex);
7425 sata_hba_event_notify(
7426 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7427 &sdevice,
7428 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7429 mutex_enter(&ahci_portp->ahciport_mutex);
7430
7431 } else { /* 0 -> 1 */
7432 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7433 "ahci_intr_phyrdy_change: port %d "
7434 "device link established", port);
7435
7436 /*
7437 * A new device has been detected. The new device
7438 * might be a port multiplier instead of a drive, so
7439 * we cannot update the signature directly.
7440 */
7441 (void) ahci_initialize_port(ahci_ctlp,
7442 ahci_portp, &port_addr);
7443
7444 /* Try to start the port */
7445 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7446 != AHCI_SUCCESS) {
7447 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7448 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7449 "ahci_intr_phyrdy_change: port %d failed "
7450 "at start port", port);
7451 }
7452
7453 /* Clear the max queue depth for inserted device */
7454 ahci_portp->ahciport_max_ncq_tags = 0;
7455
7456 mutex_exit(&ahci_portp->ahciport_mutex);
7457 sata_hba_event_notify(
7458 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7459 &sdevice,
7460 SATA_EVNT_LINK_ESTABLISHED);
7461 mutex_enter(&ahci_portp->ahciport_mutex);
7462
7463 }
7464 } else { /* No device exists now */
7465
7466 if (dev_existed_previously) { /* 1 -> 0 */
7467 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7468 "ahci_intr_phyrdy_change: port %d "
7469 "device link lost", port);
7470
7471 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7472 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7473 ahci_portp, port);
7474
7475 if (ahci_portp->ahciport_device_type ==
7476 SATA_DTYPE_PMULT) {
7477 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7478 }
7479
7480 /* An existing device is lost. */
7481 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7482 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7483
7484 mutex_exit(&ahci_portp->ahciport_mutex);
7485 sata_hba_event_notify(
7486 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7487 &sdevice,
7488 SATA_EVNT_LINK_LOST);
7489 mutex_enter(&ahci_portp->ahciport_mutex);
7490 }
7491 }
7492 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7493
7494 mutex_exit(&ahci_portp->ahciport_mutex);
7495
7496 return (AHCI_SUCCESS);
7497 }
7498
7499 /*
7500 * PxIS.UFS - Unknown FIS Error
7501 *
7502 * This interrupt event means an unknown FIS was received and has been
7503 * copied into system memory. An unknown FIS is not considered an illegal
7504 * FIS, unless the length received is more than 64 bytes. If an unknown
7505 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7506 * normal operation. If the unknown FIS is more than 64 bytes, then it
7507 * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7508 * a fatal error.
7509 *
7510 * PxIS.IPMS - Incorrect Port Multiplier Status
7511 *
7512 * IPMS Indicates that the HBA received a FIS from a device that did not
7513 * have a command outstanding. The IPMS bit may be set during enumeration
7514 * of devices on a Port Multiplier due to the normal Port Multiplier
7515 * enumeration process. It is recommended that IPMS only be used after
7516 * enumeration is complete on the Port Multiplier (copied from spec).
7517 *
7518 * PxIS.OFS - Overflow Error
7519 *
7520 * Command list overflow is defined as software building a command table
7521 * that has fewer total bytes than the transaction given to the device.
7522 * On device writes, the HBA will run out of data, and on reads, there
7523 * will be no room to put the data.
7524 *
7525 * For an overflow on data read, either PIO or DMA, the HBA will set
7526 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7527 * non-fatal error when the HBA can continues. Sometimes, it will cause
7528 * a fatal error and need the software to do something.
7529 *
7530 * For an overflow on data write, setting PxIS.OFS is optional for both
7531 * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7532 * software to clean up from this serious error.
7533 *
7534 * PxIS.INFS - Interface Non-Fatal Error
7535 *
7536 * This interrupt event indicates that the HBA encountered an error on
7537 * the Serial ATA interface but was able to continue operation. The kind
7538 * of error usually occurred during a non-Data FIS, and under this condition
7539 * the FIS will be re-transmitted by HBA automatically.
7540 *
7541 * When the FMA is implemented, there should be a stat structure to
7542 * record how many every kind of error happens.
7543 */
7544 static int
7545 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7546 uint8_t port, uint32_t intr_status)
7547 {
7548 uint32_t port_serror;
7549 #if AHCI_DEBUG
7550 uint32_t port_cmd_status;
7551 uint32_t port_cmd_issue;
7552 uint32_t port_sactive;
7553 int current_slot;
7554 uint32_t current_tags;
7555 sata_pkt_t *satapkt;
7556 ahci_cmd_header_t *cmd_header;
7557 uint32_t cmd_dmacount;
7558 #endif
7559
7560 mutex_enter(&ahci_portp->ahciport_mutex);
7561
7562 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7563 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7564
7565 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7566 "ahci_intr_non_fatal_error: port %d, "
7567 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7568
7569 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7570
7571 if (intr_status & AHCI_INTR_STATUS_UFS) {
7572 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7573 "ahci port %d has unknown FIS error", port);
7574
7575 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7576 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7577 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7578 SERROR_FIS_TYPE);
7579 }
7580
7581 #if AHCI_DEBUG
7582 if (intr_status & AHCI_INTR_STATUS_IPMS) {
7583 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7584 "has Incorrect Port Multiplier Status error", port);
7585 }
7586
7587 if (intr_status & AHCI_INTR_STATUS_OFS) {
7588 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7589 "ahci port %d has overflow error", port);
7590 }
7591
7592 if (intr_status & AHCI_INTR_STATUS_INFS) {
7593 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7594 "ahci port %d has interface non fatal error", port);
7595 }
7596
7597 /*
7598 * Record the error occurred command's slot.
7599 */
7600 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7601 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7602 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7603 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7604
7605 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7606 AHCI_CMD_STATUS_CCS_SHIFT;
7607
7608 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7609 satapkt = ahci_portp->ahciport_err_retri_pkt;
7610 ASSERT(satapkt != NULL);
7611 ASSERT(current_slot == 0);
7612 } else {
7613 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7614 }
7615
7616 if (satapkt != NULL) {
7617 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7618 "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7619 "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7620 satapkt->satapkt_cmd.satacmd_cmd_reg);
7621
7622 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7623 "ahci_intr_non_fatal_error: port %d, "
7624 "satapkt 0x%p is being processed when error occurs",
7625 port, (void *)satapkt);
7626
7627 /*
7628 * PRD Byte Count field of command header is not
7629 * required to reflect the total number of bytes
7630 * transferred when an overflow occurs, so here
7631 * just log the value.
7632 */
7633 cmd_dmacount =
7634 ahci_portp->ahciport_prd_bytecounts[current_slot];
7635 if (cmd_dmacount) {
7636 cmd_header = &ahci_portp->
7637 ahciport_cmd_list[current_slot];
7638 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7639 "ahci_intr_non_fatal_error: port %d, "
7640 "PRD Byte Count = 0x%x, "
7641 "ahciport_prd_bytecounts = 0x%x", port,
7642 cmd_header->ahcich_prd_byte_count,
7643 cmd_dmacount);
7644 }
7645 }
7646 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7647 /*
7648 * For queued command, list those command which have already
7649 * been transmitted to the device and still not completed.
7650 */
7651 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7652 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7653
7654 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7655 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7656
7657 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7658 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7659 "port_sactive = 0x%x port_cmd_issue = 0x%x",
7660 ahci_portp->ahciport_pending_ncq_tags,
7661 port_sactive, port_cmd_issue);
7662
7663 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7664 port_sactive & ~port_cmd_issue &
7665 AHCI_NCQ_SLOT_MASK(ahci_portp);
7666
7667 while (current_tags) {
7668 current_slot = ddi_ffs(current_tags) - 1;
7669 if (current_slot == -1) {
7670 goto out;
7671 }
7672
7673 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7674 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7675 ahci_ctlp, "ahci_intr_non_fatal_error: "
7676 "port %d, satapkt 0x%p is outstanding when "
7677 "error occurs", port, (void *)satapkt);
7678
7679 CLEAR_BIT(current_tags, current_slot);
7680 }
7681 }
7682 out:
7683 #endif
7684 mutex_exit(&ahci_portp->ahciport_mutex);
7685
7686 return (AHCI_SUCCESS);
7687 }
7688
7689 /*
7690 * According to the AHCI spec, the error types include system memory
7691 * errors, interface errors, port multiplier errors, device errors,
7692 * command list overflow, command list underflow, native command
7693 * queuing tag errors and pio data transfer errors.
7694 *
7695 * System memory errors such as target abort, master abort, and parity
7696 * may cause the host to stop, and they are serious errors and needed
7697 * to be recovered with software intervention. When system software
7698 * has given a pointer to the HBA that doesn't exist in physical memory,
7699 * a master/target abort error occurs, and PxIS.HBFS will be set. A
7700 * data error such as CRC or parity occurs, the HBA aborts the transfer
7701 * (if necessary) and PxIS.HBDS will be set.
7702 *
7703 * Interface errors are errors that occur due to electrical issues on
7704 * the interface, or protocol miscommunication between the device and
7705 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7706 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7707 * causes PxIS.IFS/PxIS.INFS to be set are
7708 * 1. in PxSERR.ERR, P bit is set to '1'
7709 * 2. in PxSERR.DIAG, C or H bit is set to '1'
7710 * 3. PhyRdy drop unexpectly, N bit is set to '1'
7711 * If the error occurred during a non-data FIS, the FIS must be
7712 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7713 * the error occurred during a data FIS, the transfer will stop, so
7714 * the error is fatal and PxIS.IFS is set.
7715 *
7716 * When a FIS arrives that updates the taskfile, the HBA checks to see
7717 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7718 * stops processing any more commands.
7719 *
7720 * Command list overflow is defined as software building a command table
7721 * that has fewer total bytes than the transaction given to the device.
7722 * On device writes, the HBA will run out of data, and on reads, there
7723 * will be no room to put the data. For an overflow on data read, either
7724 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7725 * For an overflow on data write, setting PxIS.OFS is optional for both
7726 * DMA and PIO, and a COMRESET is required by software to clean up from
7727 * this serious error.
7728 *
7729 * Command list underflow is defined as software building a command
7730 * table that has more total bytes than the transaction given to the
7731 * device. For data writes, both PIO and DMA, the device will detect
7732 * an error and end the transfer. And these errors are most likely going
7733 * to be fatal errors that will cause the port to be restarted. For
7734 * data reads, the HBA updates its PRD byte count, and may be
7735 * able to continue normally, but is not required to. And The HBA is
7736 * not required to detect underflow conditions for native command
7737 * queuing command.
7738 *
7739 * The HBA does not actively check incoming DMA Setup FISes to ensure
7740 * that the PxSACT register bit for that slot is set. Existing error
7741 * mechanisms, such as host bus failure, or bad protocol, are used to
7742 * recover from this case.
7743 *
7744 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7745 * DATA FIS must be an integral number of Dwords. If the HBA receives
7746 * a request which is not an integral number of Dwords, the HBA
7747 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7748 * software restarts the port. And the HBA ensures that the size
7749 * of the DATA FIS received during a PIO command matches the size in
7750 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7751 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7752 * stop running until software restarts the port.
7753 */
7754 /*
7755 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7756 *
7757 * PxIS.IFS indicates that the hba encountered an error on the serial ata
7758 * interface which caused the transfer to stop.
7759 *
7760 * PxIS.HBDS indicates that the hba encountered a data error
7761 * (uncorrectable ecc/parity) when reading from or writing to system memory.
7762 *
7763 * PxIS.HBFS indicates that the hba encountered a host bus error that it
7764 * cannot recover from, such as a bad software pointer.
7765 *
7766 * PxIS.TFES is set whenever the status register is updated by the device
7767 * and the error bit (bit 0) is set.
7768 */
7769 static int
7770 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7771 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7772 {
7773 uint32_t port_cmd_status;
7774 uint32_t port_serror;
7775 uint32_t task_file_status;
7776 int failed_slot;
7777 sata_pkt_t *spkt = NULL;
7778 uint8_t err_byte;
7779 ahci_event_arg_t *args;
7780 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7781 uint32_t failed_tags = 0;
7782 int task_fail_flag = 0, task_abort_flag = 0;
7783 uint32_t slot_status;
7784
7785 mutex_enter(&ahci_portp->ahciport_mutex);
7786
7787 /*
7788 * ahci_intr_phyrdy_change() may have rendered it to
7789 * SATA_DTYPE_NONE.
7790 */
7791 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7792 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7793 "ahci_intr_fatal_error: port %d no device attached, "
7794 "and just return without doing anything", port);
7795 goto out0;
7796 }
7797
7798 if (intr_status & AHCI_INTR_STATUS_TFES) {
7799 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7800 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7801 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7802 "ahci_intr_fatal_error: port %d "
7803 "task_file_status = 0x%x", port, task_file_status);
7804 task_fail_flag = 1;
7805
7806 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7807 >> AHCI_TFD_ERR_SHIFT;
7808 if (err_byte == SATA_ERROR_ABORT)
7809 task_abort_flag = 1;
7810 }
7811
7812 /*
7813 * Here we just log the fatal error info in interrupt context.
7814 * Misc recovery processing will be handled in task queue.
7815 */
7816 if (task_fail_flag == 1) {
7817 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7818 /*
7819 * Read PxCMD.CCS to determine the slot that the HBA
7820 * was processing when the error occurred.
7821 */
7822 port_cmd_status = ddi_get32(
7823 ahci_ctlp->ahcictl_ahci_acc_handle,
7824 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7825 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7826 AHCI_CMD_STATUS_CCS_SHIFT;
7827 failed_tags = 0x1 << failed_slot;
7828
7829 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7830 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7831 "ahci_intr_fatal_error: spkt 0x%p is being "
7832 "processed when fatal error occurred for port %d",
7833 spkt, port);
7834
7835 /*
7836 * Won't emit the error message if it is an IDENTIFY
7837 * DEVICE command sent to an ATAPI device.
7838 */
7839 if ((spkt != NULL) &&
7840 (spkt->satapkt_cmd.satacmd_cmd_reg ==
7841 SATAC_ID_DEVICE) &&
7842 (task_abort_flag == 1))
7843 goto out1;
7844
7845 /*
7846 * Won't emit the error message if it is an ATAPI PACKET
7847 * command
7848 */
7849 if ((spkt != NULL) &&
7850 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7851 goto out1;
7852
7853 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7854 slot_status = ddi_get32(
7855 ahci_ctlp->ahcictl_ahci_acc_handle,
7856 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7857 failed_tags = slot_status &
7858 AHCI_NCQ_SLOT_MASK(ahci_portp);
7859 }
7860 }
7861
7862 /* print the fatal error type */
7863 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7864 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7865
7866 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7867 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7868
7869 /* print PxSERR related error message */
7870 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7871
7872 /* print task file register value */
7873 if (task_fail_flag == 1) {
7874 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7875 "= 0x%x", instance, port, task_file_status);
7876 if (task_abort_flag == 1) {
7877 cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7878 "port %d are aborted", instance, port);
7879 ahci_dump_commands(ahci_ctlp, port, failed_tags);
7880 }
7881 }
7882
7883 out1:
7884 /* Prepare the argument for the taskq */
7885 args = ahci_portp->ahciport_event_args;
7886 args->ahciea_ctlp = (void *)ahci_ctlp;
7887 args->ahciea_portp = (void *)ahci_portp;
7888 args->ahciea_event = intr_status;
7889 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7890
7891 /* Start the taskq to handle error recovery */
7892 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7893 ahci_events_handler,
7894 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7895 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7896 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7897 "port %d failed", instance, port);
7898 }
7899 out0:
7900 mutex_exit(&ahci_portp->ahciport_mutex);
7901
7902 return (AHCI_SUCCESS);
7903 }
7904
7905 /*
7906 * Hot Plug Operation for platforms that support Cold Presence Detect.
7907 *
7908 * When set, a device status has changed as detected by the cold presence
7909 * detect logic. This bit can either be set due to a non-connected port
7910 * receiving a device, or a connected port having its device removed.
7911 * This bit is only valid if the port supports cold presence detect as
7912 * indicated by PxCMD.CPD set to '1'.
7913 *
7914 * At the moment, this interrupt is not needed and disabled and we just
7915 * log the debug message.
7916 */
7917 static int
7918 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
7919 ahci_port_t *ahci_portp, uint8_t port)
7920 {
7921 uint32_t port_cmd_status;
7922 sata_device_t sdevice;
7923
7924 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7925 "ahci_intr_cold_port_detect enter, port %d", port);
7926
7927 mutex_enter(&ahci_portp->ahciport_mutex);
7928
7929 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7930 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7931 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
7932 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7933 "port %d does not support cold presence detect, so "
7934 "we just ignore this interrupt", port);
7935 mutex_exit(&ahci_portp->ahciport_mutex);
7936 return (AHCI_SUCCESS);
7937 }
7938
7939 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7940 "port %d device status has changed", port);
7941
7942 bzero((void *)&sdevice, sizeof (sata_device_t));
7943 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7944 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7945 sdevice.satadev_addr.pmport = 0;
7946 sdevice.satadev_state = SATA_PSTATE_PWRON;
7947
7948 if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
7949 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7950 "port %d: a device is hot plugged", port);
7951 mutex_exit(&ahci_portp->ahciport_mutex);
7952 sata_hba_event_notify(
7953 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7954 &sdevice,
7955 SATA_EVNT_DEVICE_ATTACHED);
7956 mutex_enter(&ahci_portp->ahciport_mutex);
7957
7958 } else {
7959 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7960 "port %d: a device is hot unplugged", port);
7961 mutex_exit(&ahci_portp->ahciport_mutex);
7962 sata_hba_event_notify(
7963 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7964 &sdevice,
7965 SATA_EVNT_DEVICE_DETACHED);
7966 mutex_enter(&ahci_portp->ahciport_mutex);
7967 }
7968
7969 mutex_exit(&ahci_portp->ahciport_mutex);
7970
7971 return (AHCI_SUCCESS);
7972 }
7973
7974 /*
7975 * Enable the interrupts for a particular port.
7976 */
7977 static void
7978 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
7979 {
7980 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
7981
7982 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
7983 "ahci_enable_port_intrs enter, port %d", port);
7984
7985 /*
7986 * Clear port interrupt status before enabling interrupt
7987 */
7988 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7989 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
7990 AHCI_PORT_INTR_MASK);
7991
7992 /*
7993 * Clear the pending bit from IS.IPS
7994 */
7995 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7996 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
7997
7998 /*
7999 * Enable the following interrupts:
8000 * Device to Host Register FIS Interrupt (DHRS)
8001 * PIO Setup FIS Interrupt (PSS)
8002 * Set Device Bits Interrupt (SDBS)
8003 * Unknown FIS Interrupt (UFS)
8004 * Port Connect Change Status (PCS)
8005 * PhyRdy Change Status (PRCS)
8006 * Overflow Status (OFS)
8007 * Interface Non-fatal Error Status (INFS)
8008 * Interface Fatal Error Status (IFS)
8009 * Host Bus Data Error Status (HBDS)
8010 * Host Bus Fatal Error Status (HBFS)
8011 * Task File Error Status (TFES)
8012 */
8013 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8014 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8015 (AHCI_INTR_STATUS_DHRS |
8016 AHCI_INTR_STATUS_PSS |
8017 AHCI_INTR_STATUS_SDBS |
8018 AHCI_INTR_STATUS_UFS |
8019 AHCI_INTR_STATUS_DPS |
8020 AHCI_INTR_STATUS_PCS |
8021 AHCI_INTR_STATUS_PRCS |
8022 AHCI_INTR_STATUS_OFS |
8023 AHCI_INTR_STATUS_INFS |
8024 AHCI_INTR_STATUS_IFS |
8025 AHCI_INTR_STATUS_HBDS |
8026 AHCI_INTR_STATUS_HBFS |
8027 AHCI_INTR_STATUS_TFES));
8028 }
8029
8030 /*
8031 * Enable interrupts for all the ports.
8032 */
8033 static void
8034 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8035 {
8036 uint32_t ghc_control;
8037
8038 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8039
8040 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8041
8042 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8043 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8044
8045 ghc_control |= AHCI_HBA_GHC_IE;
8046
8047 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8048 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8049 }
8050
8051 /*
8052 * Disable interrupts for a particular port.
8053 */
8054 static void
8055 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8056 {
8057 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8058 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8059
8060 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8061 "ahci_disable_port_intrs enter, port %d", port);
8062
8063 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8064 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8065 }
8066
8067 /*
8068 * Disable interrupts for the whole HBA.
8069 *
8070 * The global bit is cleared, then all interrupt sources from all
8071 * ports are disabled.
8072 */
8073 static void
8074 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8075 {
8076 uint32_t ghc_control;
8077
8078 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8079 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8080
8081 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8082 NULL);
8083
8084 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8085 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8086
8087 ghc_control &= ~AHCI_HBA_GHC_IE;
8088
8089 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8090 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8091 }
8092
8093 /*
8094 * Handle FIXED or MSI interrupts.
8095 */
8096 /*
8097 * According to AHCI spec, the HBA may support several interrupt modes:
8098 * * pin based interrupts (FIXED)
8099 * * single MSI message interrupts
8100 * * multiple MSI based message interrupts
8101 *
8102 * For pin based interrupts, the software interrupt handler need to check IS
8103 * register to find out which port has pending interrupts. And then check
8104 * PxIS register to find out which interrupt events happened on that port.
8105 *
8106 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8107 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8108 * in that software interrupt handler need to check IS register to determine
8109 * which port triggered the interrupts since it uses a single message for all
8110 * port interrupts.
8111 *
8112 * HBA may optionally support multiple MSI message for better performance. In
8113 * this mode, each port may have its own interrupt message, and thus generation
8114 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8115 * represents a power-of-2 wrapper on the number of implemented ports, and
8116 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8117 * maximum number of assigned interrupts. When the number of MSI messages
8118 * allocated is less than the number requested, then hardware may have two
8119 * implementation behaviors:
8120 * * assign each ports its own interrupt and then force all additional
8121 * ports to share the last interrupt message, and this condition is
8122 * indicated by clearing GHC.MRSM to '0'
8123 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8124 * When multiple-message MSI is enabled, hardware will still set IS register
8125 * as single message case. And this IS register may be used by software when
8126 * fewer than the requested number of messages is granted in order to determine
8127 * which port had the interrupt.
8128 *
8129 * Note: The current ahci driver only supports the first two interrupt modes:
8130 * pin based interrupts and single MSI message interrupts, and the reason
8131 * is indicated in below code.
8132 */
8133 static int
8134 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8135 {
8136 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8137 int count, avail, actual;
8138 int i, rc;
8139
8140 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8141 "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8142
8143 /* get number of interrupts. */
8144 rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8145 if ((rc != DDI_SUCCESS) || (count == 0)) {
8146 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8147 "ddi_intr_get_nintrs() failed, "
8148 "rc %d count %d\n", rc, count);
8149 return (DDI_FAILURE);
8150 }
8151
8152 /* get number of available interrupts. */
8153 rc = ddi_intr_get_navail(dip, intr_type, &avail);
8154 if ((rc != DDI_SUCCESS) || (avail == 0)) {
8155 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8156 "ddi_intr_get_navail() failed, "
8157 "rc %d avail %d\n", rc, avail);
8158 return (DDI_FAILURE);
8159 }
8160
8161 #if AHCI_DEBUG
8162 if (avail < count) {
8163 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8164 "ddi_intr_get_nintrs returned %d, navail() returned %d",
8165 count, avail);
8166 }
8167 #endif
8168
8169 /*
8170 * Note: So far Solaris restricts the maximum number of messages for
8171 * x86 to 2, that is avail is 2, so here we set the count with 1 to
8172 * force the driver to use single MSI message interrupt. In future if
8173 * Solaris remove the restriction, then we need to delete the below
8174 * code and try to use multiple interrupt routine to gain better
8175 * performance.
8176 */
8177 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8178 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8179 "force to use one interrupt routine though the "
8180 "HBA supports %d interrupt", count);
8181 count = 1;
8182 }
8183
8184 /* Allocate an array of interrupt handles. */
8185 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8186 ahci_ctlp->ahcictl_intr_htable =
8187 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8188
8189 /* call ddi_intr_alloc(). */
8190 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8191 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8192
8193 if ((rc != DDI_SUCCESS) || (actual == 0)) {
8194 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8195 "ddi_intr_alloc() failed, rc %d count %d actual %d "
8196 "avail %d\n", rc, count, actual, avail);
8197 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8198 ahci_ctlp->ahcictl_intr_size);
8199 return (DDI_FAILURE);
8200 }
8201
8202 /* use interrupt count returned */
8203 #if AHCI_DEBUG
8204 if (actual < count) {
8205 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8206 "Requested: %d, Received: %d", count, actual);
8207 }
8208 #endif
8209
8210 ahci_ctlp->ahcictl_intr_cnt = actual;
8211
8212 /*
8213 * Get priority for first, assume remaining are all the same.
8214 */
8215 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8216 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8217 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8218 "ddi_intr_get_pri() failed", NULL);
8219
8220 /* Free already allocated intr. */
8221 for (i = 0; i < actual; i++) {
8222 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8223 }
8224
8225 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8226 ahci_ctlp->ahcictl_intr_size);
8227 return (DDI_FAILURE);
8228 }
8229
8230 /* Test for high level interrupt. */
8231 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8232 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8233 "ahci_add_intrs: Hi level intr not supported", NULL);
8234
8235 /* Free already allocated intr. */
8236 for (i = 0; i < actual; i++) {
8237 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8238 }
8239
8240 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8241 sizeof (ddi_intr_handle_t));
8242
8243 return (DDI_FAILURE);
8244 }
8245
8246 /* Call ddi_intr_add_handler(). */
8247 for (i = 0; i < actual; i++) {
8248 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8249 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8250 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8251 "ddi_intr_add_handler() failed", NULL);
8252
8253 /* Free already allocated intr. */
8254 for (i = 0; i < actual; i++) {
8255 (void) ddi_intr_free(
8256 ahci_ctlp->ahcictl_intr_htable[i]);
8257 }
8258
8259 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8260 ahci_ctlp->ahcictl_intr_size);
8261 return (DDI_FAILURE);
8262 }
8263 }
8264
8265 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8266 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8267 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8268 "ddi_intr_get_cap() failed", NULL);
8269
8270 /* Free already allocated intr. */
8271 for (i = 0; i < actual; i++) {
8272 (void) ddi_intr_free(
8273 ahci_ctlp->ahcictl_intr_htable[i]);
8274 }
8275
8276 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8277 ahci_ctlp->ahcictl_intr_size);
8278 return (DDI_FAILURE);
8279 }
8280
8281 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8282 /* Call ddi_intr_block_enable() for MSI. */
8283 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8284 ahci_ctlp->ahcictl_intr_cnt);
8285 } else {
8286 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8287 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8288 (void) ddi_intr_enable(
8289 ahci_ctlp->ahcictl_intr_htable[i]);
8290 }
8291 }
8292
8293 return (DDI_SUCCESS);
8294 }
8295
8296 /*
8297 * Removes the registered interrupts irrespective of whether they
8298 * were legacy or MSI.
8299 *
8300 * NOTE: The controller interrupts must be disabled before calling
8301 * this routine.
8302 */
8303 static void
8304 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8305 {
8306 int x;
8307
8308 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8309
8310 /* Disable all interrupts. */
8311 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8312 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8313 /* Call ddi_intr_block_disable(). */
8314 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8315 ahci_ctlp->ahcictl_intr_cnt);
8316 } else {
8317 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8318 (void) ddi_intr_disable(
8319 ahci_ctlp->ahcictl_intr_htable[x]);
8320 }
8321 }
8322
8323 /* Call ddi_intr_remove_handler(). */
8324 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8325 (void) ddi_intr_remove_handler(
8326 ahci_ctlp->ahcictl_intr_htable[x]);
8327 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8328 }
8329
8330 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8331 }
8332
8333 /*
8334 * This routine tries to put port into P:NotRunning state by clearing
8335 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8336 * and PxCMD.CR to '0'.
8337 */
8338 static int
8339 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8340 ahci_port_t *ahci_portp, uint8_t port)
8341 {
8342 uint32_t port_cmd_status;
8343 int loop_count;
8344
8345 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8346 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8347
8348 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8349 "ahci_put_port_into_notrunning_state enter: port %d", port);
8350
8351 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8352 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8353
8354 port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8355 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8356 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8357
8358 /* Wait until PxCMD.CR is cleared */
8359 loop_count = 0;
8360 do {
8361 port_cmd_status =
8362 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8363 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8364
8365 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8366 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8367 "clearing port %d CMD.CR timeout, "
8368 "port_cmd_status = 0x%x", port,
8369 port_cmd_status);
8370 /*
8371 * We are effectively timing out after 0.5 sec.
8372 * This value is specified in AHCI spec.
8373 */
8374 break;
8375 }
8376
8377 /* Wait for 10 millisec */
8378 drv_usecwait(AHCI_10MS_USECS);
8379 } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8380
8381 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8382
8383 if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8384 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8385 "ahci_put_port_into_notrunning_state: failed to clear "
8386 "PxCMD.CR to '0' after loop count: %d, and "
8387 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8388 return (AHCI_FAILURE);
8389 } else {
8390 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8391 "ahci_put_port_into_notrunning_state: succeeded to clear "
8392 "PxCMD.CR to '0' after loop count: %d, and "
8393 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8394 return (AHCI_SUCCESS);
8395 }
8396 }
8397
8398 /*
8399 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8400 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8401 * stable state, then set PxCMD.ST to '1' to start the port directly.
8402 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8403 * COMRESET to the device to put it in an idle state.
8404 *
8405 * The fifth argument returns whether the port reset is involved during
8406 * the process.
8407 *
8408 * The routine will be called under following scenarios:
8409 * + To reset the HBA
8410 * + To abort the packet(s)
8411 * + To reset the port
8412 * + To activate the port
8413 * + Fatal error recovery
8414 * + To abort the timeout packet(s)
8415 *
8416 * NOTES!!! During this procedure, PxSERR register will be cleared, and
8417 * according to the spec, the clearance of three bits will also clear
8418 * three interrupt status bits.
8419 * 1. PxSERR.DIAG.F will clear PxIS.UFS
8420 * 2. PxSERR.DIAG.X will clear PxIS.PCS
8421 * 3. PxSERR.DIAG.N will clear PxIS.PRCS
8422 *
8423 * Among these three interrupt events, the driver needs to take care of
8424 * PxIS.PRCS, which is the hot plug event. When the driver found out
8425 * a device was unplugged, it will call the interrupt handler.
8426 */
8427 static int
8428 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8429 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8430 {
8431 uint32_t port_sstatus;
8432 uint32_t task_file_status;
8433 sata_device_t sdevice;
8434 int rval;
8435 ahci_addr_t addr_port;
8436 ahci_pmult_info_t *pminfo = NULL;
8437 int dev_exists_begin = 0;
8438 int dev_exists_end = 0;
8439 uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8440 int npmport = 0;
8441 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8442
8443 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8444
8445 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8446 "ahci_restart_port_wait_till_ready: port %d enter", port);
8447
8448 AHCI_ADDR_SET_PORT(&addr_port, port);
8449
8450 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8451 dev_exists_begin = 1;
8452
8453 /* First clear PxCMD.ST */
8454 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8455 port);
8456 if (rval != AHCI_SUCCESS)
8457 /*
8458 * If PxCMD.CR does not clear within a reasonable time, it
8459 * may assume the interface is in a hung condition and may
8460 * continue with issuing the port reset.
8461 */
8462 goto reset;
8463
8464 /* Then clear PxSERR */
8465 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8466 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8467 AHCI_SERROR_CLEAR_ALL);
8468
8469 /* Then get PxTFD */
8470 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8471 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8472
8473 /*
8474 * Check whether the device is in a stable status, if yes,
8475 * then start the port directly. However for ahci_tran_reset_dport,
8476 * we may have to perform a port reset.
8477 */
8478 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8479 !(flag & AHCI_PORT_RESET))
8480 goto out;
8481
8482 reset:
8483 /*
8484 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8485 * a COMRESET to the device
8486 */
8487 ahci_disable_port_intrs(ahci_ctlp, port);
8488 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8489 ahci_enable_port_intrs(ahci_ctlp, port);
8490
8491 #ifdef AHCI_DEBUG
8492 if (rval != AHCI_SUCCESS)
8493 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8494 "ahci_restart_port_wait_till_ready: port %d failed",
8495 port);
8496 #endif
8497
8498 if (reset_flag != NULL)
8499 *reset_flag = 1;
8500
8501 /* Indicate to the framework that a reset has happened. */
8502 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8503 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8504 !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8505 /* Set the reset in progress flag */
8506 ahci_portp->ahciport_reset_in_progress = 1;
8507
8508 bzero((void *)&sdevice, sizeof (sata_device_t));
8509 sdevice.satadev_addr.cport =
8510 ahci_ctlp->ahcictl_port_to_cport[port];
8511 sdevice.satadev_addr.pmport = 0;
8512 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8513
8514 sdevice.satadev_state = SATA_DSTATE_RESET |
8515 SATA_DSTATE_PWR_ACTIVE;
8516 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8517 mutex_exit(&ahci_portp->ahciport_mutex);
8518 sata_hba_event_notify(
8519 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8520 &sdevice,
8521 SATA_EVNT_DEVICE_RESET);
8522 mutex_enter(&ahci_portp->ahciport_mutex);
8523 }
8524
8525 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8526 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8527 } else {
8528 ahci_portp->ahciport_reset_in_progress = 0;
8529 }
8530
8531 out:
8532 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8533
8534 /* SStatus tells the presence of device. */
8535 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8536 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8537
8538 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8539 dev_exists_end = 1;
8540 }
8541
8542 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8543 return (rval);
8544
8545 /* Check whether a hot plug event happened */
8546 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8547 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8548 "ahci_restart_port_wait_till_ready: port %d "
8549 "device is removed", port);
8550 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8551 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8552 "ahci_restart_port_wait_till_ready: port %d "
8553 "AHCI_PORT_FLAG_NODEV flag is set", port);
8554 mutex_exit(&ahci_portp->ahciport_mutex);
8555 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8556 mutex_enter(&ahci_portp->ahciport_mutex);
8557
8558 return (rval);
8559 }
8560
8561
8562 /* 0/1 -> 1 : device may change */
8563 /*
8564 * May be called by ahci_fatal_error_recovery_handler, so
8565 * don't issue software if the previous device is ATAPI.
8566 */
8567 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8568 return (rval);
8569
8570 /*
8571 * The COMRESET will make port multiplier enter legacy mode.
8572 * Issue a software reset to make it work again.
8573 */
8574 ahci_disable_port_intrs(ahci_ctlp, port);
8575 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8576 ahci_enable_port_intrs(ahci_ctlp, port);
8577
8578 /*
8579 * Following codes are specific for the port multiplier
8580 */
8581 if (previous_dev_type != SATA_DTYPE_PMULT &&
8582 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8583 /* in case previous_dev_type is corrupt */
8584 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8585 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8586 return (rval);
8587 }
8588
8589 /* Device change: PMult -> Non-PMult */
8590 if (previous_dev_type == SATA_DTYPE_PMULT &&
8591 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8592 /*
8593 * This might happen because
8594 * 1. Software reset failed. Port multiplier is not correctly
8595 * enumerated.
8596 * 2. Another non-port-multiplier device is attached. Perhaps
8597 * the port multiplier was replaced by another device by
8598 * whatever reason, but AHCI driver missed hot-plug event.
8599 *
8600 * Now that the port has been initialized, we just need to
8601 * update the port structure according new device, then report
8602 * and wait SATA framework to probe new device.
8603 */
8604
8605 /* Force to release pmult resource */
8606 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8607 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8608
8609 bzero((void *)&sdevice, sizeof (sata_device_t));
8610 sdevice.satadev_addr.cport =
8611 ahci_ctlp->ahcictl_port_to_cport[port];
8612 sdevice.satadev_addr.pmport = 0;
8613 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8614
8615 sdevice.satadev_state = SATA_DSTATE_RESET |
8616 SATA_DSTATE_PWR_ACTIVE;
8617
8618 mutex_exit(&ahci_portp->ahciport_mutex);
8619 sata_hba_event_notify(
8620 ahci_ctlp->ahcictl_dip,
8621 &sdevice,
8622 SATA_EVNT_DEVICE_RESET);
8623 mutex_enter(&ahci_portp->ahciport_mutex);
8624
8625 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8626 "Port multiplier is [Gone] at port %d ", port);
8627 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8628 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8629
8630 return (AHCI_SUCCESS);
8631 }
8632
8633 /* Device change: Non-PMult -> PMult */
8634 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8635
8636 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8637 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8638
8639 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8640 }
8641 pminfo = ahci_portp->ahciport_pmult_info;
8642 ASSERT(pminfo != NULL);
8643
8644 /* Device (may) change: PMult -> PMult */
8645 /*
8646 * First initialize port multiplier. Set state to READY and wait for
8647 * probe entry point to initialize it
8648 */
8649 ahci_portp->ahciport_port_state = SATA_STATE_READY;
8650
8651 /*
8652 * It's a little complicated while target is a port multiplier. we
8653 * need to COMRESET all pmports behind that PMult otherwise those
8654 * sub-links between the PMult and the sub-devices will be in an
8655 * inactive state (indicated by PSCR0/PxSSTS) and the following access
8656 * to those sub-devices will be rejected by Link-Fatal-Error.
8657 */
8658 /*
8659 * The PxSNTF will be set soon after the pmult is plugged. While the
8660 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8661 * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8662 * will initialized it.
8663 */
8664 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8665 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8666
8667 /* Report reset event. */
8668 ahci_portp->ahciport_reset_in_progress = 1;
8669
8670 bzero((void *)&sdevice, sizeof (sata_device_t));
8671 sdevice.satadev_addr.cport = cport;
8672 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8673 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8674 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8675 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8676 SATA_EVNT_DEVICE_RESET);
8677
8678 return (rval);
8679 }
8680
8681 /*
8682 * This routine may be called under four scenarios:
8683 * a) do the recovery from fatal error
8684 * b) or we need to timeout some commands
8685 * c) or we need to abort some commands
8686 * d) or we need reset device/port/controller
8687 *
8688 * In all these scenarios, we need to send any pending unfinished
8689 * commands up to sata framework.
8690 */
8691 static void
8692 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8693 ahci_port_t *ahci_portp,
8694 uint32_t slot_status,
8695 uint32_t failed_tags,
8696 uint32_t timeout_tags,
8697 uint32_t aborted_tags,
8698 uint32_t reset_tags)
8699 {
8700 uint32_t finished_tags = 0;
8701 uint32_t unfinished_tags = 0;
8702 int tmp_slot;
8703 sata_pkt_t *satapkt;
8704 int ncq_cmd_in_progress = 0;
8705 int err_retri_cmd_in_progress = 0;
8706 int rdwr_pmult_cmd_in_progress = 0;
8707
8708 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8709
8710 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8711 "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8712 ahci_portp->ahciport_port_num, slot_status);
8713
8714 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8715 "ahci_mop_commands: failed_tags: 0x%x, "
8716 "timeout_tags: 0x%x aborted_tags: 0x%x, "
8717 "reset_tags: 0x%x", failed_tags,
8718 timeout_tags, aborted_tags, reset_tags);
8719
8720 #ifdef AHCI_DEBUG
8721 if (ahci_debug_flags & AHCIDBG_ERRS) {
8722 int i;
8723 char msg_buf[200] = {0, };
8724 for (i = 0x1f; i >= 0; i--) {
8725 if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8726 msg_buf[i] = 'X';
8727 else
8728 msg_buf[i] = '.';
8729 }
8730 msg_buf[0x20] = '\0';
8731 cmn_err(CE_NOTE, "port[%d] slots: %s",
8732 ahci_portp->ahciport_port_num, msg_buf);
8733 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8734 (void *)ahci_portp->ahciport_err_retri_pkt,
8735 (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8736 }
8737 #endif
8738
8739 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8740 finished_tags = ahci_portp->ahciport_pending_tags &
8741 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8742
8743 unfinished_tags = slot_status &
8744 AHCI_SLOT_MASK(ahci_ctlp) &
8745 ~failed_tags &
8746 ~aborted_tags &
8747 ~reset_tags &
8748 ~timeout_tags;
8749 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8750 ncq_cmd_in_progress = 1;
8751 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8752 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8753
8754 unfinished_tags = slot_status &
8755 AHCI_NCQ_SLOT_MASK(ahci_portp) &
8756 ~failed_tags &
8757 ~aborted_tags &
8758 ~reset_tags &
8759 ~timeout_tags;
8760 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8761
8762 /*
8763 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8764 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8765 * complete successfully due to one of the following three
8766 * conditions:
8767 *
8768 * 1. Fatal error - failed_tags includes its slot
8769 * 2. Timed out - timeout_tags includes its slot
8770 * 3. Aborted when hot unplug - aborted_tags includes its
8771 * slot
8772 *
8773 * Please note that the command is always sent down in Slot 0
8774 */
8775 err_retri_cmd_in_progress = 1;
8776 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8777 "ahci_mop_commands is called for port %d while "
8778 "REQUEST SENSE or READ LOG EXT for error retrieval "
8779 "is being executed slot_status = 0x%x",
8780 ahci_portp->ahciport_port_num, slot_status);
8781 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8782 ASSERT(slot_status == 0x1);
8783 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8784 rdwr_pmult_cmd_in_progress = 1;
8785 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8786 "ahci_mop_commands is called for port %d while "
8787 "READ/WRITE PORTMULT command is being executed",
8788 ahci_portp->ahciport_port_num);
8789
8790 ASSERT(slot_status == 0x1);
8791 }
8792
8793 #ifdef AHCI_DEBUG
8794 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8795 "ahci_mop_commands: finished_tags: 0x%x, "
8796 "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8797 #endif
8798
8799 /* Send up finished packets with SATA_PKT_COMPLETED */
8800 while (finished_tags) {
8801 tmp_slot = ddi_ffs(finished_tags) - 1;
8802 if (tmp_slot == -1) {
8803 break;
8804 }
8805
8806 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8807 ASSERT(satapkt != NULL);
8808
8809 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8810 "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8811 (void *)satapkt);
8812
8813 /*
8814 * Cannot fetch the return register content since the port
8815 * was restarted, so the corresponding tag will be set to
8816 * aborted tags.
8817 */
8818 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8819 CLEAR_BIT(finished_tags, tmp_slot);
8820 aborted_tags |= tmp_slot;
8821 continue;
8822 }
8823
8824 if (ncq_cmd_in_progress)
8825 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8826 tmp_slot);
8827 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8828 CLEAR_BIT(finished_tags, tmp_slot);
8829 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8830
8831 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8832 }
8833
8834 /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8835 while (failed_tags) {
8836 if (err_retri_cmd_in_progress) {
8837 satapkt = ahci_portp->ahciport_err_retri_pkt;
8838 ASSERT(satapkt != NULL);
8839 ASSERT(failed_tags == 0x1);
8840
8841 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8842 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8843 (void *)satapkt);
8844 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8845 break;
8846 }
8847 if (rdwr_pmult_cmd_in_progress) {
8848 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8849 ASSERT(satapkt != NULL);
8850 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8851 "ahci_mop_commands: sending up "
8852 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8853 (void *)satapkt);
8854 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8855 break;
8856 }
8857
8858 tmp_slot = ddi_ffs(failed_tags) - 1;
8859 if (tmp_slot == -1) {
8860 break;
8861 }
8862
8863 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8864 ASSERT(satapkt != NULL);
8865
8866 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8867 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8868 (void *)satapkt);
8869
8870 if (ncq_cmd_in_progress)
8871 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8872 tmp_slot);
8873 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8874 CLEAR_BIT(failed_tags, tmp_slot);
8875 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8876
8877 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8878 }
8879
8880 /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8881 while (timeout_tags) {
8882 if (err_retri_cmd_in_progress) {
8883 satapkt = ahci_portp->ahciport_err_retri_pkt;
8884 ASSERT(satapkt != NULL);
8885 ASSERT(timeout_tags == 0x1);
8886
8887 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8888 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8889 (void *)satapkt);
8890 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8891 break;
8892 }
8893 if (rdwr_pmult_cmd_in_progress) {
8894 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8895 ASSERT(satapkt != NULL);
8896 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8897 "ahci_mop_commands: sending up "
8898 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8899 (void *)satapkt);
8900 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8901 break;
8902 }
8903
8904 tmp_slot = ddi_ffs(timeout_tags) - 1;
8905 if (tmp_slot == -1) {
8906 break;
8907 }
8908
8909 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8910 ASSERT(satapkt != NULL);
8911
8912 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8913 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8914 (void *)satapkt);
8915
8916 if (ncq_cmd_in_progress)
8917 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8918 tmp_slot);
8919 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8920 CLEAR_BIT(timeout_tags, tmp_slot);
8921 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8922
8923 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8924 }
8925
8926 /* Send up aborted packets with SATA_PKT_ABORTED */
8927 while (aborted_tags) {
8928 if (err_retri_cmd_in_progress) {
8929 satapkt = ahci_portp->ahciport_err_retri_pkt;
8930 ASSERT(satapkt != NULL);
8931 ASSERT(aborted_tags == 0x1);
8932
8933 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8934 "sending up pkt 0x%p with SATA_PKT_ABORTED",
8935 (void *)satapkt);
8936 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8937 break;
8938 }
8939 if (rdwr_pmult_cmd_in_progress) {
8940 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8941 ASSERT(satapkt != NULL);
8942 ASSERT(aborted_tags == 0x1);
8943 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8944 "ahci_mop_commands: sending up "
8945 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
8946 (void *)satapkt);
8947 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8948 break;
8949 }
8950
8951 tmp_slot = ddi_ffs(aborted_tags) - 1;
8952 if (tmp_slot == -1) {
8953 break;
8954 }
8955
8956 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8957 ASSERT(satapkt != NULL);
8958
8959 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8960 "sending up pkt 0x%p with SATA_PKT_ABORTED",
8961 (void *)satapkt);
8962
8963 if (ncq_cmd_in_progress)
8964 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8965 tmp_slot);
8966 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8967 CLEAR_BIT(aborted_tags, tmp_slot);
8968 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8969
8970 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8971 }
8972
8973 /* Send up reset packets with SATA_PKT_RESET. */
8974 while (reset_tags) {
8975 if (rdwr_pmult_cmd_in_progress) {
8976 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8977 ASSERT(satapkt != NULL);
8978 ASSERT(aborted_tags == 0x1);
8979 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8980 "ahci_mop_commands: sending up "
8981 "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
8982 (void *)satapkt);
8983 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
8984 break;
8985 }
8986
8987 tmp_slot = ddi_ffs(reset_tags) - 1;
8988 if (tmp_slot == -1) {
8989 break;
8990 }
8991
8992 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8993 ASSERT(satapkt != NULL);
8994
8995 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8996 "sending up pkt 0x%p with SATA_PKT_RESET",
8997 (void *)satapkt);
8998
8999 if (ncq_cmd_in_progress)
9000 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9001 tmp_slot);
9002 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9003 CLEAR_BIT(reset_tags, tmp_slot);
9004 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9005
9006 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9007 }
9008
9009 /* Send up unfinished packets with SATA_PKT_RESET */
9010 while (unfinished_tags) {
9011 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9012 if (tmp_slot == -1) {
9013 break;
9014 }
9015
9016 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9017 ASSERT(satapkt != NULL);
9018
9019 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9020 "sending up pkt 0x%p with SATA_PKT_RESET",
9021 (void *)satapkt);
9022
9023 if (ncq_cmd_in_progress)
9024 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9025 tmp_slot);
9026 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9027 CLEAR_BIT(unfinished_tags, tmp_slot);
9028 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9029
9030 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9031 }
9032
9033 ahci_portp->ahciport_mop_in_progress--;
9034 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9035
9036 if (ahci_portp->ahciport_mop_in_progress == 0)
9037 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9038
9039 ahci_flush_doneq(ahci_portp);
9040 }
9041
9042 /*
9043 * This routine is going to first request a READ LOG EXT sata pkt from sata
9044 * module, and then deliver it to the HBA to get the ncq failure context.
9045 * The return value is the exactly failed tags.
9046 */
9047 static uint32_t
9048 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9049 uint8_t port)
9050 {
9051 sata_device_t sdevice;
9052 sata_pkt_t *rdlog_spkt, *spkt;
9053 ddi_dma_handle_t buf_dma_handle;
9054 ahci_addr_t addr;
9055 int loop_count;
9056 int rval;
9057 int failed_slot;
9058 uint32_t failed_tags = 0;
9059 struct sata_ncq_error_recovery_page *ncq_err_page;
9060
9061 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9062 "ahci_get_rdlogext_data enter: port %d", port);
9063
9064 /* Prepare the sdevice data */
9065 bzero((void *)&sdevice, sizeof (sata_device_t));
9066 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9067
9068 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9069 sdevice.satadev_addr.pmport = 0;
9070
9071 /* Translate sata_device.satadev_addr -> ahci_addr */
9072 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9073
9074 /*
9075 * Call the sata hba interface to get a rdlog spkt
9076 */
9077 loop_count = 0;
9078 loop:
9079 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9080 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9081 if (rdlog_spkt == NULL) {
9082 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9083 /* Sleep for a while */
9084 drv_usecwait(AHCI_10MS_USECS);
9085 goto loop;
9086 }
9087 /* Timed out after 1s */
9088 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9089 "failed to get rdlog spkt for port %d", port);
9090 return (failed_tags);
9091 }
9092
9093 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9094
9095 /*
9096 * This flag is used to handle the specific error recovery when the
9097 * READ LOG EXT command gets a failure (fatal error or time-out).
9098 */
9099 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9100
9101 /*
9102 * This start is not supposed to fail because after port is restarted,
9103 * the whole command list is empty.
9104 */
9105 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9106 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9107 ahci_portp->ahciport_err_retri_pkt = NULL;
9108
9109 /* Remove the flag after READ LOG EXT command is completed */
9110 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9111
9112 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9113 /* Update the request log data */
9114 buf_dma_handle = *(ddi_dma_handle_t *)
9115 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9116 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9117 DDI_DMA_SYNC_FORKERNEL);
9118 if (rval == DDI_SUCCESS) {
9119 ncq_err_page =
9120 (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9121 satapkt_cmd.satacmd_bp->b_un.b_addr;
9122
9123 /* Get the failed tag */
9124 failed_slot = ncq_err_page->ncq_tag;
9125 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9126 "ahci_get_rdlogext_data: port %d "
9127 "failed slot %d", port, failed_slot);
9128 if (failed_slot & NQ) {
9129 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9130 "the failed slot is not a valid tag", NULL);
9131 goto out;
9132 }
9133
9134 failed_slot &= NCQ_TAG_MASK;
9135 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9136 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9137 "ahci_get_rdlogext_data: failed spkt 0x%p",
9138 (void *)spkt);
9139 if (spkt == NULL) {
9140 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9141 "the failed slot spkt is NULL", NULL);
9142 goto out;
9143 }
9144
9145 failed_tags = 0x1 << failed_slot;
9146
9147 /* Fill out the error context */
9148 ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9149 ncq_err_page);
9150 ahci_update_sata_registers(ahci_ctlp, port,
9151 &spkt->satapkt_device);
9152 }
9153 }
9154 out:
9155 sata_free_error_retrieval_pkt(rdlog_spkt);
9156
9157 return (failed_tags);
9158 }
9159
9160 /*
9161 * This routine is going to first request a REQUEST SENSE sata pkt from sata
9162 * module, and then deliver it to the HBA to get the sense data and copy
9163 * the sense data back to the orignal failed sata pkt, and free the REQUEST
9164 * SENSE sata pkt later.
9165 */
9166 static void
9167 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9168 uint8_t port, sata_pkt_t *spkt)
9169 {
9170 sata_device_t sdevice;
9171 sata_pkt_t *rs_spkt;
9172 sata_cmd_t *sata_cmd;
9173 ddi_dma_handle_t buf_dma_handle;
9174 ahci_addr_t addr;
9175 int loop_count;
9176 #if AHCI_DEBUG
9177 struct scsi_extended_sense *rqsense;
9178 #endif
9179
9180 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9181 "ahci_get_rqsense_data enter: port %d", port);
9182
9183 /* Prepare the sdevice data */
9184 bzero((void *)&sdevice, sizeof (sata_device_t));
9185 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9186
9187 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9188 sdevice.satadev_addr.pmport = 0;
9189
9190 /* Translate sata_device.satadev_addr -> ahci_addr */
9191 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9192
9193 sata_cmd = &spkt->satapkt_cmd;
9194
9195 /*
9196 * Call the sata hba interface to get a rs spkt
9197 */
9198 loop_count = 0;
9199 loop:
9200 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9201 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9202 if (rs_spkt == NULL) {
9203 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9204 /* Sleep for a while */
9205 drv_usecwait(AHCI_10MS_USECS);
9206 goto loop;
9207
9208 }
9209 /* Timed out after 1s */
9210 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9211 "failed to get rs spkt for port %d", port);
9212 return;
9213 }
9214
9215 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9216
9217 /*
9218 * This flag is used to handle the specific error recovery when the
9219 * REQUEST SENSE command gets a faiure (fatal error or time-out).
9220 */
9221 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9222
9223 /*
9224 * This start is not supposed to fail because after port is restarted,
9225 * the whole command list is empty.
9226 */
9227 ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9228 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9229 ahci_portp->ahciport_err_retri_pkt = NULL;
9230
9231 /* Remove the flag after REQUEST SENSE command is completed */
9232 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9233
9234 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9235 /* Update the request sense data */
9236 buf_dma_handle = *(ddi_dma_handle_t *)
9237 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9238 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9239 DDI_DMA_SYNC_FORKERNEL);
9240 /* Copy the request sense data */
9241 bcopy(rs_spkt->
9242 satapkt_cmd.satacmd_bp->b_un.b_addr,
9243 &sata_cmd->satacmd_rqsense,
9244 SATA_ATAPI_MIN_RQSENSE_LEN);
9245 #if AHCI_DEBUG
9246 rqsense = (struct scsi_extended_sense *)
9247 sata_cmd->satacmd_rqsense;
9248
9249 /* Dump the sense data */
9250 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9251 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9252 "Sense data for satapkt %p ATAPI cmd 0x%x",
9253 spkt, sata_cmd->satacmd_acdb[0]);
9254 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9255 " es_code 0x%x es_class 0x%x "
9256 "es_key 0x%x es_add_code 0x%x "
9257 "es_qual_code 0x%x",
9258 rqsense->es_code, rqsense->es_class,
9259 rqsense->es_key, rqsense->es_add_code,
9260 rqsense->es_qual_code);
9261 #endif
9262 }
9263
9264 sata_free_error_retrieval_pkt(rs_spkt);
9265 }
9266
9267 /*
9268 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9269 * the port must be restarted. When the HBA detects thus error, it may try
9270 * to abort a transfer. And if the transfer was aborted, the device is
9271 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9272 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9273 * that the device is in a stable status and transfers may be restarted without
9274 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9275 * then the software will send the COMRESET to do the port reset.
9276 *
9277 * Software should perform the appropriate error recovery actions based on
9278 * whether non-queued commands were being issued or natived command queuing
9279 * commands were being issued.
9280 *
9281 * And software will complete the command that had the error with error mark
9282 * to higher level software.
9283 *
9284 * Fatal errors include the following:
9285 * PxIS.IFS - Interface Fatal Error Status
9286 * PxIS.HBDS - Host Bus Data Error Status
9287 * PxIS.HBFS - Host Bus Fatal Error Status
9288 * PxIS.TFES - Task File Error Status
9289 */
9290 static void
9291 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9292 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9293 {
9294 uint32_t port_cmd_status;
9295 uint32_t slot_status = 0;
9296 uint32_t failed_tags = 0;
9297 int failed_slot;
9298 int reset_flag = 0, flag = 0;
9299 ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9300 sata_cmd_t *sata_cmd = NULL;
9301 sata_pkt_t *spkt = NULL;
9302 #if AHCI_DEBUG
9303 ahci_cmd_header_t *cmd_header;
9304 #endif
9305 uint8_t port = addrp->aa_port;
9306 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9307 int rval;
9308
9309 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9310
9311 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9312 "ahci_fatal_error_recovery_handler enter: port %d", port);
9313
9314 /* Port multiplier error */
9315 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9316 /* FBS code is neither completed nor tested. */
9317 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9318 port, intr_status);
9319
9320 /* Force a port reset */
9321 flag = AHCI_PORT_RESET;
9322 }
9323
9324 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9325 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9326
9327 /* Read PxCI to see which commands are still outstanding */
9328 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9329 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9330
9331 /*
9332 * Read PxCMD.CCS to determine the slot that the HBA
9333 * was processing when the error occurred.
9334 */
9335 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9336 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9337 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9338 AHCI_CMD_STATUS_CCS_SHIFT;
9339
9340 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9341 spkt = ahci_portp->ahciport_err_retri_pkt;
9342 ASSERT(spkt != NULL);
9343 } else {
9344 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9345 if (spkt == NULL) {
9346 /* May happen when interface errors occur? */
9347 goto next;
9348 }
9349 }
9350
9351 #if AHCI_DEBUG
9352 /*
9353 * Debugging purpose...
9354 */
9355 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9356 cmd_header =
9357 &ahci_portp->ahciport_cmd_list[failed_slot];
9358 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9359 "ahci_fatal_error_recovery_handler: port %d, "
9360 "PRD Byte Count = 0x%x, "
9361 "ahciport_prd_bytecounts = 0x%x", port,
9362 cmd_header->ahcich_prd_byte_count,
9363 ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9364 }
9365 #endif
9366
9367 sata_cmd = &spkt->satapkt_cmd;
9368
9369 /* Fill out the status and error registers for PxIS.TFES */
9370 if (intr_status & AHCI_INTR_STATUS_TFES) {
9371 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9372 ahcirf_d2h_register_fis);
9373
9374 /* Copy the error context back to the sata_cmd */
9375 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9376 }
9377
9378 /* The failed command must be one of the outstanding commands */
9379 failed_tags = 0x1 << failed_slot;
9380 ASSERT(failed_tags & slot_status);
9381
9382 /* Update the sata registers, especially PxSERR register */
9383 ahci_update_sata_registers(ahci_ctlp, port,
9384 &spkt->satapkt_device);
9385
9386 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9387 /* Read PxSACT to see which commands are still outstanding */
9388 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9389 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9390 }
9391 next:
9392
9393 #if AHCI_DEBUG
9394 /*
9395 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9396 * set, it means a fatal error happened after REQUEST SENSE command
9397 * or READ LOG EXT command is delivered to the HBA during the error
9398 * recovery process. At this time, the only outstanding command is
9399 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9400 */
9401 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9402 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9403 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9404 "command or READ LOG EXT command for error data retrieval "
9405 "failed", port);
9406 ASSERT(slot_status == 0x1);
9407 ASSERT(failed_slot == 0);
9408 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9409 SCMD_REQUEST_SENSE ||
9410 spkt->satapkt_cmd.satacmd_cmd_reg ==
9411 SATAC_READ_LOG_EXT);
9412 }
9413 #endif
9414
9415 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9416 ahci_portp->ahciport_mop_in_progress++;
9417
9418 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9419 port, flag, &reset_flag);
9420
9421 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9422 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9423 if (rval == AHCI_SUCCESS)
9424 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9425 "succeed", instance, port);
9426 else
9427 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9428 "failed", instance, port);
9429 }
9430
9431 /*
9432 * Won't retrieve error information:
9433 * 1. Port reset was involved to recover
9434 * 2. Device is gone
9435 * 3. IDENTIFY DEVICE command sent to ATAPI device
9436 * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9437 */
9438 if (reset_flag ||
9439 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9440 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9441 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9442 goto out;
9443
9444 /*
9445 * Deliver READ LOG EXT to gather information about the error when
9446 * a COMRESET has not been performed as part of the error recovery
9447 * during NCQ command processing.
9448 */
9449 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9450 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9451 ahci_portp, port);
9452 goto out;
9453 }
9454
9455 /*
9456 * Deliver REQUEST SENSE for ATAPI command to gather information about
9457 * the error when a COMRESET has not been performed as part of the
9458 * error recovery.
9459 */
9460 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9461 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9462 out:
9463 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9464 "ahci_fatal_error_recovery_handler: port %d fatal error "
9465 "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9466 "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9467 port, slot_status, ahci_portp->ahciport_pending_tags,
9468 ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9469
9470 ahci_mop_commands(ahci_ctlp,
9471 ahci_portp,
9472 slot_status,
9473 failed_tags, /* failed tags */
9474 0, /* timeout tags */
9475 0, /* aborted tags */
9476 0); /* reset tags */
9477 }
9478
9479 /*
9480 * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9481 * 1. device specific.PxFBS.SDE=1
9482 * 2. Non-Deivce specific.
9483 * Nothing will be done when Command-based switching is employed.
9484 *
9485 * Currently code is neither completed nor tested.
9486 */
9487 static void
9488 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9489 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9490 {
9491 #ifndef __lock_lint
9492 _NOTE(ARGUNUSED(intr_status))
9493 #endif
9494 uint32_t port_fbs_ctrl;
9495 int loop_count = 0;
9496 ahci_addr_t addr;
9497
9498 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9499
9500 /* Nothing will be done under Command-based switching. */
9501 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9502 return;
9503
9504 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9505 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9506
9507 if (!(port_fbs_ctrl & AHCI_FBS_EN))
9508 /* FBS is not enabled. */
9509 return;
9510
9511 /* Problem's getting complicated now. */
9512 /*
9513 * If FIS-based switching is used, we need to check
9514 * the PxFBS to see the error type.
9515 */
9516 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9517 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9518
9519 /* Refer to spec(v1.2) 9.3.6.1 */
9520 if (port_fbs_ctrl & AHCI_FBS_SDE) {
9521 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9522 "A Device Sepcific Error: port %d", port);
9523 /*
9524 * Controller has paused commands for all other
9525 * sub-devices until PxFBS.DEC is set.
9526 */
9527 ahci_reject_all_abort_pkts(ahci_ctlp,
9528 ahci_portp, 0);
9529
9530 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9531 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9532 port_fbs_ctrl | AHCI_FBS_DEC);
9533
9534 /*
9535 * Wait controller clear PxFBS.DEC,
9536 * then we can continue.
9537 */
9538 loop_count = 0;
9539 do {
9540 port_fbs_ctrl = ddi_get32(ahci_ctlp->
9541 ahcictl_ahci_acc_handle, (uint32_t *)
9542 AHCI_PORT_PxFBS(ahci_ctlp, port));
9543
9544 if (loop_count++ > 1000)
9545 /*
9546 * Esclate the error. Follow
9547 * non-device specific error
9548 * procedure.
9549 */
9550 return;
9551
9552 drv_usecwait(AHCI_100US_USECS);
9553 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9554
9555 /*
9556 * Issue a software reset to ensure drive is in
9557 * a known state.
9558 */
9559 (void) ahci_software_reset(ahci_ctlp,
9560 ahci_portp, &addr);
9561
9562 } else {
9563
9564 /* Process Non-Device Specific Error. */
9565 /* This will be handled later on. */
9566 cmn_err(CE_NOTE, "!FBS is not supported now.");
9567 }
9568 }
9569 /*
9570 * Handle events - fatal error recovery
9571 */
9572 static void
9573 ahci_events_handler(void *args)
9574 {
9575 ahci_event_arg_t *ahci_event_arg;
9576 ahci_ctl_t *ahci_ctlp;
9577 ahci_port_t *ahci_portp;
9578 ahci_addr_t *addrp;
9579 uint32_t event;
9580 int instance;
9581
9582 ahci_event_arg = (ahci_event_arg_t *)args;
9583
9584 ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9585 ahci_portp = ahci_event_arg->ahciea_portp;
9586 addrp = ahci_event_arg->ahciea_addrp;
9587 event = ahci_event_arg->ahciea_event;
9588 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9589
9590 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9591 "ahci_events_handler enter: port %d intr_status = 0x%x",
9592 ahci_portp->ahciport_port_num, event);
9593
9594 mutex_enter(&ahci_portp->ahciport_mutex);
9595
9596 /*
9597 * ahci_intr_phyrdy_change() may have rendered it to
9598 * SATA_DTYPE_NONE.
9599 */
9600 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9601 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9602 "ahci_events_handler: port %d no device attached, "
9603 "and just return without doing anything",
9604 ahci_portp->ahciport_port_num);
9605
9606 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9607 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9608 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9609 "succeed", instance, ahci_portp->ahciport_port_num);
9610 }
9611
9612 goto out;
9613 }
9614
9615 if (event & (AHCI_INTR_STATUS_IFS |
9616 AHCI_INTR_STATUS_HBDS |
9617 AHCI_INTR_STATUS_HBFS |
9618 AHCI_INTR_STATUS_TFES))
9619 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9620 addrp, event);
9621
9622 out:
9623 mutex_exit(&ahci_portp->ahciport_mutex);
9624 }
9625
9626 /*
9627 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9628 * detect there are some commands which are timed out.
9629 */
9630 static void
9631 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9632 uint8_t port, uint32_t tmp_timeout_tags)
9633 {
9634 uint32_t slot_status = 0;
9635 uint32_t finished_tags = 0;
9636 uint32_t timeout_tags = 0;
9637
9638 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9639 "ahci_timeout_pkts enter: port %d", port);
9640
9641 mutex_enter(&ahci_portp->ahciport_mutex);
9642
9643 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9644 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9645 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9646 /* Read PxCI to see which commands are still outstanding */
9647 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9648 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9649 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9650 /* Read PxSACT to see which commands are still outstanding */
9651 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9652 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9653 }
9654
9655 #if AHCI_DEBUG
9656 /*
9657 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9658 * set, it means a fatal error happened after REQUEST SENSE command
9659 * or READ LOG EXT command is delivered to the HBA during the error
9660 * recovery process. At this time, the only outstanding command is
9661 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9662 */
9663 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9664 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9665 "ahci_timeout_pkts called while REQUEST SENSE "
9666 "command or READ LOG EXT command for error recovery "
9667 "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9668 "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9669 tmp_timeout_tags, slot_status,
9670 ahci_portp->ahciport_pending_tags,
9671 ahci_portp->ahciport_pending_ncq_tags);
9672 ASSERT(slot_status == 0x1);
9673 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9674 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9675 "ahci_timeout_pkts called while executing R/W PMULT "
9676 "command timeout_tags = 0x%x, slot_status = 0x%x",
9677 tmp_timeout_tags, slot_status);
9678 ASSERT(slot_status == 0x1);
9679 }
9680 #endif
9681
9682 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9683 ahci_portp->ahciport_mop_in_progress++;
9684
9685 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9686 port, AHCI_PORT_RESET, NULL);
9687
9688 /*
9689 * Re-identify timeout tags because some previously checked commands
9690 * could already complete.
9691 */
9692 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9693 finished_tags = ahci_portp->ahciport_pending_tags &
9694 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9695 timeout_tags = tmp_timeout_tags & ~finished_tags;
9696
9697 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9698 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9699 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9700 "pending_tags = 0x%x ",
9701 port, finished_tags, timeout_tags,
9702 slot_status, ahci_portp->ahciport_pending_tags);
9703 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9704 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9705 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9706 timeout_tags = tmp_timeout_tags & ~finished_tags;
9707
9708 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9709 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9710 "timeout_tags = 0x%x, port_sactive = 0x%x, "
9711 "pending_ncq_tags = 0x%x ",
9712 port, finished_tags, timeout_tags,
9713 slot_status, ahci_portp->ahciport_pending_ncq_tags);
9714 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9715 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9716 timeout_tags = tmp_timeout_tags;
9717 }
9718
9719 ahci_mop_commands(ahci_ctlp,
9720 ahci_portp,
9721 slot_status,
9722 0, /* failed tags */
9723 timeout_tags, /* timeout tags */
9724 0, /* aborted tags */
9725 0); /* reset tags */
9726
9727 mutex_exit(&ahci_portp->ahciport_mutex);
9728 }
9729
9730 /*
9731 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9732 * for long time.
9733 */
9734 static void
9735 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9736 {
9737 ahci_port_t *ahci_portp;
9738 sata_pkt_t *spkt;
9739 uint32_t pending_tags;
9740 uint32_t timeout_tags;
9741 uint32_t port_cmd_status;
9742 uint32_t port_sactive;
9743 uint8_t port;
9744 int tmp_slot;
9745 int current_slot;
9746 uint32_t current_tags;
9747 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9748
9749 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9750
9751 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9752 "ahci_watchdog_handler entered", NULL);
9753
9754 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9755 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9756 continue;
9757 }
9758
9759 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9760
9761 mutex_enter(&ahci_portp->ahciport_mutex);
9762 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9763 mutex_exit(&ahci_portp->ahciport_mutex);
9764 continue;
9765 }
9766
9767 /* Skip the check for those ports in error recovery */
9768 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9769 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9770 mutex_exit(&ahci_portp->ahciport_mutex);
9771 continue;
9772 }
9773
9774 pending_tags = 0;
9775 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9776 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9777
9778 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9779 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9780 current_slot = 0;
9781 pending_tags = 0x1;
9782 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9783 current_slot =
9784 (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9785 AHCI_CMD_STATUS_CCS_SHIFT;
9786 pending_tags = ahci_portp->ahciport_pending_tags;
9787 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9788 port_sactive = ddi_get32(
9789 ahci_ctlp->ahcictl_ahci_acc_handle,
9790 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9791 current_tags = port_sactive &
9792 ~port_cmd_status &
9793 AHCI_NCQ_SLOT_MASK(ahci_portp);
9794 pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9795 }
9796
9797 timeout_tags = 0;
9798 while (pending_tags) {
9799 tmp_slot = ddi_ffs(pending_tags) - 1;
9800 if (tmp_slot == -1) {
9801 break;
9802 }
9803
9804 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9805 spkt = ahci_portp->ahciport_err_retri_pkt;
9806 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9807 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9808 else
9809 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9810
9811 if ((spkt != NULL) && spkt->satapkt_time &&
9812 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9813 /*
9814 * If a packet has survived for more than it's
9815 * max life cycles, it is a candidate for time
9816 * out.
9817 */
9818 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9819 ahci_watchdog_timeout;
9820
9821 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9822 > 0)
9823 goto next;
9824
9825 #if AHCI_DEBUG
9826 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9827 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9828 ahci_ctlp, "watchdog: the current "
9829 "tags is 0x%x", current_tags);
9830 } else {
9831 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9832 ahci_ctlp, "watchdog: the current "
9833 "slot is %d", current_slot);
9834 }
9835 #endif
9836
9837 /*
9838 * We need to check whether the HBA has
9839 * begun to execute the command, if not,
9840 * then re-set the timer of the command.
9841 */
9842 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9843 (tmp_slot != current_slot) ||
9844 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9845 ((0x1 << tmp_slot) & current_tags)) {
9846 ahci_portp->ahciport_slot_timeout \
9847 [tmp_slot] = spkt->satapkt_time;
9848 } else {
9849 timeout_tags |= (0x1 << tmp_slot);
9850 cmn_err(CE_WARN, "!ahci%d: watchdog "
9851 "port %d satapkt 0x%p timed out\n",
9852 instance, port, (void *)spkt);
9853 }
9854 }
9855 next:
9856 CLEAR_BIT(pending_tags, tmp_slot);
9857 }
9858
9859 if (timeout_tags) {
9860 mutex_exit(&ahci_portp->ahciport_mutex);
9861 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9862 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9863 port, timeout_tags);
9864 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9865 mutex_enter(&ahci_portp->ahciport_mutex);
9866 }
9867
9868 mutex_exit(&ahci_portp->ahciport_mutex);
9869 }
9870
9871 /* Re-install the watchdog timeout handler */
9872 if (ahci_ctlp->ahcictl_timeout_id != 0) {
9873 ahci_ctlp->ahcictl_timeout_id =
9874 timeout((void (*)(void *))ahci_watchdog_handler,
9875 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9876 }
9877
9878 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9879 }
9880
9881 /*
9882 * Fill the error context into sata_cmd for non-queued command error.
9883 */
9884 static void
9885 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9886 {
9887 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9888 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9889 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9890 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9891 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9892 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9893 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9894
9895 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9896 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9897 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9898 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9899 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9900 }
9901 }
9902
9903 /*
9904 * Fill the ncq error page into sata_cmd for queued command error.
9905 */
9906 static void
9907 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9908 struct sata_ncq_error_recovery_page *ncq_err_page)
9909 {
9910 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9911 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9912 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
9913 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
9914 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
9915 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
9916 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
9917 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
9918 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
9919 scmd->satacmd_status_reg = ncq_err_page->ncq_status;
9920 scmd->satacmd_error_reg = ncq_err_page->ncq_error;
9921 }
9922
9923 /*
9924 * Put the respective register value to sata_cmd_t for satacmd_flags.
9925 */
9926 static void
9927 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9928 {
9929 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
9930 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9931 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
9932 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9933 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
9934 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9935 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
9936 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9937 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
9938 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9939 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
9940 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9941 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
9942 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9943 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
9944 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9945 if (scmd->satacmd_flags.sata_copy_out_device_reg)
9946 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9947 if (scmd->satacmd_flags.sata_copy_out_error_reg)
9948 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9949 }
9950
9951 static void
9952 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
9953 uint32_t intr_status)
9954 {
9955 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9956
9957 if (intr_status & AHCI_INTR_STATUS_IFS)
9958 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
9959 "error", instance, port);
9960
9961 if (intr_status & AHCI_INTR_STATUS_HBDS)
9962 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
9963 instance, port);
9964
9965 if (intr_status & AHCI_INTR_STATUS_HBFS)
9966 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
9967 instance, port);
9968
9969 if (intr_status & AHCI_INTR_STATUS_TFES)
9970 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
9971 instance, port);
9972
9973 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
9974 "recovery", instance, port);
9975 }
9976
9977 static void
9978 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
9979 uint32_t slot_tags)
9980 {
9981 ahci_port_t *ahci_portp;
9982 int tmp_slot;
9983 sata_pkt_t *spkt;
9984 sata_cmd_t cmd;
9985
9986 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9987 ASSERT(ahci_portp != NULL);
9988
9989 while (slot_tags) {
9990 tmp_slot = ddi_ffs(slot_tags) - 1;
9991 if (tmp_slot == -1) {
9992 break;
9993 }
9994
9995 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9996 ASSERT(spkt != NULL);
9997 cmd = spkt->satapkt_cmd;
9998
9999 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
10000 "features_reg = 0x%x sec_count_msb = 0x%x "
10001 "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
10002 "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
10003 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
10004 "lba_high_lsb = 0x%x device_reg = 0x%x "
10005 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10006 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10007 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10008 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10009 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10010 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10011 cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10012 *((uint32_t *)&(cmd.satacmd_flags)));
10013
10014 CLEAR_BIT(slot_tags, tmp_slot);
10015 }
10016 }
10017
10018 /*
10019 * Dump the serror message to the log.
10020 */
10021 static void
10022 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10023 uint32_t port_serror, int debug_only)
10024 {
10025 static char err_buf[512];
10026 static char err_msg_header[16];
10027 char *err_msg = err_buf;
10028
10029 *err_buf = '\0';
10030 *err_msg_header = '\0';
10031
10032 if (port_serror & SERROR_DATA_ERR_FIXED) {
10033 err_msg = strcat(err_msg,
10034 "\tRecovered Data Integrity Error (I)\n");
10035 }
10036
10037 if (port_serror & SERROR_COMM_ERR_FIXED) {
10038 err_msg = strcat(err_msg,
10039 "\tRecovered Communication Error (M)\n");
10040 }
10041
10042 if (port_serror & SERROR_DATA_ERR) {
10043 err_msg = strcat(err_msg,
10044 "\tTransient Data Integrity Error (T)\n");
10045 }
10046
10047 if (port_serror & SERROR_PERSISTENT_ERR) {
10048 err_msg = strcat(err_msg,
10049 "\tPersistent Communication or Data Integrity Error (C)\n");
10050 }
10051
10052 if (port_serror & SERROR_PROTOCOL_ERR) {
10053 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10054 }
10055
10056 if (port_serror & SERROR_INT_ERR) {
10057 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10058 }
10059
10060 if (port_serror & SERROR_PHY_RDY_CHG) {
10061 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10062 }
10063
10064 if (port_serror & SERROR_PHY_INT_ERR) {
10065 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10066 }
10067
10068 if (port_serror & SERROR_COMM_WAKE) {
10069 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10070 }
10071
10072 if (port_serror & SERROR_10B_TO_8B_ERR) {
10073 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10074 }
10075
10076 if (port_serror & SERROR_DISPARITY_ERR) {
10077 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10078 }
10079
10080 if (port_serror & SERROR_CRC_ERR) {
10081 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10082 }
10083
10084 if (port_serror & SERROR_HANDSHAKE_ERR) {
10085 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10086 }
10087
10088 if (port_serror & SERROR_LINK_SEQ_ERR) {
10089 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10090 }
10091
10092 if (port_serror & SERROR_TRANS_ERR) {
10093 err_msg = strcat(err_msg,
10094 "\tTransport state transition error (T)\n");
10095 }
10096
10097 if (port_serror & SERROR_FIS_TYPE) {
10098 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10099 }
10100
10101 if (port_serror & SERROR_EXCHANGED_ERR) {
10102 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10103 }
10104
10105 if (*err_msg == '\0')
10106 return;
10107
10108 if (debug_only) {
10109 (void) sprintf(err_msg_header, "port %d", port);
10110 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10111 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10112 } else if (ahci_ctlp) {
10113 cmn_err(CE_WARN, "!ahci%d: %s %s",
10114 ddi_get_instance(ahci_ctlp->ahcictl_dip),
10115 err_msg_header, err_msg);
10116
10117 /* sata trace debug */
10118 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10119 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10120 err_msg_header, err_msg);
10121 } else {
10122 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10123
10124 /* sata trace debug */
10125 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10126 }
10127 }
10128
10129 /*
10130 * Translate the sata_address_t type into the ahci_addr_t type.
10131 * sata_device.satadev_addr structure is used as source.
10132 */
10133 static void
10134 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10135 ahci_addr_t *ahci_addrp)
10136 {
10137 sata_address_t *sata_addrp = &sd->satadev_addr;
10138 ahci_addrp->aa_port =
10139 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10140 ahci_addrp->aa_pmport = sata_addrp->pmport;
10141
10142 switch (sata_addrp->qual) {
10143 case SATA_ADDR_DCPORT:
10144 case SATA_ADDR_CPORT:
10145 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10146 break;
10147 case SATA_ADDR_PMULT:
10148 case SATA_ADDR_PMULT_SPEC:
10149 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10150 break;
10151 case SATA_ADDR_DPMPORT:
10152 case SATA_ADDR_PMPORT:
10153 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10154 break;
10155 case SATA_ADDR_NULL:
10156 default:
10157 /* something went wrong */
10158 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10159 break;
10160 }
10161 }
10162
10163 /*
10164 * This routine is to calculate the total number of ports implemented
10165 * by the HBA.
10166 */
10167 static int
10168 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10169 {
10170 uint8_t i;
10171 int num = 0;
10172
10173 for (i = 0; i < AHCI_MAX_PORTS; i++) {
10174 if (((uint32_t)0x1 << i) & ports_implemented)
10175 num++;
10176 }
10177
10178 return (num);
10179 }
10180
10181 #if AHCI_DEBUG
10182 static void
10183 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10184 {
10185 static char name[16];
10186 va_list ap;
10187
10188 mutex_enter(&ahci_log_mutex);
10189
10190 va_start(ap, fmt);
10191 if (ahci_ctlp) {
10192 (void) sprintf(name, "ahci%d: ",
10193 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10194 } else {
10195 (void) sprintf(name, "ahci: ");
10196 }
10197
10198 (void) vsprintf(ahci_log_buf, fmt, ap);
10199 va_end(ap);
10200
10201 cmn_err(level, "%s%s", name, ahci_log_buf);
10202
10203 mutex_exit(&ahci_log_mutex);
10204 }
10205 #endif
10206
10207 /*
10208 * quiesce(9E) entry point.
10209 *
10210 * This function is called when the system is single-threaded at high
10211 * PIL with preemption disabled. Therefore, this function must not be
10212 * blocked.
10213 *
10214 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10215 * DDI_FAILURE indicates an error condition and should almost never happen.
10216 */
10217 static int
10218 ahci_quiesce(dev_info_t *dip)
10219 {
10220 ahci_ctl_t *ahci_ctlp;
10221 ahci_port_t *ahci_portp;
10222 int instance, port;
10223
10224 instance = ddi_get_instance(dip);
10225 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10226
10227 if (ahci_ctlp == NULL)
10228 return (DDI_FAILURE);
10229
10230 #if AHCI_DEBUG
10231 ahci_debug_flags = 0;
10232 #endif
10233
10234 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10235
10236 /* disable all the interrupts. */
10237 ahci_disable_all_intrs(ahci_ctlp);
10238
10239 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10240 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10241 continue;
10242 }
10243
10244 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10245
10246 /*
10247 * Stop the port by clearing PxCMD.ST
10248 *
10249 * Here we must disable the port interrupt because
10250 * ahci_disable_all_intrs only clear GHC.IE, and IS
10251 * register will be still set if PxIE is enabled.
10252 * When ahci shares one IRQ with other drivers, the
10253 * intr handler may claim the intr mistakenly.
10254 */
10255 ahci_disable_port_intrs(ahci_ctlp, port);
10256 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10257 ahci_portp, port);
10258 }
10259
10260 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10261
10262 return (DDI_SUCCESS);
10263 }
10264
10265 /*
10266 * The function will add a sata packet to the done queue.
10267 */
10268 static void
10269 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10270 {
10271 ASSERT(satapkt != NULL);
10272 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10273
10274 /* set the reason for all packets */
10275 satapkt->satapkt_reason = reason;
10276 satapkt->satapkt_hba_driver_private = NULL;
10277
10278 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10279 satapkt->satapkt_comp) {
10280 /*
10281 * only add to queue when mode is not synch and there is
10282 * completion callback
10283 */
10284 *ahci_portp->ahciport_doneqtail = satapkt;
10285 ahci_portp->ahciport_doneqtail =
10286 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10287 ahci_portp->ahciport_doneq_len++;
10288
10289 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10290 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10291 /*
10292 * for sync/non-poll mode, just call cv_broadcast
10293 */
10294 cv_broadcast(&ahci_portp->ahciport_cv);
10295 }
10296
10297 /*
10298 * The function will call completion callback of sata packet on the
10299 * completed queue
10300 */
10301 static void
10302 ahci_flush_doneq(ahci_port_t *ahci_portp)
10303 {
10304 sata_pkt_t *satapkt, *next;
10305
10306 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10307
10308 if (ahci_portp->ahciport_doneq) {
10309 satapkt = ahci_portp->ahciport_doneq;
10310
10311 ahci_portp->ahciport_doneq = NULL;
10312 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10313 ahci_portp->ahciport_doneq_len = 0;
10314
10315 mutex_exit(&ahci_portp->ahciport_mutex);
10316
10317 while (satapkt != NULL) {
10318 next = satapkt->satapkt_hba_driver_private;
10319 satapkt->satapkt_hba_driver_private = NULL;
10320
10321 /* Call the callback */
10322 (*satapkt->satapkt_comp)(satapkt);
10323
10324 satapkt = next;
10325 }
10326
10327 mutex_enter(&ahci_portp->ahciport_mutex);
10328 }
10329 }