1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 */
26
27 /*
28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
29 *
30 * Power Management Support
31 * ------------------------
32 *
33 * At the moment, the ahci driver only implements suspend/resume to
34 * support Suspend to RAM on X86 feature. Device power management isn't
35 * implemented, link power management is disabled, and hot plug isn't
36 * allowed during the period from suspend to resume.
37 *
38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
39 * and DDI_RESUME entries, and don't need to take care of new requests
40 * sent down after suspend because the target driver (sd) has already
41 * handled these conditions, and blocked these requests. For the detailed
42 * information, please check with sdopen, sdclose and sdioctl routines.
43 *
44 */
45
46 #include <sys/note.h>
47 #include <sys/scsi/scsi.h>
48 #include <sys/pci.h>
49 #include <sys/disp.h>
50 #include <sys/sata/sata_hba.h>
51 #include <sys/sata/adapters/ahci/ahcireg.h>
52 #include <sys/sata/adapters/ahci/ahcivar.h>
53
54 /*
55 * FMA header files
56 */
57 #include <sys/ddifm.h>
58 #include <sys/fm/protocol.h>
59 #include <sys/fm/util.h>
60 #include <sys/fm/io/ddi.h>
61
62 /*
63 * This is the string displayed by modinfo, etc.
64 */
65 static char ahci_ident[] = "ahci driver";
66
67 /*
68 * Function prototypes for driver entry points
69 */
70 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
71 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
72 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
73 static int ahci_quiesce(dev_info_t *);
74
75 /*
76 * Function prototypes for SATA Framework interfaces
77 */
78 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
79 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
80
81 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
82 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
83 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
84 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
85 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
86 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
87 #if defined(__lock_lint)
88 static int ahci_selftest(dev_info_t *, sata_device_t *);
89 #endif
90
91 /*
92 * FMA Prototypes
93 */
94 static void ahci_fm_init(ahci_ctl_t *);
95 static void ahci_fm_fini(ahci_ctl_t *);
96 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
97 int ahci_check_acc_handle(ddi_acc_handle_t);
98 int ahci_check_dma_handle(ddi_dma_handle_t);
99 void ahci_fm_ereport(ahci_ctl_t *, char *);
100 static int ahci_check_all_handle(ahci_ctl_t *);
101 static int ahci_check_ctl_handle(ahci_ctl_t *);
102 static int ahci_check_port_handle(ahci_ctl_t *, int);
103 static int ahci_check_slot_handle(ahci_port_t *, int);
104
105 /*
106 * Local function prototypes
107 */
108 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
109 static int ahci_alloc_ports_state(ahci_ctl_t *);
110 static void ahci_dealloc_ports_state(ahci_ctl_t *);
111 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
112 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
113 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
114 static void ahci_dealloc_rcvd_fis(ahci_port_t *);
115 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
116 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
117 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
118 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
119 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
120 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
121
122 static int ahci_initialize_controller(ahci_ctl_t *);
123 static void ahci_uninitialize_controller(ahci_ctl_t *);
124 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
125 static int ahci_config_space_init(ahci_ctl_t *);
126 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
127
128 static void ahci_drain_ports_taskq(ahci_ctl_t *);
129 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
130 uint8_t);
131 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
132 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
133 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
134 sata_device_t *);
135 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
136 sata_pmult_gscr_t *);
137 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
138 sata_device_t *);
139 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
140 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
141 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
142 sata_device_t *);
143
144 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
145 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
146 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
147 ahci_addr_t *);
148 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
149 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
150 ahci_addr_t *, sata_pkt_t *);
151 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
152 ahci_addr_t *, sata_pkt_t *);
153 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
154 ahci_addr_t *, int);
155 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
156 static void ahci_copy_ncq_err_page(sata_cmd_t *,
157 struct sata_ncq_error_recovery_page *);
158 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
159 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
160 static void ahci_flush_doneq(ahci_port_t *);
161
162 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
163 static int ahci_hba_reset(ahci_ctl_t *);
164 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
165 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
166 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
167 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
168 ahci_addr_t *);
169 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
170 ahci_addr_t *);
171 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
172 ahci_addr_t *);
173 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
174 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
175 uint8_t);
176 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
177 uint8_t, int, int *);
178 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
179 uint32_t, uint32_t, uint32_t, uint32_t);
180 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
181 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
182 uint8_t, sata_pkt_t *);
183 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
184 ahci_addr_t *, uint32_t);
185 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
186 uint8_t, uint32_t);
187 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
188 uint8_t, uint32_t);
189 static void ahci_events_handler(void *);
190 static void ahci_watchdog_handler(ahci_ctl_t *);
191
192 static uint_t ahci_intr(caddr_t, caddr_t);
193 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
194 static int ahci_add_intrs(ahci_ctl_t *, int);
195 static void ahci_rem_intrs(ahci_ctl_t *);
196 static void ahci_enable_all_intrs(ahci_ctl_t *);
197 static void ahci_disable_all_intrs(ahci_ctl_t *);
198 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
199 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
200
201 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
202 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
203 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
204 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
205 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
206 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
207 ahci_port_t *, uint8_t);
208 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
209 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
210 uint8_t, uint32_t);
211 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
212 uint8_t, uint32_t);
213 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
214
215 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
216 static int ahci_get_num_implemented_ports(uint32_t);
217 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
218 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
219 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
220 #if AHCI_DEBUG
221 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
222 #endif
223
224
225 /*
226 * DMA attributes for the data buffer
227 *
228 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
229 * does not support 64-bit addressing
230 */
231 static ddi_dma_attr_t buffer_dma_attr = {
232 DMA_ATTR_V0, /* dma_attr_version */
233 0x0ull, /* dma_attr_addr_lo: lowest bus address */
234 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
235 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */
236 0x2ull, /* dma_attr_align: word aligned */
237 1, /* dma_attr_burstsizes */
238 1, /* dma_attr_minxfer */
239 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
240 0xffffffffull, /* dma_attr_seg */
241 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */
242 512, /* dma_attr_granular */
243 0, /* dma_attr_flags */
244 };
245
246 /*
247 * DMA attributes for the rcvd FIS
248 *
249 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
250 * does not support 64-bit addressing
251 */
252 static ddi_dma_attr_t rcvd_fis_dma_attr = {
253 DMA_ATTR_V0, /* dma_attr_version */
254 0x0ull, /* dma_attr_addr_lo: lowest bus address */
255 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
256 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
257 0x100ull, /* dma_attr_align: 256-byte aligned */
258 1, /* dma_attr_burstsizes */
259 1, /* dma_attr_minxfer */
260 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
261 0xffffffffull, /* dma_attr_seg */
262 1, /* dma_attr_sgllen */
263 1, /* dma_attr_granular */
264 0, /* dma_attr_flags */
265 };
266
267 /*
268 * DMA attributes for the command list
269 *
270 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
271 * does not support 64-bit addressing
272 */
273 static ddi_dma_attr_t cmd_list_dma_attr = {
274 DMA_ATTR_V0, /* dma_attr_version */
275 0x0ull, /* dma_attr_addr_lo: lowest bus address */
276 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
277 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
278 0x400ull, /* dma_attr_align: 1K-byte aligned */
279 1, /* dma_attr_burstsizes */
280 1, /* dma_attr_minxfer */
281 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
282 0xffffffffull, /* dma_attr_seg */
283 1, /* dma_attr_sgllen */
284 1, /* dma_attr_granular */
285 0, /* dma_attr_flags */
286 };
287
288 /*
289 * DMA attributes for cmd tables
290 *
291 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
292 * does not support 64-bit addressing
293 */
294 static ddi_dma_attr_t cmd_table_dma_attr = {
295 DMA_ATTR_V0, /* dma_attr_version */
296 0x0ull, /* dma_attr_addr_lo: lowest bus address */
297 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
298 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
299 0x80ull, /* dma_attr_align: 128-byte aligned */
300 1, /* dma_attr_burstsizes */
301 1, /* dma_attr_minxfer */
302 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
303 0xffffffffull, /* dma_attr_seg */
304 1, /* dma_attr_sgllen */
305 1, /* dma_attr_granular */
306 0, /* dma_attr_flags */
307 };
308
309
310 /* Device access attributes */
311 static ddi_device_acc_attr_t accattr = {
312 DDI_DEVICE_ATTR_V1,
313 DDI_STRUCTURE_LE_ACC,
314 DDI_STRICTORDER_ACC,
315 DDI_DEFAULT_ACC
316 };
317
318
319 static struct dev_ops ahcictl_dev_ops = {
320 DEVO_REV, /* devo_rev */
321 0, /* refcnt */
322 ahci_getinfo, /* info */
323 nulldev, /* identify */
324 nulldev, /* probe */
325 ahci_attach, /* attach */
326 ahci_detach, /* detach */
327 nodev, /* no reset */
328 (struct cb_ops *)0, /* driver operations */
329 NULL, /* bus operations */
330 NULL, /* power */
331 ahci_quiesce, /* quiesce */
332 };
333
334 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
335 SATA_TRAN_HOTPLUG_OPS_REV_1,
336 ahci_tran_hotplug_port_activate,
337 ahci_tran_hotplug_port_deactivate
338 };
339
340 extern struct mod_ops mod_driverops;
341
342 static struct modldrv modldrv = {
343 &mod_driverops, /* driverops */
344 ahci_ident, /* short description */
345 &ahcictl_dev_ops, /* driver ops */
346 };
347
348 static struct modlinkage modlinkage = {
349 MODREV_1,
350 &modldrv,
351 NULL
352 };
353
354 /* The following variables are watchdog handler related */
355 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
356 static clock_t ahci_watchdog_tick;
357
358 /*
359 * This static variable indicates the size of command table,
360 * and it's changeable with prdt number, which ahci_dma_prdt_number
361 * indicates.
362 */
363 static size_t ahci_cmd_table_size;
364
365 /*
366 * The below global variables are tunable via /etc/system
367 *
368 * ahci_dma_prdt_number
369 * ahci_msi_enabled
370 * ahci_buf_64bit_dma
371 * ahci_commu_64bit_dma
372 */
373
374 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
375 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
376
377 /* AHCI MSI is tunable */
378 boolean_t ahci_msi_enabled = B_TRUE;
379
380 /*
381 * 64-bit dma addressing for data buffer is tunable
382 *
383 * The variable controls only the below value:
384 * DBAU (upper 32-bits physical address of data block)
385 */
386 boolean_t ahci_buf_64bit_dma = B_TRUE;
387
388 /*
389 * 64-bit dma addressing for communication system descriptors is tunable
390 *
391 * The variable controls the below three values:
392 *
393 * PxCLBU (upper 32-bits for the command list base physical address)
394 * PxFBU (upper 32-bits for the received FIS base physical address)
395 * CTBAU (upper 32-bits of command table base)
396 */
397 boolean_t ahci_commu_64bit_dma = B_TRUE;
398
399 /*
400 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
401 * chipset. If the users want to have a try with 64-bit dma, please change
402 * the below variable value to enable it.
403 */
404 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
405
406 /*
407 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
408 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
409 * please change the below value to enable it.
410 */
411 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
412
413
414 /*
415 * End of global tunable variable definition
416 */
417
418 #if AHCI_DEBUG
419 uint32_t ahci_debug_flags = 0;
420 #else
421 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
422 #endif
423
424
425 #if AHCI_DEBUG
426 /* The following is needed for ahci_log() */
427 static kmutex_t ahci_log_mutex;
428 static char ahci_log_buf[512];
429 #endif
430
431 /* Opaque state pointer initialized by ddi_soft_state_init() */
432 static void *ahci_statep = NULL;
433
434 /*
435 * ahci module initialization.
436 */
437 int
438 _init(void)
439 {
440 int ret;
441
442 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
443 if (ret != 0) {
444 goto err_out;
445 }
446
447 #if AHCI_DEBUG
448 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
449 #endif
450
451 if ((ret = sata_hba_init(&modlinkage)) != 0) {
452 #if AHCI_DEBUG
453 mutex_destroy(&ahci_log_mutex);
454 #endif
455 ddi_soft_state_fini(&ahci_statep);
456 goto err_out;
457 }
458
459 /* watchdog tick */
460 ahci_watchdog_tick = drv_usectohz(
461 (clock_t)ahci_watchdog_timeout * 1000000);
462
463 ret = mod_install(&modlinkage);
464 if (ret != 0) {
465 sata_hba_fini(&modlinkage);
466 #if AHCI_DEBUG
467 mutex_destroy(&ahci_log_mutex);
468 #endif
469 ddi_soft_state_fini(&ahci_statep);
470 goto err_out;
471 }
472
473 return (ret);
474
475 err_out:
476 cmn_err(CE_WARN, "!ahci: Module init failed");
477 return (ret);
478 }
479
480 /*
481 * ahci module uninitialize.
482 */
483 int
484 _fini(void)
485 {
486 int ret;
487
488 ret = mod_remove(&modlinkage);
489 if (ret != 0) {
490 return (ret);
491 }
492
493 /* Remove the resources allocated in _init(). */
494 sata_hba_fini(&modlinkage);
495 #if AHCI_DEBUG
496 mutex_destroy(&ahci_log_mutex);
497 #endif
498 ddi_soft_state_fini(&ahci_statep);
499
500 return (ret);
501 }
502
503 /*
504 * _info entry point
505 */
506 int
507 _info(struct modinfo *modinfop)
508 {
509 return (mod_info(&modlinkage, modinfop));
510 }
511
512 /*
513 * The attach entry point for dev_ops.
514 */
515 static int
516 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
517 {
518 ahci_ctl_t *ahci_ctlp = NULL;
519 int instance = ddi_get_instance(dip);
520 int status;
521 int attach_state;
522 uint32_t cap_status, ahci_version;
523 uint32_t ghc_control;
524 int intr_types;
525 int i;
526 pci_regspec_t *regs;
527 int regs_length;
528 int rnumber;
529 #if AHCI_DEBUG
530 int speed;
531 #endif
532
533 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
534 NULL);
535
536 switch (cmd) {
537 case DDI_ATTACH:
538 break;
539
540 case DDI_RESUME:
541
542 /*
543 * During DDI_RESUME, the hardware state of the device
544 * (power may have been removed from the device) must be
545 * restored, allow pending requests to continue, and
546 * service new requests.
547 */
548 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
549 mutex_enter(&ahci_ctlp->ahcictl_mutex);
550
551 /*
552 * GHC.AE must be set to 1 before any other AHCI register
553 * is accessed
554 */
555 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
556 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
557 ghc_control |= AHCI_HBA_GHC_AE;
558 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
559 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
560
561 /* Restart watch thread */
562 if (ahci_ctlp->ahcictl_timeout_id == 0)
563 ahci_ctlp->ahcictl_timeout_id = timeout(
564 (void (*)(void *))ahci_watchdog_handler,
565 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
566
567 mutex_exit(&ahci_ctlp->ahcictl_mutex);
568
569 /*
570 * Re-initialize the controller and enable the interrupts and
571 * restart all the ports.
572 *
573 * Note that so far we don't support hot-plug during
574 * suspend/resume.
575 */
576 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
577 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
578 "Failed to initialize the controller "
579 "during DDI_RESUME", NULL);
580 return (DDI_FAILURE);
581 }
582
583 mutex_enter(&ahci_ctlp->ahcictl_mutex);
584 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
585 mutex_exit(&ahci_ctlp->ahcictl_mutex);
586
587 return (DDI_SUCCESS);
588
589 default:
590 return (DDI_FAILURE);
591 }
592
593 attach_state = AHCI_ATTACH_STATE_NONE;
594
595 /* Allocate soft state */
596 status = ddi_soft_state_zalloc(ahci_statep, instance);
597 if (status != DDI_SUCCESS) {
598 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
599 instance);
600 goto err_out;
601 }
602
603 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
604 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
605 ahci_ctlp->ahcictl_dip = dip;
606
607 /* Initialize the cport/port mapping */
608 for (i = 0; i < AHCI_MAX_PORTS; i++) {
609 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
610 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
611 }
612
613 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
614
615 /* Initialize FMA properties */
616 ahci_fm_init(ahci_ctlp);
617
618 attach_state |= AHCI_ATTACH_STATE_FMA;
619
620 /*
621 * Now map the AHCI base address; which includes global
622 * registers and port control registers
623 *
624 * According to the spec, the AHCI Base Address is BAR5,
625 * but BAR0-BAR4 are optional, so we need to check which
626 * rnumber is used for BAR5.
627 */
628
629 /*
630 * search through DDI "reg" property for the AHCI register set
631 */
632 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
633 DDI_PROP_DONTPASS, "reg", (int **)®s,
634 (uint_t *)®s_length) != DDI_PROP_SUCCESS) {
635 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
636 instance);
637 goto err_out;
638 }
639
640 /* AHCI Base Address is located at 0x24 offset */
641 for (rnumber = 0; rnumber < regs_length; ++rnumber) {
642 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
643 == AHCI_PCI_RNUM)
644 break;
645 }
646
647 ddi_prop_free(regs);
648
649 if (rnumber == regs_length) {
650 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
651 instance);
652 goto err_out;
653 }
654
655 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
656
657 status = ddi_regs_map_setup(dip,
658 rnumber,
659 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
660 0,
661 0,
662 &accattr,
663 &ahci_ctlp->ahcictl_ahci_acc_handle);
664 if (status != DDI_SUCCESS) {
665 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
666 instance);
667 goto err_out;
668 }
669
670 attach_state |= AHCI_ATTACH_STATE_REG_MAP;
671
672 /*
673 * GHC.AE must be set to 1 before any other AHCI register
674 * is accessed
675 */
676 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
677 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
678 ghc_control |= AHCI_HBA_GHC_AE;
679 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
680 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
681
682 /* Get the AHCI version information */
683 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
684 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
685
686 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
687 (ahci_version & 0xffff0000) >> 16,
688 ((ahci_version & 0x0000ff00) >> 4 |
689 (ahci_version & 0x000000ff)));
690
691 /* We don't support controllers whose versions are lower than 1.0 */
692 if (!(ahci_version & 0xffff0000)) {
693 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
694 "than version 1.0", instance);
695 goto err_out;
696 }
697
698 /* Get the HBA capabilities information */
699 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
700 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
701
702 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
703 cap_status);
704
705 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
706 if (ahci_version >= 0x00010200) {
707 uint32_t cap2_status;
708
709 /* Get the HBA capabilities extended information */
710 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
711 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
712
713 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
714 "hba capabilities extended = 0x%x", cap2_status);
715 }
716
717 #if AHCI_DEBUG
718 /* Get the interface speed supported by the HBA */
719 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
720 if (speed == 0x01) {
721 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
722 "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
723 } else if (speed == 0x10) {
724 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
725 "hba interface speed support: Gen 2 (3 Gbps)", NULL);
726 } else if (speed == 0x11) {
727 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
728 "hba interface speed support: Gen 3 (6 Gbps)", NULL);
729 }
730 #endif
731
732 /* Get the number of command slots supported by the HBA */
733 ahci_ctlp->ahcictl_num_cmd_slots =
734 ((cap_status & AHCI_HBA_CAP_NCS) >>
735 AHCI_HBA_CAP_NCS_SHIFT) + 1;
736
737 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
738 ahci_ctlp->ahcictl_num_cmd_slots);
739
740 /* Get the bit map which indicates ports implemented by the HBA */
741 ahci_ctlp->ahcictl_ports_implemented =
742 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
743 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
744
745 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
746 ahci_ctlp->ahcictl_ports_implemented);
747
748 /* Max port number implemented */
749 ahci_ctlp->ahcictl_num_ports =
750 ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
751
752 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
753 (cap_status & AHCI_HBA_CAP_NP) + 1);
754
755 /* Get the number of implemented ports by the HBA */
756 ahci_ctlp->ahcictl_num_implemented_ports =
757 ahci_get_num_implemented_ports(
758 ahci_ctlp->ahcictl_ports_implemented);
759
760 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
761 "hba number of implemented ports: %d",
762 ahci_ctlp->ahcictl_num_implemented_ports);
763
764 /* Check whether HBA supports 64bit DMA addressing */
765 if (!(cap_status & AHCI_HBA_CAP_S64A)) {
766 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
767 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
768 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
769 "hba does not support 64-bit addressing", NULL);
770 }
771
772 /* Checking for the support of Port Multiplier */
773 if (cap_status & AHCI_HBA_CAP_SPM) {
774 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
775 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
776 "hba supports port multiplier (CBSS)", NULL);
777
778 /* Support FIS-based switching ? */
779 if (cap_status & AHCI_HBA_CAP_FBSS) {
780 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
781 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
782 "hba supports FIS-based switching (FBSS)", NULL);
783 }
784 }
785
786 /* Checking for Support Command List Override */
787 if (cap_status & AHCI_HBA_CAP_SCLO) {
788 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
789 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
790 "hba supports command list override.", NULL);
791 }
792
793 /* Checking for Asynchronous Notification */
794 if (cap_status & AHCI_HBA_CAP_SSNTF) {
795 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
796 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
797 "hba supports asynchronous notification.", NULL);
798 }
799
800 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
801 != DDI_SUCCESS) {
802 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
803 instance);
804 goto err_out;
805 }
806
807 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
808
809 /*
810 * Check the pci configuration space, and set caps. We also
811 * handle the hardware defect in this function.
812 *
813 * For example, force ATI SB600 to use 32-bit dma addressing
814 * since it doesn't support 64-bit dma though its CAP register
815 * declares it support.
816 */
817 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
818 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
819 instance);
820 goto err_out;
821 }
822
823 /*
824 * Disable the whole controller interrupts before adding
825 * interrupt handlers(s).
826 */
827 ahci_disable_all_intrs(ahci_ctlp);
828
829 /* Get supported interrupt types */
830 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
831 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
832 instance);
833 goto err_out;
834 }
835
836 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
837 "ddi_intr_get_supported_types() returned: 0x%x",
838 intr_types);
839
840 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
841 /*
842 * Try MSI first, but fall back to FIXED if failed
843 */
844 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
845 DDI_SUCCESS) {
846 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
847 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
848 "Using MSI interrupt type", NULL);
849 goto intr_done;
850 }
851
852 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
853 "MSI registration failed, "
854 "trying FIXED interrupts", NULL);
855 }
856
857 if (intr_types & DDI_INTR_TYPE_FIXED) {
858 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
859 DDI_SUCCESS) {
860 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
861 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
862 "Using FIXED interrupt type", NULL);
863 goto intr_done;
864 }
865
866 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
867 "FIXED interrupt registration failed", NULL);
868 }
869
870 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
871
872 goto err_out;
873
874 intr_done:
875
876 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
877
878 /* Initialize the controller mutex */
879 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
880 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
881
882 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
883
884 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
885 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
886 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
887 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
888 }
889
890 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
891 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
892 sizeof (ahci_prdt_item_t));
893
894 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
895 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
896 " ahci_cmd_table_size is 0x%x",
897 ahci_dma_prdt_number, ahci_cmd_table_size);
898
899 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
900 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
901 ahci_dma_prdt_number;
902
903 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
904 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
905 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
906 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
907
908 /*
909 * enable 64bit dma for data buffer for SB600 if
910 * sb600_buf_64bit_dma_disable is B_FALSE
911 */
912 if ((ahci_buf_64bit_dma == B_FALSE) ||
913 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
914 !(sb600_buf_64bit_dma_disable == B_FALSE &&
915 ahci_ctlp->ahcictl_venid == 0x1002 &&
916 ahci_ctlp->ahcictl_devid == 0x4380))) {
917 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
918 0xffffffffull;
919 }
920
921 /*
922 * enable 64bit dma for command buffer for SB600/700/710/800
923 * if sbxxx_commu_64bit_dma_disable is B_FALSE
924 */
925 if ((ahci_commu_64bit_dma == B_FALSE) ||
926 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
927 !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
928 ahci_ctlp->ahcictl_venid == 0x1002 &&
929 (ahci_ctlp->ahcictl_devid == 0x4380 ||
930 ahci_ctlp->ahcictl_devid == 0x4391)))) {
931 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
932 0xffffffffull;
933 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
934 0xffffffffull;
935 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
936 0xffffffffull;
937 }
938
939 /* Allocate the ports structure */
940 status = ahci_alloc_ports_state(ahci_ctlp);
941 if (status != AHCI_SUCCESS) {
942 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
943 instance);
944 goto err_out;
945 }
946
947 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
948
949 /*
950 * Initialize the controller and ports.
951 */
952 status = ahci_initialize_controller(ahci_ctlp);
953 if (status != AHCI_SUCCESS) {
954 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
955 instance);
956 goto err_out;
957 }
958
959 attach_state |= AHCI_ATTACH_STATE_HW_INIT;
960
961 /* Start one thread to check packet timeouts */
962 ahci_ctlp->ahcictl_timeout_id = timeout(
963 (void (*)(void *))ahci_watchdog_handler,
964 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
965
966 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
967
968 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
969 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
970 instance);
971 goto err_out;
972 }
973
974 /* Check all handles at the end of the attach operation. */
975 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
976 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
977 instance);
978 goto err_out;
979 }
980
981 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
982
983 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
984
985 return (DDI_SUCCESS);
986
987 err_out:
988 /* FMA message */
989 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
990 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
991
992 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
993 mutex_enter(&ahci_ctlp->ahcictl_mutex);
994 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
995 ahci_ctlp->ahcictl_timeout_id = 0;
996 mutex_exit(&ahci_ctlp->ahcictl_mutex);
997 }
998
999 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1000 ahci_uninitialize_controller(ahci_ctlp);
1001 }
1002
1003 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1004 ahci_dealloc_ports_state(ahci_ctlp);
1005 }
1006
1007 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1008 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1009 }
1010
1011 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1012 ahci_rem_intrs(ahci_ctlp);
1013 }
1014
1015 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1016 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1017 }
1018
1019 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1020 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1021 }
1022
1023 if (attach_state & AHCI_ATTACH_STATE_FMA) {
1024 ahci_fm_fini(ahci_ctlp);
1025 }
1026
1027 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1028 ddi_soft_state_free(ahci_statep, instance);
1029 }
1030
1031 return (DDI_FAILURE);
1032 }
1033
1034 /*
1035 * The detach entry point for dev_ops.
1036 */
1037 static int
1038 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1039 {
1040 ahci_ctl_t *ahci_ctlp;
1041 int instance;
1042 int ret;
1043
1044 instance = ddi_get_instance(dip);
1045 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1046
1047 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1048
1049 switch (cmd) {
1050 case DDI_DETACH:
1051
1052 /* disable the interrupts for an uninterrupted detach */
1053 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1054 ahci_disable_all_intrs(ahci_ctlp);
1055 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1056
1057 /* unregister from the sata framework. */
1058 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1059 if (ret != AHCI_SUCCESS) {
1060 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1061 ahci_enable_all_intrs(ahci_ctlp);
1062 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1063 return (DDI_FAILURE);
1064 }
1065
1066 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1067
1068 /* stop the watchdog handler */
1069 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1070 ahci_ctlp->ahcictl_timeout_id = 0;
1071
1072 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1073
1074 /* uninitialize the controller */
1075 ahci_uninitialize_controller(ahci_ctlp);
1076
1077 /* remove the interrupts */
1078 ahci_rem_intrs(ahci_ctlp);
1079
1080 /* deallocate the ports structures */
1081 ahci_dealloc_ports_state(ahci_ctlp);
1082
1083 /* destroy mutex */
1084 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1085
1086 /* teardown the pci config */
1087 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1088
1089 /* remove the reg maps. */
1090 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1091
1092 /* release fma resource */
1093 ahci_fm_fini(ahci_ctlp);
1094
1095 /* free the soft state. */
1096 ddi_soft_state_free(ahci_statep, instance);
1097
1098 return (DDI_SUCCESS);
1099
1100 case DDI_SUSPEND:
1101
1102 /*
1103 * The steps associated with suspension must include putting
1104 * the underlying device into a quiescent state so that it
1105 * will not generate interrupts or modify or access memory.
1106 */
1107 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1108 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1109 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1110 return (DDI_SUCCESS);
1111 }
1112
1113 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1114
1115 /* stop the watchdog handler */
1116 if (ahci_ctlp->ahcictl_timeout_id) {
1117 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1118 ahci_ctlp->ahcictl_timeout_id = 0;
1119 }
1120
1121 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1122
1123 /*
1124 * drain the taskq
1125 */
1126 ahci_drain_ports_taskq(ahci_ctlp);
1127
1128 /*
1129 * Disable the interrupts and stop all the ports.
1130 */
1131 ahci_uninitialize_controller(ahci_ctlp);
1132
1133 return (DDI_SUCCESS);
1134
1135 default:
1136 return (DDI_FAILURE);
1137 }
1138 }
1139
1140 /*
1141 * The info entry point for dev_ops.
1142 *
1143 */
1144 static int
1145 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1146 void *arg, void **result)
1147 {
1148 #ifndef __lock_lint
1149 _NOTE(ARGUNUSED(dip))
1150 #endif /* __lock_lint */
1151
1152 ahci_ctl_t *ahci_ctlp;
1153 int instance;
1154 dev_t dev;
1155
1156 dev = (dev_t)arg;
1157 instance = getminor(dev);
1158
1159 switch (infocmd) {
1160 case DDI_INFO_DEVT2DEVINFO:
1161 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1162 if (ahci_ctlp != NULL) {
1163 *result = ahci_ctlp->ahcictl_dip;
1164 return (DDI_SUCCESS);
1165 } else {
1166 *result = NULL;
1167 return (DDI_FAILURE);
1168 }
1169 case DDI_INFO_DEVT2INSTANCE:
1170 *(int *)result = instance;
1171 break;
1172 default:
1173 break;
1174 }
1175
1176 return (DDI_SUCCESS);
1177 }
1178
1179 /*
1180 * Registers the ahci with sata framework.
1181 */
1182 static int
1183 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1184 {
1185 struct sata_hba_tran *sata_hba_tran;
1186
1187 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1188 "ahci_register_sata_hba_tran enter", NULL);
1189
1190 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1191
1192 /* Allocate memory for the sata_hba_tran */
1193 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1194
1195 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1196 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1197 sata_hba_tran->sata_tran_hba_dma_attr =
1198 &ahci_ctlp->ahcictl_buffer_dma_attr;
1199
1200 /* Report the number of implemented ports */
1201 sata_hba_tran->sata_tran_hba_num_cports =
1202 ahci_ctlp->ahcictl_num_implemented_ports;
1203
1204 /* Support ATAPI device */
1205 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1206
1207 /* Get the data transfer capability for PIO command by the HBA */
1208 if (cap_status & AHCI_HBA_CAP_PMD) {
1209 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1210 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1211 "DRQ block data transfer for PIO command protocol", NULL);
1212 }
1213
1214 /*
1215 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1216 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1217 * QUEUED (EXT), and SERVICE commands). Queued operations are
1218 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1219 * QUEUED commands when the HBA and device support native command
1220 * queuing(NCQ).
1221 *
1222 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1223 * CAP register of the HBA indicates NCQ is supported.
1224 *
1225 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1226 * set because the previous register content of PxCI can be re-written
1227 * in the register write.
1228 */
1229 if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1230 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1231 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1232 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1233 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1234 "Command Queuing", NULL);
1235 }
1236
1237 /* Support port multiplier? */
1238 if (cap_status & AHCI_HBA_CAP_SPM) {
1239 sata_hba_tran->sata_tran_hba_features_support |=
1240 SATA_CTLF_PORT_MULTIPLIER;
1241
1242 /* Support FIS-based switching for port multiplier? */
1243 if (cap_status & AHCI_HBA_CAP_FBSS) {
1244 sata_hba_tran->sata_tran_hba_features_support |=
1245 SATA_CTLF_PMULT_FBS;
1246 }
1247 }
1248
1249 /* Report the number of command slots */
1250 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1251
1252 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1253 sata_hba_tran->sata_tran_start = ahci_tran_start;
1254 sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1255 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1256 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1257 #ifdef __lock_lint
1258 sata_hba_tran->sata_tran_selftest = ahci_selftest;
1259 #endif
1260 /*
1261 * When SATA framework adds support for pwrmgt the
1262 * pwrmgt_ops needs to be updated
1263 */
1264 sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1265 sata_hba_tran->sata_tran_ioctl = NULL;
1266
1267 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1268
1269 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1270
1271 /* Attach it to SATA framework */
1272 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1273 != DDI_SUCCESS) {
1274 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1275 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1276 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1277 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1278 return (AHCI_FAILURE);
1279 }
1280
1281 return (AHCI_SUCCESS);
1282 }
1283
1284 /*
1285 * Unregisters the ahci with sata framework.
1286 */
1287 static int
1288 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1289 {
1290 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1291 "ahci_unregister_sata_hba_tran enter", NULL);
1292
1293 /* Detach from the SATA framework. */
1294 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1295 DDI_SUCCESS) {
1296 return (AHCI_FAILURE);
1297 }
1298
1299 /* Deallocate sata_hba_tran. */
1300 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1301 sizeof (sata_hba_tran_t));
1302
1303 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1304 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1305 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1306
1307 return (AHCI_SUCCESS);
1308 }
1309
1310 #define SET_PORTSTR(str, addrp) \
1311 if (AHCI_ADDR_IS_PORT(addrp)) \
1312 (void) sprintf((str), "%d", (addrp)->aa_port); \
1313 else if (AHCI_ADDR_IS_PMULT(addrp)) \
1314 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \
1315 else \
1316 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \
1317 (addrp)->aa_pmport);
1318
1319 /*
1320 * ahci_tran_probe_port is called by SATA framework. It returns port state,
1321 * port status registers and an attached device type via sata_device
1322 * structure.
1323 *
1324 * We return the cached information from a previous hardware probe. The
1325 * actual hardware probing itself was done either from within
1326 * ahci_initialize_controller() during the driver attach or from a phy
1327 * ready change interrupt handler.
1328 */
1329 static int
1330 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1331 {
1332 ahci_ctl_t *ahci_ctlp;
1333 ahci_port_t *ahci_portp;
1334 ahci_addr_t addr, pmult_addr;
1335 uint8_t cport = sd->satadev_addr.cport;
1336 char portstr[10];
1337 uint8_t device_type;
1338 uint32_t port_state;
1339 uint8_t port;
1340 int rval = SATA_SUCCESS, rval_init;
1341
1342 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1343 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1344
1345 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1346
1347 mutex_enter(&ahci_portp->ahciport_mutex);
1348
1349 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1350 ASSERT(AHCI_ADDR_IS_VALID(&addr));
1351 SET_PORTSTR(portstr, &addr);
1352
1353 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1354 "ahci_tran_probe_port enter: port %s", portstr);
1355
1356 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1357 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1358 ahci_portp->ahciport_pmult_info == NULL)) {
1359 /* port mutliplier is removed. */
1360 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1361 "ahci_tran_probe_port: "
1362 "pmult is removed from port %s", portstr);
1363 mutex_exit(&ahci_portp->ahciport_mutex);
1364 return (SATA_FAILURE);
1365 }
1366
1367 /*
1368 * The sata_device may refer to
1369 * 1. A controller port.
1370 * A controller port should be ready here.
1371 * 2. A port multiplier.
1372 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1373 * it and register the port multiplier to the framework.
1374 * SATA_ADDR_PMULT - check the status of all its device ports.
1375 * 3. A port multiplier port.
1376 * If it has not been initialized, initialized it.
1377 *
1378 * A port multiplier or a port multiplier port may require some
1379 * initialization because we cannot do these time-consuming jobs in an
1380 * interrupt context.
1381 */
1382 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1383 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1384 /* Initialize registers on a port multiplier */
1385 rval_init = ahci_initialize_pmult(ahci_ctlp,
1386 ahci_portp, &pmult_addr, sd);
1387 if (rval_init != AHCI_SUCCESS) {
1388 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1389 "ahci_tran_probe_port: "
1390 "pmult initialization failed.", NULL);
1391 mutex_exit(&ahci_portp->ahciport_mutex);
1392 return (SATA_FAILURE);
1393 }
1394 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1395 /* Check pmports hotplug events */
1396 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1397 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1398 SATA_ADDR_DPMPORT)) {
1399 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1400 &addr, sd) != AHCI_SUCCESS) {
1401 rval = SATA_FAILURE;
1402 goto out;
1403 }
1404 }
1405
1406 /* Update port state and device type */
1407 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1408
1409 switch (port_state) {
1410
1411 case SATA_PSTATE_FAILED:
1412 sd->satadev_state = SATA_PSTATE_FAILED;
1413 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1414 "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1415 goto out;
1416
1417 case SATA_PSTATE_SHUTDOWN:
1418 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1419 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1420 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1421 goto out;
1422
1423 case SATA_PSTATE_PWROFF:
1424 sd->satadev_state = SATA_PSTATE_PWROFF;
1425 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1426 "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1427 goto out;
1428
1429 case SATA_PSTATE_PWRON:
1430 sd->satadev_state = SATA_PSTATE_PWRON;
1431 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1432 "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1433 break;
1434
1435 default:
1436 sd->satadev_state = port_state;
1437 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1438 "ahci_tran_probe_port: port %s PORT NORMAL %x",
1439 portstr, port_state);
1440 break;
1441 }
1442
1443 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1444
1445 switch (device_type) {
1446
1447 case SATA_DTYPE_ATADISK:
1448 sd->satadev_type = SATA_DTYPE_ATADISK;
1449 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1450 "ahci_tran_probe_port: port %s DISK found", portstr);
1451 break;
1452
1453 case SATA_DTYPE_ATAPI:
1454 /*
1455 * HBA driver only knows it's an ATAPI device, and don't know
1456 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1457 * type need to be determined by checking IDENTIFY PACKET
1458 * DEVICE data
1459 */
1460 sd->satadev_type = SATA_DTYPE_ATAPI;
1461 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1462 "ahci_tran_probe_port: port %s ATAPI found", portstr);
1463 break;
1464
1465 case SATA_DTYPE_PMULT:
1466 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1467 sd->satadev_type = SATA_DTYPE_PMULT;
1468
1469 /* Update the number of pmports. */
1470 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1471 sd->satadev_add_info = ahci_portp->
1472 ahciport_pmult_info->ahcipmi_num_dev_ports;
1473
1474 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1475 "ahci_tran_probe_port: port %s Port Multiplier found",
1476 portstr);
1477 break;
1478
1479 case SATA_DTYPE_UNKNOWN:
1480 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1481 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1482 "ahci_tran_probe_port: port %s Unknown device found",
1483 portstr);
1484 break;
1485
1486 default:
1487 /* we don't support any other device types */
1488 sd->satadev_type = SATA_DTYPE_NONE;
1489 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1490 "ahci_tran_probe_port: port %s No device found", portstr);
1491 break;
1492 }
1493
1494 out:
1495 /* Register update only fails while probing a pmult/pmport */
1496 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1497 ahci_update_sata_registers(ahci_ctlp, port, sd);
1498 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1499 if (port_state & SATA_STATE_READY)
1500 if (ahci_update_pmult_pscr(ahci_ctlp,
1501 &addr, sd) != AHCI_SUCCESS)
1502 rval = SATA_FAILURE;
1503 }
1504
1505 /* Check handles for the sata registers access */
1506 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1507 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1508 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1509 DDI_SERVICE_UNAFFECTED);
1510 rval = SATA_FAILURE;
1511 }
1512
1513 mutex_exit(&ahci_portp->ahciport_mutex);
1514 return (rval);
1515 }
1516
1517 /*
1518 * There are four operation modes in sata framework:
1519 * SATA_OPMODE_INTERRUPTS
1520 * SATA_OPMODE_POLLING
1521 * SATA_OPMODE_ASYNCH
1522 * SATA_OPMODE_SYNCH
1523 *
1524 * Their combined meanings as following:
1525 *
1526 * SATA_OPMODE_SYNCH
1527 * The command has to be completed before sata_tran_start functions returns.
1528 * Either interrupts or polling could be used - it's up to the driver.
1529 * Mode used currently for internal, sata-module initiated operations.
1530 *
1531 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1532 * It is the same as the one above.
1533 *
1534 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1535 * The command has to be completed before sata_tran_start function returns.
1536 * No interrupt used, polling only. This should be the mode used for scsi
1537 * packets with FLAG_NOINTR.
1538 *
1539 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1540 * The command may be queued (callback function specified). Interrupts could
1541 * be used. It's normal operation mode.
1542 */
1543 /*
1544 * Called by sata framework to transport a sata packet down stream.
1545 */
1546 static int
1547 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1548 {
1549 ahci_ctl_t *ahci_ctlp;
1550 ahci_port_t *ahci_portp;
1551 ahci_addr_t addr;
1552 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1553 uint8_t port;
1554 char portstr[10];
1555
1556 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1557 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1558
1559 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1560 "ahci_tran_start enter: cport %d satapkt 0x%p",
1561 cport, (void *)spkt);
1562
1563 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1564
1565 mutex_enter(&ahci_portp->ahciport_mutex);
1566 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1567 SET_PORTSTR(portstr, &addr);
1568
1569 /* Sanity check */
1570 if (AHCI_ADDR_IS_PMPORT(&addr)) {
1571 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1572 ahci_portp->ahciport_pmult_info == NULL) {
1573
1574 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1575 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1576 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1577 ahci_update_sata_registers(ahci_ctlp, port,
1578 &spkt->satapkt_device);
1579 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1580 "ahci_tran_start returning PORT_ERROR while "
1581 "pmult removed: port: %s", portstr);
1582 mutex_exit(&ahci_portp->ahciport_mutex);
1583 return (SATA_TRAN_PORT_ERROR);
1584 }
1585
1586 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1587 SATA_STATE_READY)) {
1588 if (!ddi_in_panic() ||
1589 ahci_initialize_pmport(ahci_ctlp,
1590 ahci_portp, &addr) != AHCI_SUCCESS) {
1591 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1592 spkt->satapkt_device.satadev_type =
1593 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1594 spkt->satapkt_device.satadev_state =
1595 AHCIPORT_GET_STATE(ahci_portp, &addr);
1596 ahci_update_sata_registers(ahci_ctlp, port,
1597 &spkt->satapkt_device);
1598 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1599 "ahci_tran_start returning PORT_ERROR "
1600 "while sub-link is not initialized "
1601 "at port: %s", portstr);
1602 mutex_exit(&ahci_portp->ahciport_mutex);
1603 return (SATA_TRAN_PORT_ERROR);
1604 }
1605 }
1606 }
1607
1608 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1609 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1610 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1611 /*
1612 * In case the target driver would send the packet before
1613 * sata framework can have the opportunity to process those
1614 * event reports.
1615 */
1616 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1617 spkt->satapkt_device.satadev_state =
1618 ahci_portp->ahciport_port_state;
1619 ahci_update_sata_registers(ahci_ctlp, port,
1620 &spkt->satapkt_device);
1621 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1622 "ahci_tran_start returning PORT_ERROR while "
1623 "port in FAILED/SHUTDOWN/PWROFF state: "
1624 "port: %s", portstr);
1625 mutex_exit(&ahci_portp->ahciport_mutex);
1626 return (SATA_TRAN_PORT_ERROR);
1627 }
1628
1629 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1630 /*
1631 * ahci_intr_phyrdy_change() may have rendered it to
1632 * SATA_DTYPE_NONE.
1633 */
1634 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1635 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1636 spkt->satapkt_device.satadev_state =
1637 ahci_portp->ahciport_port_state;
1638 ahci_update_sata_registers(ahci_ctlp, port,
1639 &spkt->satapkt_device);
1640 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1641 "ahci_tran_start returning PORT_ERROR while "
1642 "no device attached: port: %s", portstr);
1643 mutex_exit(&ahci_portp->ahciport_mutex);
1644 return (SATA_TRAN_PORT_ERROR);
1645 }
1646
1647 /* R/W PMULT command will occupy the whole HBA port */
1648 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1649 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1650 "ahci_tran_start returning BUSY while "
1651 "executing READ/WRITE PORT-MULT command: "
1652 "port: %s", portstr);
1653 spkt->satapkt_reason = SATA_PKT_BUSY;
1654 mutex_exit(&ahci_portp->ahciport_mutex);
1655 return (SATA_TRAN_BUSY);
1656 }
1657
1658 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1659 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1660 "ahci_tran_start returning BUSY while "
1661 "hot-plug in progress: port: %s", portstr);
1662 spkt->satapkt_reason = SATA_PKT_BUSY;
1663 mutex_exit(&ahci_portp->ahciport_mutex);
1664 return (SATA_TRAN_BUSY);
1665 }
1666
1667 /*
1668 * SATA HBA driver should remember that a device was reset and it
1669 * is supposed to reject any packets which do not specify either
1670 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1671 *
1672 * This is to prevent a race condition when a device was arbitrarily
1673 * reset by the HBA driver (and lost it's setting) and a target
1674 * driver sending some commands to a device before the sata framework
1675 * has a chance to restore the device setting (such as cache enable/
1676 * disable or other resettable stuff).
1677 */
1678 /*
1679 * It is unnecessary to use specific flags to indicate
1680 * reset_in_progress for a pmport. While mopping, all command will be
1681 * mopped so that the entire HBA port is being dealt as a single
1682 * object.
1683 */
1684 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1685 ahci_portp->ahciport_reset_in_progress = 0;
1686 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1687 "ahci_tran_start [CLEAR] the "
1688 "reset_in_progress for port: %d", port);
1689 }
1690
1691 if (ahci_portp->ahciport_reset_in_progress &&
1692 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1693 ! ddi_in_panic()) {
1694 spkt->satapkt_reason = SATA_PKT_BUSY;
1695 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1696 "ahci_tran_start returning BUSY while "
1697 "reset in progress: port: %d", port);
1698 mutex_exit(&ahci_portp->ahciport_mutex);
1699 return (SATA_TRAN_BUSY);
1700 }
1701
1702 #ifdef AHCI_DEBUG
1703 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1704 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1705 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1706 spkt, port);
1707 }
1708 #endif
1709
1710 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1711 spkt->satapkt_reason = SATA_PKT_BUSY;
1712 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1713 "ahci_tran_start returning BUSY while "
1714 "mopping in progress: port: %d", port);
1715 mutex_exit(&ahci_portp->ahciport_mutex);
1716 return (SATA_TRAN_BUSY);
1717 }
1718
1719 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1720 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1721 DDI_SERVICE_UNAFFECTED);
1722 mutex_exit(&ahci_portp->ahciport_mutex);
1723 return (SATA_TRAN_BUSY);
1724 }
1725
1726 if (spkt->satapkt_op_mode &
1727 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1728 /*
1729 * If a SYNC command to be executed in interrupt context,
1730 * bounce it back to sata module.
1731 */
1732 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1733 servicing_interrupt()) {
1734 spkt->satapkt_reason = SATA_PKT_BUSY;
1735 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1736 "ahci_tran_start returning BUSY while "
1737 "sending SYNC mode under interrupt context: "
1738 "port : %d", port);
1739 mutex_exit(&ahci_portp->ahciport_mutex);
1740 return (SATA_TRAN_BUSY);
1741 }
1742
1743 /* We need to do the sync start now */
1744 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1745 spkt) == AHCI_FAILURE) {
1746 goto fail_out;
1747 }
1748 } else {
1749 /* Async start, using interrupt */
1750 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1751 == AHCI_FAILURE) {
1752 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1753 goto fail_out;
1754 }
1755 }
1756
1757 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1758 "sata tran accepted: port %s", portstr);
1759
1760 mutex_exit(&ahci_portp->ahciport_mutex);
1761 return (SATA_TRAN_ACCEPTED);
1762
1763 fail_out:
1764 /*
1765 * Failed to deliver packet to the controller.
1766 * Check if it's caused by invalid handles.
1767 */
1768 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1769 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1770 spkt->satapkt_device.satadev_type =
1771 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1772 spkt->satapkt_device.satadev_state =
1773 AHCIPORT_GET_STATE(ahci_portp, &addr);
1774 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1775 mutex_exit(&ahci_portp->ahciport_mutex);
1776 return (SATA_TRAN_PORT_ERROR);
1777 }
1778
1779 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1780 "return QUEUE_FULL: port %d", port);
1781 mutex_exit(&ahci_portp->ahciport_mutex);
1782 return (SATA_TRAN_QUEUE_FULL);
1783 }
1784
1785 /*
1786 * SATA_OPMODE_SYNCH flag is set
1787 *
1788 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1789 * without interrupt, otherwise we can still use the interrupt.
1790 */
1791 static int
1792 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1793 ahci_addr_t *addrp, sata_pkt_t *spkt)
1794 {
1795 int pkt_timeout_ticks;
1796 uint32_t timeout_tags;
1797 int rval;
1798 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1799 uint8_t port = addrp->aa_port;
1800
1801 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1802
1803 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1804 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1805
1806 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1807 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1808 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1809 addrp, spkt)) == AHCI_FAILURE) {
1810 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1811 return (rval);
1812 }
1813
1814 pkt_timeout_ticks =
1815 drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1816
1817 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1818 mutex_exit(&ahci_portp->ahciport_mutex);
1819
1820 /* Simulate the interrupt */
1821 ahci_port_intr(ahci_ctlp, ahci_portp, port);
1822
1823 drv_usecwait(AHCI_10MS_USECS);
1824
1825 mutex_enter(&ahci_portp->ahciport_mutex);
1826 pkt_timeout_ticks -= AHCI_10MS_TICKS;
1827 if (pkt_timeout_ticks < 0) {
1828 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1829 "port %d satapkt 0x%p timed out\n",
1830 instance, port, (void *)spkt);
1831 timeout_tags = (0x1 << rval);
1832 mutex_exit(&ahci_portp->ahciport_mutex);
1833 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1834 port, timeout_tags);
1835 mutex_enter(&ahci_portp->ahciport_mutex);
1836 }
1837 }
1838 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1839 return (AHCI_SUCCESS);
1840
1841 } else {
1842 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1843 addrp, spkt)) == AHCI_FAILURE)
1844 return (rval);
1845
1846 #if AHCI_DEBUG
1847 /*
1848 * Note that the driver always uses the slot 0 to deliver
1849 * REQUEST SENSE or READ LOG EXT command
1850 */
1851 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1852 ASSERT(rval == 0);
1853 #endif
1854
1855 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1856 cv_wait(&ahci_portp->ahciport_cv,
1857 &ahci_portp->ahciport_mutex);
1858
1859 return (AHCI_SUCCESS);
1860 }
1861 }
1862
1863 /*
1864 * Searches for and claims a free command slot.
1865 *
1866 * Returns value:
1867 *
1868 * AHCI_FAILURE returned only if
1869 * 1. No empty slot left
1870 * 2. Non-queued command requested while queued command(s) is outstanding
1871 * 3. Queued command requested while non-queued command(s) is outstanding
1872 * 4. HBA doesn't support multiple-use of command list while already a
1873 * non-queued command is oustanding
1874 * 5. Queued command requested while some queued command(s) has been
1875 * outstanding on a different port multiplier port. (AHCI spec 1.2,
1876 * 9.1.2)
1877 *
1878 * claimed slot number returned if succeeded
1879 *
1880 * NOTE: it will always return slot 0 for following commands to simplify the
1881 * algorithm.
1882 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process
1883 * 2. READ/WRITE PORTMULT command
1884 */
1885 static int
1886 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1887 ahci_addr_t *addrp, int command_type)
1888 {
1889 uint32_t port_cmd_issue;
1890 uint32_t free_slots;
1891 int slot;
1892
1893 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1894
1895 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1896 "ahciport_pending_tags = 0x%x "
1897 "ahciport_pending_ncq_tags = 0x%x",
1898 ahci_portp->ahciport_pending_tags,
1899 ahci_portp->ahciport_pending_ncq_tags);
1900
1901 /*
1902 * According to the AHCI spec, system software is responsible to
1903 * ensure that queued and non-queued commands are not mixed in
1904 * the command list.
1905 */
1906 if (command_type == AHCI_NON_NCQ_CMD) {
1907 /* Non-NCQ command request */
1908 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1909 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1910 "ahci_claim_free_slot: there is still pending "
1911 "queued command(s) in the command list, "
1912 "so no available slot for the non-queued "
1913 "command", NULL);
1914 return (AHCI_FAILURE);
1915 }
1916 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1917 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1918 "ahci_claim_free_slot: there is still pending "
1919 "read/write port-mult command(s) in command list, "
1920 "so no available slot for the non-queued command",
1921 NULL);
1922 return (AHCI_FAILURE);
1923 }
1924 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
1925 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1926 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1927 "ahci_claim_free_slot: HBA cannot support multiple-"
1928 "use of the command list for non-queued commands",
1929 NULL);
1930 return (AHCI_FAILURE);
1931 }
1932 free_slots = (~ahci_portp->ahciport_pending_tags) &
1933 AHCI_SLOT_MASK(ahci_ctlp);
1934 } else if (command_type == AHCI_NCQ_CMD) {
1935 /* NCQ command request */
1936 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1937 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1938 "ahci_claim_free_slot: there is still pending "
1939 "non-queued command(s) in the command list, "
1940 "so no available slot for the queued command",
1941 NULL);
1942 return (AHCI_FAILURE);
1943 }
1944
1945 /*
1946 * NCQ commands cannot be sent to different port multiplier
1947 * ports in Command-Based Switching mode
1948 */
1949 /*
1950 * NOTE: In Command-Based Switching mode, AHCI controller
1951 * usually reports a 'Handshake Error' when multiple NCQ
1952 * commands are outstanding simultaneously.
1953 */
1954 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
1955 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1956 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
1957 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
1958 AHCIPORT_NCQ_PMPORT(ahci_portp) !=
1959 addrp->aa_pmport) {
1960 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1961 "ahci_claim_free_slot: there is still "
1962 "pending queued command(s) in the "
1963 "command list for another Port Multiplier "
1964 "port, so no available slot.", NULL);
1965 return (AHCI_FAILURE);
1966 }
1967 }
1968
1969 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
1970 AHCI_NCQ_SLOT_MASK(ahci_portp);
1971 } else if (command_type == AHCI_ERR_RETRI_CMD) {
1972 /* Error retrieval command request */
1973 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1974 "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
1975 "SENSE or READ LOG EXT command", NULL);
1976 slot = 0;
1977 goto out;
1978 } else if (command_type == AHCI_RDWR_PMULT_CMD) {
1979 /*
1980 * An extra check on PxCI. Sometimes PxCI bits may not be
1981 * cleared during hot-plug or error recovery process.
1982 */
1983 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1984 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
1985
1986 if (port_cmd_issue != 0) {
1987 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1988 "ahci_claim_free_slot: there is still pending "
1989 "command(s) in command list (0x%x/0x%x, PxCI %x),"
1990 "so no available slot for R/W PMULT command.",
1991 NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
1992 NCQ_CMD_IN_PROGRESS(ahci_portp),
1993 port_cmd_issue);
1994 return (AHCI_FAILURE);
1995 }
1996
1997 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1998 "ahci_claim_free_slot: slot 0 is allocated for "
1999 "READ/WRITE PORTMULT command", NULL);
2000 slot = 0;
2001 goto out;
2002 }
2003
2004 slot = ddi_ffs(free_slots) - 1;
2005 if (slot == -1) {
2006 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2007 "ahci_claim_free_slot: no empty slots", NULL);
2008 return (AHCI_FAILURE);
2009 }
2010
2011 /*
2012 * According to the AHCI spec, to allow a simple mechanism for the
2013 * HBA to map command list slots to queue entries, software must
2014 * match the tag number it uses to the slot it is placing the command
2015 * in. For example, if a queued command is placed in slot 5, the tag
2016 * for that command must be 5.
2017 */
2018 if (command_type == AHCI_NCQ_CMD) {
2019 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2020 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2021 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2022 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2023 }
2024 }
2025
2026 ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2027
2028 out:
2029 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2030 "ahci_claim_free_slot: found slot: 0x%x", slot);
2031
2032 return (slot);
2033 }
2034
2035 /*
2036 * Builds the Command Table for the sata packet and delivers it to controller.
2037 *
2038 * Returns:
2039 * slot number if we can obtain a slot successfully
2040 * otherwise, return AHCI_FAILURE
2041 */
2042 static int
2043 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2044 ahci_addr_t *addrp, sata_pkt_t *spkt)
2045 {
2046 int cmd_slot;
2047 sata_cmd_t *scmd;
2048 ahci_fis_h2d_register_t *h2d_register_fisp;
2049 ahci_cmd_table_t *cmd_table;
2050 ahci_cmd_header_t *cmd_header;
2051 int ncookies;
2052 int i;
2053 int command_type = AHCI_NON_NCQ_CMD;
2054 int ncq_qdepth;
2055 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2056 uint8_t port, pmport;
2057 #if AHCI_DEBUG
2058 uint32_t *ptr;
2059 uint8_t *ptr2;
2060 #endif
2061
2062 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2063
2064 port = addrp->aa_port;
2065 pmport = addrp->aa_pmport;
2066
2067 spkt->satapkt_reason = SATA_PKT_BUSY;
2068
2069 scmd = &spkt->satapkt_cmd;
2070
2071 /* Check if the command is a NCQ command */
2072 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2073 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2074 command_type = AHCI_NCQ_CMD;
2075
2076 /*
2077 * When NCQ is support, system software must determine the
2078 * maximum tag allowed by the device and the HBA, and it
2079 * must use a value not beyond of the lower bound of the two.
2080 *
2081 * Sata module is going to calculate the qdepth and send
2082 * down to HBA driver via sata_cmd.
2083 */
2084 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2085
2086 /*
2087 * At the moment, the driver doesn't support the dynamic
2088 * setting of the maximum ncq depth, and the value can be
2089 * set either during the attach or after hot-plug insertion.
2090 */
2091 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2092 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2093 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2094 "ahci_deliver_satapkt: port %d the max tags for "
2095 "NCQ command is %d", port, ncq_qdepth);
2096 } else {
2097 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2098 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2099 " port %d the max tag for NCQ command is "
2100 "requested to change from %d to %d, at the"
2101 " moment the driver doesn't support the "
2102 "dynamic change so it's going to "
2103 "still use the previous tag value",
2104 instance, port,
2105 ahci_portp->ahciport_max_ncq_tags,
2106 ncq_qdepth);
2107 }
2108 }
2109 }
2110
2111 /* Check if the command is an error retrieval command */
2112 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2113 command_type = AHCI_ERR_RETRI_CMD;
2114
2115 /* Check if the command is an read/write pmult command */
2116 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2117 command_type = AHCI_RDWR_PMULT_CMD;
2118
2119 /* Check if there is an empty command slot */
2120 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2121 addrp, command_type);
2122 if (cmd_slot == AHCI_FAILURE) {
2123 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2124 return (AHCI_FAILURE);
2125 }
2126
2127 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2128 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2129 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2130 cmd_slot, port, (void *)spkt);
2131
2132 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2133 bzero((void *)cmd_table, ahci_cmd_table_size);
2134
2135 /* For data transfer operations, it is the H2D Register FIS */
2136 h2d_register_fisp =
2137 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2138
2139 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2140
2141 /*
2142 * PMP field only make sense when target is a port multiplier or a
2143 * device behind a port multiplier. Otherwise should set it to 0.
2144 */
2145 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2146 SET_FIS_PMP(h2d_register_fisp, pmport);
2147
2148 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2149 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2150 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2151 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2152
2153 switch (scmd->satacmd_addr_type) {
2154
2155 case 0:
2156 /*
2157 * satacmd_addr_type will be 0 for the commands below:
2158 * ATAPI command
2159 * SATAC_IDLE_IM
2160 * SATAC_STANDBY_IM
2161 * SATAC_DOWNLOAD_MICROCODE
2162 * SATAC_FLUSH_CACHE
2163 * SATAC_SET_FEATURES
2164 * SATAC_SMART
2165 * SATAC_ID_PACKET_DEVICE
2166 * SATAC_ID_DEVICE
2167 * SATAC_READ_PORTMULT
2168 * SATAC_WRITE_PORTMULT
2169 */
2170 /* FALLTHRU */
2171
2172 case ATA_ADDR_LBA:
2173 /* FALLTHRU */
2174
2175 case ATA_ADDR_LBA28:
2176 /* LBA[7:0] */
2177 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2178
2179 /* LBA[15:8] */
2180 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2181
2182 /* LBA[23:16] */
2183 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2184
2185 /* LBA [27:24] (also called dev_head) */
2186 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2187
2188 break;
2189
2190 case ATA_ADDR_LBA48:
2191 /* LBA[7:0] */
2192 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2193
2194 /* LBA[15:8] */
2195 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2196
2197 /* LBA[23:16] */
2198 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2199
2200 /* LBA [31:24] */
2201 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2202 scmd->satacmd_lba_low_msb);
2203
2204 /* LBA [39:32] */
2205 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2206 scmd->satacmd_lba_mid_msb);
2207
2208 /* LBA [47:40] */
2209 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2210 scmd->satacmd_lba_high_msb);
2211
2212 /* Set dev_head */
2213 SET_FIS_DEV_HEAD(h2d_register_fisp,
2214 scmd->satacmd_device_reg);
2215
2216 /* Set the extended sector count and features */
2217 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2218 scmd->satacmd_sec_count_msb);
2219 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2220 scmd->satacmd_features_reg_ext);
2221 break;
2222 }
2223
2224 /*
2225 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2226 * filled into features field, and sector count 8:15 is filled into
2227 * features (exp) field. The hba driver doesn't need to anything
2228 * special with regard to this, since sata framework has already
2229 * done so.
2230 *
2231 * However the driver needs to make sure TAG is filled into sector
2232 * field.
2233 */
2234 if (command_type == AHCI_NCQ_CMD) {
2235 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2236 (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2237 }
2238
2239 ncookies = scmd->satacmd_num_dma_cookies;
2240 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2241 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2242 ncookies, ahci_dma_prdt_number);
2243
2244 ASSERT(ncookies <= ahci_dma_prdt_number);
2245 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2246
2247 /* *** now fill the scatter gather list ******* */
2248 for (i = 0; i < ncookies; i++) {
2249 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2250 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2251 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2252 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2253 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2254 scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2255 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2256 scmd->satacmd_dma_cookie_list[i].dmac_size;
2257 }
2258
2259 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2260 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2261 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2262
2263 /* The ACMD field is filled in for ATAPI command */
2264 if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2265 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2266 SATA_ATAPI_MAX_CDB_LEN);
2267 }
2268
2269 /* Set Command Header in Command List */
2270 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2271 BZERO_DESCR_INFO(cmd_header);
2272 BZERO_PRD_BYTE_COUNT(cmd_header);
2273
2274 /* Set the number of entries in the PRD table */
2275 SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2276
2277 /* Set the length of the command in the CFIS area */
2278 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2279
2280 /*
2281 * PMP field only make sense when target is a port multiplier or a
2282 * device behind a port multiplier. Otherwise should set it to 0.
2283 */
2284 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2285 SET_PORT_MULTI_PORT(cmd_header, pmport);
2286
2287 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2288 "sata_data_direction = 0x%x",
2289 scmd->satacmd_flags.sata_data_direction);
2290
2291 /* Set A bit if it is an ATAPI command */
2292 if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2293 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2294
2295 /* Set W bit if data is going to the device */
2296 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2297 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2298
2299 /*
2300 * Set the prefetchable bit - this bit is only valid if the PRDTL
2301 * field is non-zero or the ATAPI 'A' bit is set in the command
2302 * header. This bit cannot be set when using native command
2303 * queuing commands or when using FIS-based switching with a Port
2304 * multiplier.
2305 */
2306 if (command_type != AHCI_NCQ_CMD)
2307 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2308
2309 /*
2310 * Now remember the sata packet in ahciport_slot_pkts[].
2311 * Error retrieval command and r/w port multiplier command will
2312 * be stored specifically for each port.
2313 */
2314 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2315 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2316 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2317
2318 /*
2319 * Keep the timeout value
2320 */
2321 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2322
2323 /*
2324 * If the intial timout is less than 1 tick, then make it longer by
2325 * 1 tick to avoid immediate timeout
2326 */
2327 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2328 ahci_watchdog_timeout)
2329 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2330 ahci_watchdog_timeout;
2331
2332 #if AHCI_DEBUG
2333 if (ahci_debug_flags & AHCIDBG_ATACMD &&
2334 scmd->satacmd_cmd_reg != SATAC_PACKET ||
2335 ahci_debug_flags & AHCIDBG_ATAPICMD &&
2336 scmd->satacmd_cmd_reg == SATAC_PACKET) {
2337
2338 /* Dump the command header and table */
2339 ahci_log(ahci_ctlp, CE_WARN, "\n");
2340 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2341 "0x%p cmd_reg 0x%x port %d", spkt,
2342 scmd->satacmd_cmd_reg, port);
2343 ptr = (uint32_t *)cmd_header;
2344 ahci_log(ahci_ctlp, CE_WARN,
2345 " Command Header:%8x %8x %8x %8x",
2346 ptr[0], ptr[1], ptr[2], ptr[3]);
2347
2348 /* Dump the H2D register FIS */
2349 ptr = (uint32_t *)h2d_register_fisp;
2350 ahci_log(ahci_ctlp, CE_WARN,
2351 " Command FIS: %8x %8x %8x %8x",
2352 ptr[0], ptr[1], ptr[2], ptr[3]);
2353
2354 /* Dump the ACMD register FIS */
2355 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2356 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2357 if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2358 ahci_log(ahci_ctlp, CE_WARN,
2359 " ATAPI command: %2x %2x %2x %2x "
2360 "%2x %2x %2x %2x",
2361 ptr2[8 * i], ptr2[8 * i + 1],
2362 ptr2[8 * i + 2], ptr2[8 * i + 3],
2363 ptr2[8 * i + 4], ptr2[8 * i + 5],
2364 ptr2[8 * i + 6], ptr2[8 * i + 7]);
2365
2366 /* Dump the PRDT */
2367 for (i = 0; i < ncookies; i++) {
2368 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2369 ahci_log(ahci_ctlp, CE_WARN,
2370 " Cookie %d: %8x %8x %8x %8x",
2371 i, ptr[0], ptr[1], ptr[2], ptr[3]);
2372 }
2373 }
2374 #endif
2375
2376 (void) ddi_dma_sync(
2377 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2378 0,
2379 ahci_cmd_table_size,
2380 DDI_DMA_SYNC_FORDEV);
2381
2382 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2383 cmd_slot * sizeof (ahci_cmd_header_t),
2384 sizeof (ahci_cmd_header_t),
2385 DDI_DMA_SYNC_FORDEV);
2386
2387 if ((ahci_check_dma_handle(ahci_portp->
2388 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2389 ahci_check_dma_handle(ahci_portp->
2390 ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2391 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2392 DDI_SERVICE_UNAFFECTED);
2393 return (AHCI_FAILURE);
2394 }
2395
2396 /* Set the corresponding bit in the PxSACT.DS for queued command */
2397 if (command_type == AHCI_NCQ_CMD) {
2398 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2399 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2400 (0x1 << cmd_slot));
2401 }
2402
2403 /* Indicate to the HBA that a command is active. */
2404 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2405 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2406 (0x1 << cmd_slot));
2407
2408 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2409 "exit: port %d", port);
2410
2411 /* Make sure the command is started by the PxSACT/PxCI */
2412 if (ahci_check_acc_handle(ahci_ctlp->
2413 ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2414 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2415 DDI_SERVICE_UNAFFECTED);
2416 return (AHCI_FAILURE);
2417 }
2418
2419 return (cmd_slot);
2420 }
2421
2422 /*
2423 * Called by the sata framework to abort the previously sent packet(s).
2424 *
2425 * Reset device to abort commands.
2426 */
2427 static int
2428 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2429 {
2430 ahci_ctl_t *ahci_ctlp;
2431 ahci_port_t *ahci_portp;
2432 uint32_t slot_status = 0;
2433 uint32_t aborted_tags = 0;
2434 uint32_t finished_tags = 0;
2435 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2436 uint8_t port;
2437 int tmp_slot;
2438 int instance = ddi_get_instance(dip);
2439
2440 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2441 port = ahci_ctlp->ahcictl_cport_to_port[cport];
2442
2443 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2444 "ahci_tran_abort enter: port %d", port);
2445
2446 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2447 mutex_enter(&ahci_portp->ahciport_mutex);
2448
2449 /*
2450 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2451 * commands are being mopped, therefore there is nothing else to do
2452 */
2453 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2454 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2455 "ahci_tran_abort: port %d is in "
2456 "mopping process, so just return directly ", port);
2457 mutex_exit(&ahci_portp->ahciport_mutex);
2458 return (SATA_SUCCESS);
2459 }
2460
2461 /*
2462 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2463 * command is being executed so no other commands is outstanding,
2464 * nothing to do.
2465 */
2466 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2467 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2468 "ahci_tran_abort: port %d is reading/writing "
2469 "port multiplier, so just return directly ", port);
2470 mutex_exit(&ahci_portp->ahciport_mutex);
2471 return (SATA_SUCCESS);
2472 }
2473
2474 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2475 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2476 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2477 /*
2478 * In case the targer driver would send the request before
2479 * sata framework can have the opportunity to process those
2480 * event reports.
2481 */
2482 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2483 spkt->satapkt_device.satadev_state =
2484 ahci_portp->ahciport_port_state;
2485 ahci_update_sata_registers(ahci_ctlp, port,
2486 &spkt->satapkt_device);
2487 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2488 "ahci_tran_abort returning SATA_FAILURE while "
2489 "port in FAILED/SHUTDOWN/PWROFF state: "
2490 "port: %d", port);
2491 mutex_exit(&ahci_portp->ahciport_mutex);
2492 return (SATA_FAILURE);
2493 }
2494
2495 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2496 /*
2497 * ahci_intr_phyrdy_change() may have rendered it to
2498 * AHCI_PORT_TYPE_NODEV.
2499 */
2500 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2501 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2502 spkt->satapkt_device.satadev_state =
2503 ahci_portp->ahciport_port_state;
2504 ahci_update_sata_registers(ahci_ctlp, port,
2505 &spkt->satapkt_device);
2506 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2507 "ahci_tran_abort returning SATA_FAILURE while "
2508 "no device attached: port: %d", port);
2509 mutex_exit(&ahci_portp->ahciport_mutex);
2510 return (SATA_FAILURE);
2511 }
2512
2513 if (flag == SATA_ABORT_ALL_PACKETS) {
2514 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2515 aborted_tags = ahci_portp->ahciport_pending_tags;
2516 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2517 aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2518
2519 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2520 instance, port);
2521 } else {
2522 aborted_tags = 0xffffffff;
2523 /*
2524 * Aborting one specific packet, first search the
2525 * ahciport_slot_pkts[] list for matching spkt.
2526 */
2527 for (tmp_slot = 0;
2528 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2529 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2530 aborted_tags = (0x1 << tmp_slot);
2531 break;
2532 }
2533 }
2534
2535 if (aborted_tags == 0xffffffff) {
2536 /* request packet is not on the pending list */
2537 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2538 "Cannot find the aborting pkt 0x%p on the "
2539 "pending list", (void *)spkt);
2540 ahci_update_sata_registers(ahci_ctlp, port,
2541 &spkt->satapkt_device);
2542 mutex_exit(&ahci_portp->ahciport_mutex);
2543 return (SATA_FAILURE);
2544 }
2545 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2546 instance, port, (void *)spkt);
2547 }
2548
2549 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2550 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2551 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2552 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2553 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2554 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2555
2556 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2557 ahci_portp->ahciport_mop_in_progress++;
2558
2559 /*
2560 * To abort the packet(s), first we are trying to clear PxCMD.ST
2561 * to stop the port, and if the port can be stopped
2562 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2563 * then we just send back the aborted packet(s) with ABORTED flag
2564 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2565 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2566 * perform a COMRESET.
2567 */
2568 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2569 ahci_portp, port, NULL, NULL);
2570
2571 /*
2572 * Compute which have finished and which need to be retried.
2573 *
2574 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2575 * minus the slot_status. The aborted_tags has to be deducted by
2576 * finished_tags since we can't possibly abort a tag which had finished
2577 * already.
2578 */
2579 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2580 finished_tags = ahci_portp->ahciport_pending_tags &
2581 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2582 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2583 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2584 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2585
2586 aborted_tags &= ~finished_tags;
2587
2588 ahci_mop_commands(ahci_ctlp,
2589 ahci_portp,
2590 slot_status,
2591 0, /* failed tags */
2592 0, /* timeout tags */
2593 aborted_tags,
2594 0); /* reset tags */
2595
2596 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2597 mutex_exit(&ahci_portp->ahciport_mutex);
2598
2599 return (SATA_SUCCESS);
2600 }
2601
2602 /*
2603 * Used to do device reset and reject all the pending packets on a device
2604 * during the reset operation.
2605 *
2606 * NOTE: ONLY called by ahci_tran_reset_dport
2607 */
2608 static int
2609 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2610 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2611 {
2612 uint32_t slot_status = 0;
2613 uint32_t reset_tags = 0;
2614 uint32_t finished_tags = 0;
2615 uint8_t port = addrp->aa_port;
2616 sata_device_t sdevice;
2617 int ret;
2618
2619 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2620
2621 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2622 "ahci_reset_device_reject_pkts on port: %d", port);
2623
2624 /*
2625 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2626 * commands are being mopped, therefore there is nothing else to do
2627 */
2628 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2629 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2630 "ahci_reset_device_reject_pkts: port %d is in "
2631 "mopping process, so return directly ", port);
2632 return (SATA_SUCCESS);
2633 }
2634
2635 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2636 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2637 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2638 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2639 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2640 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2641 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2642 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2643 }
2644
2645 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2646 != AHCI_SUCCESS) {
2647 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2648 "Try to do a port reset after software "
2649 "reset failed", port);
2650 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2651 if (ret != AHCI_SUCCESS) {
2652 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2653 "ahci_reset_device_reject_pkts: port %d "
2654 "failed", port);
2655 return (SATA_FAILURE);
2656 }
2657 }
2658 /* Set the reset in progress flag */
2659 ahci_portp->ahciport_reset_in_progress = 1;
2660
2661 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2662 ahci_portp->ahciport_mop_in_progress++;
2663
2664 /* Indicate to the framework that a reset has happened */
2665 bzero((void *)&sdevice, sizeof (sata_device_t));
2666 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2667 sdevice.satadev_addr.pmport = 0;
2668 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2669 sdevice.satadev_state = SATA_DSTATE_RESET |
2670 SATA_DSTATE_PWR_ACTIVE;
2671 mutex_exit(&ahci_portp->ahciport_mutex);
2672 sata_hba_event_notify(
2673 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2674 &sdevice,
2675 SATA_EVNT_DEVICE_RESET);
2676 mutex_enter(&ahci_portp->ahciport_mutex);
2677
2678 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2679 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2680
2681 /* Next try to mop the pending commands */
2682 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2683 finished_tags = ahci_portp->ahciport_pending_tags &
2684 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2685 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2686 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2687 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2688
2689 reset_tags &= ~finished_tags;
2690
2691 ahci_mop_commands(ahci_ctlp,
2692 ahci_portp,
2693 slot_status,
2694 0, /* failed tags */
2695 0, /* timeout tags */
2696 0, /* aborted tags */
2697 reset_tags); /* reset tags */
2698
2699 return (SATA_SUCCESS);
2700 }
2701
2702 /*
2703 * Used to do device reset and reject all the pending packets on a device
2704 * during the reset operation.
2705 *
2706 * NOTE: ONLY called by ahci_tran_reset_dport
2707 */
2708 static int
2709 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2710 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2711 {
2712 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2713 uint8_t port = addrp->aa_port;
2714 uint8_t pmport = addrp->aa_pmport;
2715 sata_device_t sdevice;
2716
2717 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2718
2719 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2720 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2721
2722 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2723 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2724 "ahci_reset_pmdevice_reject_pkts: port %d is in "
2725 "mopping process, so return directly ", port);
2726 return (SATA_SUCCESS);
2727 }
2728
2729 /* Checking for outstanding commands */
2730 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2731 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2732 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2733 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2734 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2735 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2736 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2737 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2738 }
2739
2740 /* Issue SOFTWARE reset command. */
2741 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2742 != AHCI_SUCCESS) {
2743 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2744 "Try to do a port reset after software "
2745 "reset failed", port);
2746 return (SATA_FAILURE);
2747 }
2748
2749 /* Set the reset in progress flag */
2750 ahci_portp->ahciport_reset_in_progress = 1;
2751
2752 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2753 ahci_portp->ahciport_mop_in_progress++;
2754
2755 /* Indicate to the framework that a reset has happened */
2756 bzero((void *)&sdevice, sizeof (sata_device_t));
2757 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2758 sdevice.satadev_addr.pmport = pmport;
2759 if (AHCI_ADDR_IS_PMULT(addrp))
2760 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2761 else
2762 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2763 sdevice.satadev_state = SATA_DSTATE_RESET |
2764 SATA_DSTATE_PWR_ACTIVE;
2765 mutex_exit(&ahci_portp->ahciport_mutex);
2766 sata_hba_event_notify(
2767 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2768 &sdevice,
2769 SATA_EVNT_DEVICE_RESET);
2770 mutex_enter(&ahci_portp->ahciport_mutex);
2771
2772 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2773 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2774 port, pmport);
2775
2776 /* Next try to mop the pending commands */
2777 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2778 finished_tags = ahci_portp->ahciport_pending_tags &
2779 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2780 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2781 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2782 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2783 reset_tags &= ~finished_tags;
2784
2785 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2786 "reset_tags = %x, finished_tags = %x, slot_status = %x",
2787 reset_tags, finished_tags, slot_status);
2788
2789 /*
2790 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2791 * try to reset a single device behind a port multiplier will
2792 * terminate all the commands on that HBA port. We need mop these
2793 * commands as well.
2794 */
2795 ahci_mop_commands(ahci_ctlp,
2796 ahci_portp,
2797 slot_status,
2798 0, /* failed tags */
2799 0, /* timeout tags */
2800 0, /* aborted tags */
2801 reset_tags); /* reset tags */
2802
2803 return (SATA_SUCCESS);
2804 }
2805
2806 /*
2807 * Used to do port reset and reject all the pending packets on a port during
2808 * the reset operation.
2809 */
2810 static int
2811 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2812 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2813 {
2814 uint32_t slot_status = 0;
2815 uint32_t reset_tags = 0;
2816 uint32_t finished_tags = 0;
2817 uint8_t port = addrp->aa_port;
2818
2819 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2820
2821 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2822 "ahci_reset_port_reject_pkts at port: %d", port);
2823
2824 /*
2825 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2826 * commands are being mopped, therefore there is nothing else to do
2827 */
2828 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2829 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2830 "ahci_reset_port_reject_pkts: port %d is in "
2831 "mopping process, so return directly ", port);
2832 return (SATA_SUCCESS);
2833 }
2834
2835 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2836 ahci_portp->ahciport_mop_in_progress++;
2837
2838 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2839 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2840 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2841 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2842 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2843 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2844 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2845 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2846 }
2847
2848 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2849 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2850 NULL) != AHCI_SUCCESS) {
2851
2852 /* Clear mop flag */
2853 ahci_portp->ahciport_mop_in_progress--;
2854 if (ahci_portp->ahciport_mop_in_progress == 0)
2855 ahci_portp->ahciport_flags &=
2856 ~AHCI_PORT_FLAG_MOPPING;
2857 return (SATA_FAILURE);
2858 }
2859
2860 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2861 finished_tags = ahci_portp->ahciport_pending_tags &
2862 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2863 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2864 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2865 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2866
2867 reset_tags &= ~finished_tags;
2868
2869 ahci_mop_commands(ahci_ctlp,
2870 ahci_portp,
2871 slot_status,
2872 0, /* failed tags */
2873 0, /* timeout tags */
2874 0, /* aborted tags */
2875 reset_tags); /* reset tags */
2876
2877 return (SATA_SUCCESS);
2878 }
2879
2880 /*
2881 * Used to do hba reset and reject all the pending packets on all ports
2882 * during the reset operation.
2883 */
2884 static int
2885 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2886 {
2887 ahci_port_t *ahci_portp;
2888 uint32_t slot_status[AHCI_MAX_PORTS];
2889 uint32_t reset_tags[AHCI_MAX_PORTS];
2890 uint32_t finished_tags[AHCI_MAX_PORTS];
2891 int port;
2892 int ret = SATA_SUCCESS;
2893
2894 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2895 "ahci_reset_hba_reject_pkts enter", NULL);
2896
2897 bzero(slot_status, sizeof (slot_status));
2898 bzero(reset_tags, sizeof (reset_tags));
2899 bzero(finished_tags, sizeof (finished_tags));
2900
2901 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2902 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2903 continue;
2904 }
2905
2906 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2907
2908 mutex_enter(&ahci_portp->ahciport_mutex);
2909 ahci_portp->ahciport_reset_in_progress = 1;
2910 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2911 slot_status[port] = ddi_get32(
2912 ahci_ctlp->ahcictl_ahci_acc_handle,
2913 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2914 reset_tags[port] = slot_status[port] &
2915 AHCI_SLOT_MASK(ahci_ctlp);
2916 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2917 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2918 port, reset_tags[port],
2919 ahci_portp->ahciport_pending_tags);
2920 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2921 slot_status[port] = ddi_get32(
2922 ahci_ctlp->ahcictl_ahci_acc_handle,
2923 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2924 reset_tags[port] = slot_status[port] &
2925 AHCI_NCQ_SLOT_MASK(ahci_portp);
2926 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2927 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2928 port, reset_tags[port],
2929 ahci_portp->ahciport_pending_tags);
2930 }
2931 mutex_exit(&ahci_portp->ahciport_mutex);
2932 }
2933
2934 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
2935 ret = SATA_FAILURE;
2936 }
2937
2938 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2939 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2940 continue;
2941 }
2942
2943 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2944
2945 mutex_enter(&ahci_portp->ahciport_mutex);
2946 /*
2947 * To prevent recursive enter to ahci_mop_commands, we need
2948 * check AHCI_PORT_FLAG_MOPPING flag.
2949 */
2950 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2951 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2952 "ahci_reset_hba_reject_pkts: port %d is in "
2953 "mopping process, so return directly ", port);
2954 mutex_exit(&ahci_portp->ahciport_mutex);
2955 continue;
2956 }
2957
2958 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2959 ahci_portp->ahciport_mop_in_progress++;
2960
2961 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2962 finished_tags[port] =
2963 ahci_portp->ahciport_pending_tags &
2964 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
2965 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2966 finished_tags[port] =
2967 ahci_portp->ahciport_pending_ncq_tags &
2968 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
2969
2970 reset_tags[port] &= ~finished_tags[port];
2971
2972 ahci_mop_commands(ahci_ctlp,
2973 ahci_portp,
2974 slot_status[port],
2975 0, /* failed tags */
2976 0, /* timeout tags */
2977 0, /* aborted tags */
2978 reset_tags[port]); /* reset tags */
2979 mutex_exit(&ahci_portp->ahciport_mutex);
2980 }
2981 out:
2982 return (ret);
2983 }
2984
2985 /*
2986 * Called by sata framework to reset a port(s) or device.
2987 */
2988 static int
2989 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
2990 {
2991 ahci_ctl_t *ahci_ctlp;
2992 ahci_port_t *ahci_portp;
2993 ahci_addr_t addr;
2994 uint8_t cport = sd->satadev_addr.cport;
2995 uint8_t pmport = sd->satadev_addr.pmport;
2996 uint8_t port;
2997 int ret = SATA_SUCCESS;
2998 int instance = ddi_get_instance(dip);
2999
3000 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3001 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3002 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3003
3004 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3005
3006 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3007 "ahci_tran_reset_dport enter: cport %d", cport);
3008
3009 switch (sd->satadev_addr.qual) {
3010 case SATA_ADDR_PMPORT:
3011 /*
3012 * If we want to issue a COMRESET on a pmport, we need to
3013 * reject the outstanding commands on that pmport. According
3014 * to AHCI spec, PxCI register could only be cleared by
3015 * clearing PxCMD.ST, which will halt the controller port - as
3016 * well as other pmports.
3017 *
3018 * Therefore we directly reset the controller port for
3019 * simplicity. ahci_tran_probe_port() will handle reset stuff
3020 * like initializing the given pmport.
3021 */
3022 /* FALLTHRU */
3023 case SATA_ADDR_CPORT:
3024 /* Port reset */
3025 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3026 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3027 "port %d reset port", instance, port);
3028
3029 mutex_enter(&ahci_portp->ahciport_mutex);
3030 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3031 mutex_exit(&ahci_portp->ahciport_mutex);
3032
3033 break;
3034
3035 case SATA_ADDR_DPMPORT:
3036 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3037 "port %d:%d reset device", instance, port, pmport);
3038 /* FALLTHRU */
3039 case SATA_ADDR_DCPORT:
3040 /* Device reset */
3041 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3042 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3043 "port %d reset device", instance, port);
3044
3045 mutex_enter(&ahci_portp->ahciport_mutex);
3046 /*
3047 * software reset request must be sent to SATA_PMULT_HOSTPORT
3048 * if target is a port multiplier:
3049 */
3050 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3051 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3052 AHCI_ADDR_SET_PMULT(&addr, port);
3053
3054 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3055 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3056 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3057 /*
3058 * In case the targer driver would send the request
3059 * before sata framework can have the opportunity to
3060 * process those event reports.
3061 */
3062 sd->satadev_state = ahci_portp->ahciport_port_state;
3063 ahci_update_sata_registers(ahci_ctlp, port, sd);
3064 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3065 "ahci_tran_reset_dport returning SATA_FAILURE "
3066 "while port in FAILED/SHUTDOWN/PWROFF state: "
3067 "port: %d", port);
3068 mutex_exit(&ahci_portp->ahciport_mutex);
3069 ret = SATA_FAILURE;
3070 break;
3071 }
3072
3073 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3074 SATA_DTYPE_NONE) {
3075 /*
3076 * ahci_intr_phyrdy_change() may have rendered it to
3077 * AHCI_PORT_TYPE_NODEV.
3078 */
3079 sd->satadev_type = SATA_DTYPE_NONE;
3080 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3081 &addr);
3082 ahci_update_sata_registers(ahci_ctlp, port, sd);
3083 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3084 "ahci_tran_reset_dport returning SATA_FAILURE "
3085 "while no device attached: port: %d", port);
3086 mutex_exit(&ahci_portp->ahciport_mutex);
3087 ret = SATA_FAILURE;
3088 break;
3089 }
3090
3091 if (AHCI_ADDR_IS_PORT(&addr)) {
3092 ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3093 ahci_portp, &addr);
3094 } else {
3095 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3096 ahci_portp, &addr);
3097 }
3098
3099 mutex_exit(&ahci_portp->ahciport_mutex);
3100 break;
3101
3102 case SATA_ADDR_CNTRL:
3103 /* Reset the whole controller */
3104 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3105 "reset the whole hba", instance);
3106 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3107 break;
3108
3109 default:
3110 ret = SATA_FAILURE;
3111 }
3112
3113 return (ret);
3114 }
3115
3116 /*
3117 * Called by sata framework to activate a port as part of hotplug.
3118 * (cfgadm -c connect satax/y)
3119 * Support port multiplier.
3120 */
3121 static int
3122 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3123 {
3124 ahci_ctl_t *ahci_ctlp;
3125 ahci_port_t *ahci_portp;
3126 ahci_addr_t addr;
3127 uint8_t cport = satadev->satadev_addr.cport;
3128 uint8_t pmport = satadev->satadev_addr.pmport;
3129 uint8_t port;
3130 int instance = ddi_get_instance(dip);
3131
3132 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3133 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3134
3135 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3136 "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3137
3138 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3139
3140 mutex_enter(&ahci_portp->ahciport_mutex);
3141 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3142 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3143
3144 if (AHCI_ADDR_IS_PORT(&addr)) {
3145 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3146 instance, port);
3147
3148 /* Enable the interrupts on the port */
3149 ahci_enable_port_intrs(ahci_ctlp, port);
3150
3151 /*
3152 * Reset the port so that the PHY communication would be
3153 * re-established. But this reset is an internal operation
3154 * and the sata module doesn't need to know about it.
3155 * Moreover, the port with a device attached will be started
3156 * too.
3157 */
3158 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3159 ahci_portp, port,
3160 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3161 NULL);
3162
3163 /*
3164 * Need to check the link status and device status of the port
3165 * and consider raising power if the port was in D3 state
3166 */
3167 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3168 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3169 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3170 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3171 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3172 instance, port, pmport);
3173 /* AHCI_ADDR_PMPORT */
3174 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3175 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3176 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3177 }
3178
3179 satadev->satadev_state = ahci_portp->ahciport_port_state;
3180
3181 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3182
3183 mutex_exit(&ahci_portp->ahciport_mutex);
3184 return (SATA_SUCCESS);
3185 }
3186
3187 /*
3188 * Called by sata framework to deactivate a port as part of hotplug.
3189 * (cfgadm -c disconnect satax/y)
3190 * Support port multiplier.
3191 */
3192 static int
3193 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3194 {
3195 ahci_ctl_t *ahci_ctlp;
3196 ahci_port_t *ahci_portp;
3197 ahci_addr_t addr;
3198 uint8_t cport = satadev->satadev_addr.cport;
3199 uint8_t pmport = satadev->satadev_addr.pmport;
3200 uint8_t port;
3201 uint32_t port_scontrol;
3202 int instance = ddi_get_instance(dip);
3203
3204 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3205 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3206
3207 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3208 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3209
3210 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3211 mutex_enter(&ahci_portp->ahciport_mutex);
3212 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3213 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3214
3215 if (AHCI_ADDR_IS_PORT(&addr)) {
3216 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3217 instance, port);
3218
3219 /* Disable the interrupts on the port */
3220 ahci_disable_port_intrs(ahci_ctlp, port);
3221
3222 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3223
3224 /* First to abort all the pending commands */
3225 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3226
3227 /* Then stop the port */
3228 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3229 ahci_portp, port);
3230 }
3231
3232 /* Next put the PHY offline */
3233 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3234 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3235 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3236 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3237 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3238 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3239 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3240 instance, port, pmport);
3241
3242 ahci_disable_port_intrs(ahci_ctlp, port);
3243 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3244 != SATA_DTYPE_NONE)
3245 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3246
3247 /* Re-enable the interrupts for the other pmports */
3248 ahci_enable_port_intrs(ahci_ctlp, port);
3249 }
3250
3251 /* Update port state */
3252 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3253 satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3254
3255 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3256
3257 mutex_exit(&ahci_portp->ahciport_mutex);
3258 return (SATA_SUCCESS);
3259 }
3260
3261 /*
3262 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3263 * when a device is unplugged or a port is deactivated.
3264 */
3265 static void
3266 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3267 ahci_port_t *ahci_portp, uint8_t port)
3268 {
3269 uint32_t slot_status = 0;
3270 uint32_t abort_tags = 0;
3271
3272 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3273
3274 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3275 "ahci_reject_all_abort_pkts at port: %d", port);
3276
3277 /* Read/write port multiplier command takes highest priority */
3278 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3279 slot_status = 0x1;
3280 abort_tags = 0x1;
3281 goto out;
3282 }
3283
3284 /*
3285 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3286 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3287 * to get the error data, if yes when the device is removed, the
3288 * command needs to be aborted too.
3289 */
3290 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3291 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3292 slot_status = 0x1;
3293 abort_tags = 0x1;
3294 goto out;
3295 } else {
3296 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3297 "ahci_reject_all_abort_pkts return directly "
3298 "port %d no needs to reject any outstanding "
3299 "commands", port);
3300 return;
3301 }
3302 }
3303
3304 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3305 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3306 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3307 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3308 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3309 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3310 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3311 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3312 }
3313
3314 out:
3315 /* No need to do mop when there is no outstanding commands */
3316 if (slot_status != 0) {
3317 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3318 ahci_portp->ahciport_mop_in_progress++;
3319
3320 ahci_mop_commands(ahci_ctlp,
3321 ahci_portp,
3322 slot_status,
3323 0, /* failed tags */
3324 0, /* timeout tags */
3325 abort_tags, /* aborting tags */
3326 0); /* reset tags */
3327 }
3328 }
3329
3330 #if defined(__lock_lint)
3331 static int
3332 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3333 {
3334 return (SATA_SUCCESS);
3335 }
3336 #endif
3337
3338 /*
3339 * Initialize fma capabilities and register with IO fault services.
3340 */
3341 static void
3342 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3343 {
3344 /*
3345 * Need to change iblock to priority for new MSI intr
3346 */
3347 ddi_iblock_cookie_t fm_ibc;
3348
3349 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3350 ahci_ctlp->ahcictl_dip,
3351 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3352 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3353 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3354
3355 /* Only register with IO Fault Services if we have some capability */
3356 if (ahci_ctlp->ahcictl_fm_cap) {
3357 /* Adjust access and dma attributes for FMA */
3358 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3359 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3360 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3361 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3362 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3363
3364 /*
3365 * Register capabilities with IO Fault Services.
3366 * ahcictl_fm_cap will be updated to indicate
3367 * capabilities actually supported (not requested.)
3368 */
3369 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3370 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3371
3372 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3373 cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3374 ddi_get_instance(ahci_ctlp->ahcictl_dip));
3375 return;
3376 }
3377 /*
3378 * Initialize pci ereport capabilities if ereport
3379 * capable (should always be.)
3380 */
3381 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3382 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3383 pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3384 }
3385
3386 /*
3387 * Register error callback if error callback capable.
3388 */
3389 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3390 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3391 ahci_fm_error_cb, (void *) ahci_ctlp);
3392 }
3393
3394 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3395 "ahci_fm_fini: fma enabled.", NULL);
3396 }
3397 }
3398
3399 /*
3400 * Releases fma capabilities and un-registers with IO fault services.
3401 */
3402 static void
3403 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3404 {
3405 /* Only unregister FMA capabilities if registered */
3406 if (ahci_ctlp->ahcictl_fm_cap) {
3407 /*
3408 * Un-register error callback if error callback capable.
3409 */
3410 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3411 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3412 }
3413
3414 /*
3415 * Release any resources allocated by pci_ereport_setup()
3416 */
3417 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3418 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3419 pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3420 }
3421
3422 /* Unregister from IO Fault Services */
3423 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3424
3425 /* Adjust access and dma attributes for FMA */
3426 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3427 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3428 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3429 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3430 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3431
3432 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3433 "ahci_fm_fini: fma disabled.", NULL);
3434 }
3435 }
3436
3437 /*ARGSUSED*/
3438 static int
3439 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3440 {
3441 /*
3442 * as the driver can always deal with an error in any dma or
3443 * access handle, we can just return the fme_status value.
3444 */
3445 pci_ereport_post(dip, err, NULL);
3446 return (err->fme_status);
3447 }
3448
3449 int
3450 ahci_check_acc_handle(ddi_acc_handle_t handle)
3451 {
3452 ddi_fm_error_t de;
3453
3454 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3455 return (de.fme_status);
3456 }
3457
3458 int
3459 ahci_check_dma_handle(ddi_dma_handle_t handle)
3460 {
3461 ddi_fm_error_t de;
3462
3463 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3464 return (de.fme_status);
3465 }
3466
3467 /*
3468 * Generate an ereport
3469 */
3470 void
3471 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3472 {
3473 uint64_t ena;
3474 char buf[FM_MAX_CLASS];
3475
3476 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3477 ena = fm_ena_generate(0, FM_ENA_FMT1);
3478 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3479 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3480 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3481 FM_EREPORT_VERSION, NULL);
3482 }
3483 }
3484
3485 /*
3486 * Check if all handles are correctly allocated.
3487 */
3488 static int
3489 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3490 {
3491 int port;
3492
3493 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3494 return (DDI_FAILURE);
3495 }
3496
3497 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3498 ahci_port_t *ahci_portp;
3499
3500 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3501 continue;
3502
3503 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3504
3505 mutex_enter(&ahci_portp->ahciport_mutex);
3506 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3507 mutex_exit(&ahci_portp->ahciport_mutex);
3508 return (DDI_FAILURE);
3509 }
3510 mutex_exit(&ahci_portp->ahciport_mutex);
3511 }
3512
3513 return (DDI_SUCCESS);
3514 }
3515
3516 /*
3517 * Check the access handles for the controller. Note that
3518 * ahcictl_pci_conf_handle is only used in attach process.
3519 */
3520 static int
3521 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3522 {
3523 if ((ahci_check_acc_handle(ahci_ctlp->
3524 ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3525 (ahci_check_acc_handle(ahci_ctlp->
3526 ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3527 return (DDI_FAILURE);
3528 }
3529 return (DDI_SUCCESS);
3530 }
3531
3532 /*
3533 * Check the DMA handles and the access handles of a controller port.
3534 */
3535 static int
3536 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3537 {
3538 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3539 int slot;
3540
3541 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3542
3543 if ((ahci_check_dma_handle(ahci_portp->
3544 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3545 (ahci_check_dma_handle(ahci_portp->
3546 ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3547 (ahci_check_acc_handle(ahci_portp->
3548 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3549 (ahci_check_acc_handle(ahci_portp->
3550 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3551 return (DDI_FAILURE);
3552 }
3553 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3554 if (ahci_check_slot_handle(ahci_portp, slot)
3555 != DDI_SUCCESS) {
3556 return (DDI_FAILURE);
3557 }
3558 }
3559 return (DDI_SUCCESS);
3560 }
3561
3562 /*
3563 * Check the DMA handles and the access handles of a cmd table slot.
3564 */
3565 static int
3566 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3567 {
3568 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3569
3570 if ((ahci_check_acc_handle(ahci_portp->
3571 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3572 (ahci_check_dma_handle(ahci_portp->
3573 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3574 return (DDI_FAILURE);
3575 }
3576 return (DDI_SUCCESS);
3577 }
3578
3579 /*
3580 * Allocate the ports structure, only called by ahci_attach
3581 */
3582 static int
3583 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3584 {
3585 int port, cport = 0;
3586
3587 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3588 "ahci_alloc_ports_state enter", NULL);
3589
3590 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3591
3592 /* Allocate structures only for the implemented ports */
3593 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3594 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3595 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3596 "hba port %d not implemented", port);
3597 continue;
3598 }
3599
3600 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3601 ahci_ctlp->ahcictl_port_to_cport[port] =
3602 (uint8_t)cport++;
3603
3604 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3605 goto err_out;
3606 }
3607 }
3608
3609 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3610 return (AHCI_SUCCESS);
3611
3612 err_out:
3613 for (port--; port >= 0; port--) {
3614 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3615 ahci_dealloc_port_state(ahci_ctlp, port);
3616 }
3617 }
3618
3619 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3620 return (AHCI_FAILURE);
3621 }
3622
3623 /*
3624 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3625 */
3626 static void
3627 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3628 {
3629 int port;
3630
3631 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3632 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3633 /* if this port is implemented by the HBA */
3634 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3635 ahci_dealloc_port_state(ahci_ctlp, port);
3636 }
3637 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3638 }
3639
3640 /*
3641 * Drain the taskq.
3642 */
3643 static void
3644 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3645 {
3646 ahci_port_t *ahci_portp;
3647 int port;
3648
3649 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3650 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3651 continue;
3652 }
3653
3654 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3655
3656 mutex_enter(&ahci_portp->ahciport_mutex);
3657 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3658 mutex_exit(&ahci_portp->ahciport_mutex);
3659 }
3660 }
3661
3662 /*
3663 * Initialize the controller and all ports. And then try to start the ports
3664 * if there are devices attached.
3665 *
3666 * This routine can be called from three seperate cases: DDI_ATTACH,
3667 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3668 * other two cases; device signature probing are attempted only during
3669 * DDI_ATTACH case.
3670 */
3671 static int
3672 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3673 {
3674 ahci_port_t *ahci_portp;
3675 ahci_addr_t addr;
3676 int port;
3677
3678 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3679 "ahci_initialize_controller enter", NULL);
3680
3681 /* Disable the whole controller interrupts */
3682 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3683 ahci_disable_all_intrs(ahci_ctlp);
3684 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3685
3686 /* Initialize the implemented ports and structures */
3687 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3688 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3689 continue;
3690 }
3691
3692 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3693 mutex_enter(&ahci_portp->ahciport_mutex);
3694
3695 /*
3696 * Ensure that the controller is not in the running state
3697 * by checking every implemented port's PxCMD register
3698 */
3699 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3700
3701 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3702 != AHCI_SUCCESS) {
3703 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3704 "ahci_initialize_controller: failed to "
3705 "initialize port %d", port);
3706 /*
3707 * Set the port state to SATA_PSTATE_FAILED if
3708 * failed to initialize it.
3709 */
3710 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3711 }
3712
3713 mutex_exit(&ahci_portp->ahciport_mutex);
3714 }
3715
3716 /* Enable the whole controller interrupts */
3717 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3718 ahci_enable_all_intrs(ahci_ctlp);
3719 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3720
3721 return (AHCI_SUCCESS);
3722 }
3723
3724 /*
3725 * Reverse of ahci_initialize_controller()
3726 *
3727 * We only need to stop the ports and disable the interrupt.
3728 */
3729 static void
3730 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3731 {
3732 ahci_port_t *ahci_portp;
3733 int port;
3734
3735 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3736 "ahci_uninitialize_controller enter", NULL);
3737
3738 /* disable all the interrupts. */
3739 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3740 ahci_disable_all_intrs(ahci_ctlp);
3741 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3742
3743 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3744 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3745 continue;
3746 }
3747
3748 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3749
3750 /* Stop the port by clearing PxCMD.ST */
3751 mutex_enter(&ahci_portp->ahciport_mutex);
3752
3753 /*
3754 * Here we must disable the port interrupt because
3755 * ahci_disable_all_intrs only clear GHC.IE, and IS
3756 * register will be still set if PxIE is enabled.
3757 * When ahci shares one IRQ with other drivers, the
3758 * intr handler may claim the intr mistakenly.
3759 */
3760 ahci_disable_port_intrs(ahci_ctlp, port);
3761 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3762 ahci_portp, port);
3763 mutex_exit(&ahci_portp->ahciport_mutex);
3764 }
3765 }
3766
3767 /*
3768 * ahci_alloc_pmult()
3769 * 1. Setting HBA port registers which are necessary for a port multiplier.
3770 * (Set PxCMD.PMA while PxCMD.ST is '0')
3771 * 2. Allocate ahci_pmult_info structure.
3772 *
3773 * NOTE: Must stop port before the function is called.
3774 */
3775 static void
3776 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3777 {
3778 uint32_t port_cmd_status;
3779 uint8_t port = ahci_portp->ahciport_port_num;
3780
3781 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3782
3783 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3784 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3785
3786 /* The port must have been stopped before. */
3787 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3788
3789 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3790 /* set PMA bit */
3791 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3792 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3793 port_cmd_status|AHCI_CMD_STATUS_PMA);
3794
3795 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3796 "ahci_alloc_pmult: "
3797 "PxCMD.PMA bit set at port %d.", port);
3798 }
3799
3800 /* Allocate port multiplier information structure */
3801 if (ahci_portp->ahciport_pmult_info == NULL) {
3802 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3803 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3804 }
3805
3806 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3807 }
3808
3809 /*
3810 * ahci_dealloc_pmult()
3811 * 1. Clearing related registers when a port multiplier is detached.
3812 * (Clear PxCMD.PMA while PxCMD.ST is '0')
3813 * 2. Deallocate ahci_pmult_info structure.
3814 *
3815 * NOTE: Must stop port before the function is called.
3816 */
3817 static void
3818 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3819 {
3820 uint32_t port_cmd_status;
3821 uint8_t port = ahci_portp->ahciport_port_num;
3822
3823 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3824
3825 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3826 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3827
3828 if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3829 /* Clear PMA bit */
3830 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3831 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3832 (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3833
3834 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3835 "ahci_dealloc_pmult: "
3836 "PxCMD.PMA bit cleared at port %d.", port);
3837 }
3838
3839 /* Release port multiplier information structure */
3840 if (ahci_portp->ahciport_pmult_info != NULL) {
3841 kmem_free(ahci_portp->ahciport_pmult_info,
3842 sizeof (ahci_pmult_info_t));
3843 ahci_portp->ahciport_pmult_info = NULL;
3844 }
3845 }
3846
3847 /*
3848 * Staggered Spin-up.
3849 */
3850 static void
3851 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3852 {
3853 uint32_t cap_status;
3854 uint32_t port_cmd_status;
3855
3856 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3857
3858 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3859 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3860
3861 /* Check for staggered spin-up support */
3862 if (!(cap_status & AHCI_HBA_CAP_SSS))
3863 return;
3864
3865 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3866 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3867
3868 /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3869 if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3870 return;
3871
3872 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3873
3874 /* Set PxCMD.SUD */
3875 port_cmd_status |= AHCI_CMD_STATUS_SUD;
3876 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3877 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3878 port_cmd_status);
3879 }
3880
3881 /*
3882 * The routine is to initialize a port. First put the port in NotRunning
3883 * state, then enable port interrupt and clear Serror register. And under
3884 * AHCI_ATTACH case, find device signature and then try to start the port.
3885 *
3886 * Called by
3887 * 1. ahci_initialize_controller
3888 * 2. ahci_intr_phyrdy_change (hotplug)
3889 */
3890 static int
3891 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3892 ahci_addr_t *addrp)
3893 {
3894 uint32_t port_sstatus, port_task_file, port_cmd_status;
3895 uint8_t port = addrp->aa_port;
3896 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */
3897 int ret;
3898
3899 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3900
3901 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */
3902 ASSERT(AHCI_ADDR_IS_PORT(addrp));
3903
3904 /*
3905 * At the time being, only probe ports/devices and get the types of
3906 * attached devices during DDI_ATTACH. In fact, the device can be
3907 * changed during power state changes, but at the time being, we
3908 * don't support the situation.
3909 */
3910 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
3911 resuming = B_FALSE;
3912 } else {
3913 /* check for DDI_RESUME case */
3914 mutex_exit(&ahci_portp->ahciport_mutex);
3915 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3916 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
3917 resuming = B_FALSE;
3918 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3919 mutex_enter(&ahci_portp->ahciport_mutex);
3920 }
3921
3922 if (resuming) {
3923 /*
3924 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
3925 * and PxFBU registers in case these registers were cleared
3926 * during the suspend.
3927 */
3928 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
3929 "ahci_initialize_port: port %d "
3930 "set PxCLB, PxCLBU, PxFB and PxFBU "
3931 "during resume", port);
3932
3933 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
3934 AHCI_SUCCESS)
3935 return (AHCI_FAILURE);
3936 }
3937
3938 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3939 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3940
3941 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3942 "ahci_initialize_port: port %d ", port);
3943
3944 /*
3945 * Check whether the port is in NotRunning state, if not,
3946 * put the port in NotRunning state
3947 */
3948 if (port_cmd_status &
3949 (AHCI_CMD_STATUS_ST |
3950 AHCI_CMD_STATUS_CR |
3951 AHCI_CMD_STATUS_FRE |
3952 AHCI_CMD_STATUS_FR)) {
3953 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3954 ahci_portp, port);
3955 }
3956
3957 /* Make sure the drive is spun-up */
3958 ahci_staggered_spin_up(ahci_ctlp, port);
3959
3960 /* Disable interrupt */
3961 ahci_disable_port_intrs(ahci_ctlp, port);
3962
3963 /* Device is unknown at first */
3964 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
3965
3966 /* Disable the interface power management */
3967 ahci_disable_interface_pm(ahci_ctlp, port);
3968
3969 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3970 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
3971 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3972 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
3973
3974 /* Check physcial link status */
3975 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
3976 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
3977
3978 /* Check interface status */
3979 port_task_file & AHCI_TFD_STS_BSY ||
3980 port_task_file & AHCI_TFD_STS_DRQ ||
3981
3982 /* Check whether port reset must be executed */
3983 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
3984
3985 /* Always reset port on RESUME */
3986 resuming != B_FALSE) {
3987
3988 /* Something went wrong, we need do some reset things */
3989 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
3990
3991 /* Does port reset succeed on HBA port? */
3992 if (ret != AHCI_SUCCESS) {
3993 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
3994 "ahci_initialize_port:"
3995 "port reset failed at port %d", port);
3996 return (AHCI_FAILURE);
3997 }
3998
3999 /* Is port failed? */
4000 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4001 SATA_PSTATE_FAILED) {
4002 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4003 "ahci_initialize_port: port %d state 0x%x",
4004 port, ahci_portp->ahciport_port_state);
4005 return (AHCI_FAILURE);
4006 }
4007 }
4008
4009 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4010 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4011
4012 /*
4013 * Try to get the device signature if the port is not empty.
4014 */
4015 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4016 SATA_DTYPE_NONE)
4017 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4018
4019 /* Return directly if no device connected */
4020 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4021 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4022 "No device connected to port %d", port);
4023 goto out;
4024 }
4025
4026 /* If this is a port multiplier, we need do some initialization */
4027 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4028 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4029 "Port multiplier found at port %d", port);
4030 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4031 }
4032
4033 /* Try to start the port */
4034 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4035 != AHCI_SUCCESS) {
4036 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4037 "failed to start port %d", port);
4038 return (AHCI_FAILURE);
4039 }
4040 out:
4041 /* Enable port interrupts */
4042 ahci_enable_port_intrs(ahci_ctlp, port);
4043
4044 return (AHCI_SUCCESS);
4045 }
4046
4047 /*
4048 * Handle hardware defect, and check the capabilities. For example,
4049 * power management capabilty and MSI capability.
4050 */
4051 static int
4052 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4053 {
4054 ushort_t caps_ptr, cap_count, cap;
4055 #if AHCI_DEBUG
4056 ushort_t pmcap, pmcsr;
4057 ushort_t msimc;
4058 #endif
4059 uint8_t revision;
4060
4061 ahci_ctlp->ahcictl_venid =
4062 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4063 PCI_CONF_VENID);
4064
4065 ahci_ctlp->ahcictl_devid =
4066 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4067 PCI_CONF_DEVID);
4068
4069 /*
4070 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4071 * controllers with 0x00 revision id work on 4-byte aligned buffer,
4072 * which is a bug and was fixed after 0x00 revision id controllers.
4073 *
4074 * Moreover, VT8251 cannot use multiple command slots in the command
4075 * list for non-queued commands because the previous register content
4076 * of PxCI can be re-written in the register write, so a flag will be
4077 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4078 *
4079 * For VT8251, software reset also has the same defect as the below
4080 * AMD/ATI chipset. That is, software reset will get failed if 0xf
4081 * is filled in pmport field. Therefore, another software reset need
4082 * to be done with 0 filled in pmport field.
4083 */
4084 if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4085 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4086 PCI_CONF_REVID);
4087 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4088 "revision id = 0x%x", revision);
4089 if (revision == 0x00) {
4090 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4091 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4092 "change ddi_attr_align to 0x4", NULL);
4093 }
4094
4095 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4096 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4097 "VT8251 cannot use multiple command lists for "
4098 "non-queued commands", NULL);
4099
4100 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4101 }
4102
4103 /*
4104 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4105 * DMA addressing for communication memory descriptors though S64A bit
4106 * of CAP register declares it supports. Even though 64-bit DMA for
4107 * data buffer works on ASUS M2A-VM with newer BIOS, three other
4108 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4109 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4110 *
4111 * Due to certain hardware issue, the chipset must do port reset during
4112 * initialization, otherwise, when retrieving device signature,
4113 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4114 * need to set.
4115 *
4116 * For this chipset software reset will get failure if the pmport of
4117 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4118 * multiplier is connected to the port. In order to fix the issue,
4119 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4120 * reset got failure, the driver will try to do another software reset
4121 * with pmport 0.
4122 */
4123 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4124 ahci_ctlp->ahcictl_devid == 0x4380) {
4125 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4126 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4127 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4128 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4129
4130 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4131 "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4132 "communication memory descriptors though CAP indicates "
4133 "support, so force it to use 32-bit DMA", NULL);
4134 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4135 "ATI SB600 need to do a port reset during initialization",
4136 NULL);
4137 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4138 "ATI SB600 will get software reset failure if pmport "
4139 "is set 0xf and no port multiplier is attached", NULL);
4140 }
4141
4142 /*
4143 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4144 * vendor ID and device ID (0x1002,0x4391).
4145 *
4146 * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4147 * DMA addressing for communication memory descriptors though S64A bit
4148 * of CAP register declares the support. However, it does support
4149 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4150 * set for this controller.
4151 *
4152 * SB710 has the same initialization issue as SB600, so it also need
4153 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4154 *
4155 * SB700 also has the same issue about software reset, and thus
4156 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4157 */
4158 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4159 ahci_ctlp->ahcictl_devid == 0x4391) {
4160 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4161 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4162 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4163
4164 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4165 "ATI SB700/750 cannot do 64-bit DMA for communication "
4166 "memory descriptors though CAP indicates support, "
4167 "so force it to use 32-bit DMA", NULL);
4168 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4169 "ATI SB710 need to do a port reset during initialization",
4170 NULL);
4171 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4172 "ATI SB700 will get software reset failure if pmport "
4173 "is set 0xf and no port multiplier is attached", NULL);
4174 }
4175
4176 /*
4177 * Check if capabilities list is supported and if so,
4178 * get initial capabilities pointer and clear bits 0,1.
4179 */
4180 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4181 PCI_CONF_STAT) & PCI_STAT_CAP) {
4182 caps_ptr = P2ALIGN(pci_config_get8(
4183 ahci_ctlp->ahcictl_pci_conf_handle,
4184 PCI_CONF_CAP_PTR), 4);
4185 } else {
4186 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4187 }
4188
4189 /*
4190 * Walk capabilities if supported.
4191 */
4192 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4193
4194 /*
4195 * Check that we haven't exceeded the maximum number of
4196 * capabilities and that the pointer is in a valid range.
4197 */
4198 if (++cap_count > PCI_CAP_MAX_PTR) {
4199 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4200 "too many device capabilities", NULL);
4201 return (AHCI_FAILURE);
4202 }
4203 if (caps_ptr < PCI_CAP_PTR_OFF) {
4204 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4205 "capabilities pointer 0x%x out of range",
4206 caps_ptr);
4207 return (AHCI_FAILURE);
4208 }
4209
4210 /*
4211 * Get next capability and check that it is valid.
4212 * For now, we only support power management.
4213 */
4214 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4215 caps_ptr);
4216 switch (cap) {
4217 case PCI_CAP_ID_PM:
4218
4219 /* power management supported */
4220 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4221
4222 /* Save PMCSR offset */
4223 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4224
4225 #if AHCI_DEBUG
4226 pmcap = pci_config_get16(
4227 ahci_ctlp->ahcictl_pci_conf_handle,
4228 caps_ptr + PCI_PMCAP);
4229 pmcsr = pci_config_get16(
4230 ahci_ctlp->ahcictl_pci_conf_handle,
4231 ahci_ctlp->ahcictl_pmcsr_offset);
4232 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4233 "Power Management capability found PCI_PMCAP "
4234 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4235 if ((pmcap & 0x3) == 0x3)
4236 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4237 "PCI Power Management Interface "
4238 "spec 1.2 compliant", NULL);
4239 #endif
4240 break;
4241
4242 case PCI_CAP_ID_MSI:
4243 #if AHCI_DEBUG
4244 msimc = pci_config_get16(
4245 ahci_ctlp->ahcictl_pci_conf_handle,
4246 caps_ptr + PCI_MSI_CTRL);
4247 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4248 "Message Signaled Interrupt capability found "
4249 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4250 #endif
4251 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4252 "MSI capability found", NULL);
4253 break;
4254
4255 case PCI_CAP_ID_PCIX:
4256 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4257 "PCI-X capability found", NULL);
4258 break;
4259
4260 case PCI_CAP_ID_PCI_E:
4261 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4262 "PCI Express capability found", NULL);
4263 break;
4264
4265 case PCI_CAP_ID_MSI_X:
4266 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4267 "MSI-X capability found", NULL);
4268 break;
4269
4270 case PCI_CAP_ID_SATA:
4271 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4272 "SATA capability found", NULL);
4273 break;
4274
4275 case PCI_CAP_ID_VS:
4276 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4277 "Vendor Specific capability found", NULL);
4278 break;
4279
4280 default:
4281 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4282 "unrecognized capability 0x%x", cap);
4283 break;
4284 }
4285
4286 /*
4287 * Get next capabilities pointer and clear bits 0,1.
4288 */
4289 caps_ptr = P2ALIGN(pci_config_get8(
4290 ahci_ctlp->ahcictl_pci_conf_handle,
4291 (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4292 }
4293
4294 return (AHCI_SUCCESS);
4295 }
4296
4297 /*
4298 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4299 * PORTMULT command. SYNC & POLLING mode is used.
4300 */
4301 static int
4302 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4303 uint8_t regn, uint32_t *pregv, uint8_t type)
4304 {
4305 ahci_port_t *ahci_portp;
4306 ahci_addr_t pmult_addr;
4307 sata_pkt_t *spkt;
4308 sata_cmd_t *scmd;
4309 sata_device_t sata_device;
4310 uint8_t port = addrp->aa_port;
4311 uint8_t pmport = addrp->aa_pmport;
4312 uint8_t cport;
4313 uint32_t intr_mask;
4314 int rval;
4315 char portstr[10];
4316
4317 SET_PORTSTR(portstr, addrp);
4318 cport = ahci_ctlp->ahcictl_port_to_cport[port];
4319 ahci_portp = ahci_ctlp->ahcictl_ports[port];
4320
4321 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4322 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4323
4324 /* Check the existence of the port multiplier */
4325 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4326 return (AHCI_FAILURE);
4327
4328 /* Request a READ/WRITE PORTMULT sata packet. */
4329 bzero(&sata_device, sizeof (sata_device_t));
4330 sata_device.satadev_addr.cport = cport;
4331 sata_device.satadev_addr.pmport = pmport;
4332 sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4333 sata_device.satadev_rev = SATA_DEVICE_REV;
4334
4335 /*
4336 * Make sure no command is outstanding here. All R/W PMULT requests
4337 * come from
4338 *
4339 * 1. ahci_attach()
4340 * The port should be empty.
4341 *
4342 * 2. ahci_tran_probe_port()
4343 * Any request from SATA framework (via ahci_tran_start) should be
4344 * rejected if R/W PMULT command is outstanding.
4345 *
4346 * If we are doing mopping, do not check those flags because no
4347 * command will be actually outstanding.
4348 *
4349 * If the port has been occupied by any other commands, the probe
4350 * function will return a SATA_RETRY. SATA framework will retry
4351 * later.
4352 */
4353 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4354 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4355 "R/W PMULT failed: R/W PMULT in progress at port %d.",
4356 port, ahci_portp->ahciport_flags);
4357 return (AHCI_FAILURE);
4358 }
4359
4360 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4361 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4362 NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4363 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4364 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4365 "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4366 port, ahci_portp->ahciport_flags);
4367 return (AHCI_FAILURE);
4368 }
4369
4370 /*
4371 * The port multiplier is gone. This may happen when
4372 * 1. Cutting off the power of an enclosure. The device lose the power
4373 * before port multiplier.
4374 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4375 *
4376 * The issued command should be aborted and the following command
4377 * should not be continued.
4378 */
4379 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4380 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4381 "READ/WRITE PMULT failed: "
4382 "port-mult is removed from port %d", port);
4383 return (AHCI_FAILURE);
4384 }
4385
4386 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4387
4388 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4389 &sata_device, regn, *pregv, type);
4390
4391 /*
4392 * READ/WRITE PORTMULT command is intended to sent to the control port
4393 * of the port multiplier.
4394 */
4395 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4396
4397 ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4398
4399 /* No interrupt here. Store the interrupt enable mask. */
4400 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4401 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4402 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4403 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4404
4405 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4406
4407 if (rval == AHCI_SUCCESS &&
4408 spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4409 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4410 scmd = &spkt->satapkt_cmd;
4411 *pregv = scmd->satacmd_lba_high_lsb << 24 |
4412 scmd->satacmd_lba_mid_lsb << 16 |
4413 scmd->satacmd_lba_low_lsb << 8 |
4414 scmd->satacmd_sec_count_lsb;
4415 }
4416 } else {
4417 /* Failed or not completed. */
4418 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4419 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4420 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4421 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4422 rval = AHCI_FAILURE;
4423 }
4424 out:
4425 /* Restore the interrupt mask */
4426 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4427 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4428
4429 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4430 ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4431 sata_free_rdwr_pmult_pkt(spkt);
4432 return (rval);
4433 }
4434
4435 static int
4436 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4437 uint8_t regn, uint32_t *pregv)
4438 {
4439 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4440 SATA_RDWR_PMULT_PKT_TYPE_READ);
4441 }
4442
4443 static int
4444 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4445 uint8_t regn, uint32_t regv)
4446 {
4447 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v,
4448 SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4449 }
4450
4451 #define READ_PMULT(addrp, r, pv, out) \
4452 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \
4453 goto out;
4454
4455 #define WRITE_PMULT(addrp, r, v, out) \
4456 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \
4457 goto out;
4458
4459 /*
4460 * Update sata registers on port multiplier, including GSCR/PSCR registers.
4461 * ahci_update_pmult_gscr()
4462 * ahci_update_pmult_pscr()
4463 */
4464 static int
4465 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4466 sata_pmult_gscr_t *sg)
4467 {
4468 ASSERT(MUTEX_HELD(
4469 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4470
4471 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4472 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4473 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4474 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4475
4476 return (AHCI_SUCCESS);
4477
4478 err: /* R/W PMULT error */
4479 return (AHCI_FAILURE);
4480 }
4481
4482 static int
4483 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4484 sata_device_t *sd)
4485 {
4486 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4487 ASSERT(MUTEX_HELD(
4488 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4489
4490 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4491 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4492 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4493 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4494
4495 return (AHCI_SUCCESS);
4496
4497 err: /* R/W PMULT error */
4498 return (AHCI_FAILURE);
4499 }
4500
4501 /*
4502 * ahci_initialize_pmult()
4503 *
4504 * Initialize a port multiplier, including
4505 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4506 * 2. Redefine MASK register. (SATA Chap 16.?)
4507 */
4508 static int
4509 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4510 ahci_addr_t *addrp, sata_device_t *sd)
4511 {
4512 sata_pmult_gscr_t sg;
4513 uint32_t gscr64;
4514 uint8_t port = addrp->aa_port;
4515
4516 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4517
4518 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4519 "[Initialize] Port-multiplier at port %d.", port);
4520
4521 /*
4522 * Enable features of port multiplier. Currently only
4523 * Asynchronous Notification is enabled.
4524 */
4525 /* Check gscr64 for supported features. */
4526 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4527
4528 if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4529 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4530 "port %d: Port Multiplier supports "
4531 "Asynchronous Notification.", port);
4532
4533 /* Write to gscr96 to enabled features */
4534 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4535 SATA_PMULT_CAP_SNOTIF, err);
4536
4537 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4538 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4539 AHCI_SNOTIF_CLEAR_ALL);
4540 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4541 "port %d: PMult PxSNTF cleared.", port);
4542
4543 }
4544
4545 /*
4546 * Now we need to update gscr33 register to enable hot-plug interrupt
4547 * for sub devices behind port multiplier.
4548 */
4549 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4550 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4551 "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4552
4553 /*
4554 * Fetch the number of device ports of the port multiplier
4555 */
4556 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4557 return (AHCI_FAILURE);
4558
4559 /* Register the port multiplier to SATA Framework. */
4560 mutex_exit(&ahci_portp->ahciport_mutex);
4561 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4562 mutex_enter(&ahci_portp->ahciport_mutex);
4563
4564 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4565 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4566
4567 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4568 "port %d: pmult sub-port number updated to %x.", port,
4569 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4570
4571 /* Till now port-mult is successfully initialized */
4572 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4573 return (AHCI_SUCCESS);
4574
4575 err: /* R/W PMULT error */
4576 return (AHCI_FAILURE);
4577 }
4578
4579 /*
4580 * Initialize a port multiplier port. According to spec, firstly we need
4581 * issue a COMRESET, then a software reset to get its signature.
4582 *
4583 * NOTE: This function should only be called in ahci_probe_pmport()
4584 */
4585 static int
4586 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4587 ahci_addr_t *addrp)
4588 {
4589 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4590 uint8_t port = addrp->aa_port;
4591 uint8_t pmport = addrp->aa_pmport;
4592 int ret = AHCI_FAILURE;
4593
4594 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4595 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4596
4597 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4598 "ahci_initialize_pmport: port %d:%d", port, pmport);
4599
4600 /* Check HBA port state */
4601 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4602 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4603 "ahci_initialize_pmport:"
4604 "port %d:%d Port Multiplier is failed.",
4605 port, pmport);
4606 return (AHCI_FAILURE);
4607 }
4608
4609 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4610 return (AHCI_FAILURE);
4611 }
4612 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4613
4614 /* Checking for outstanding commands */
4615 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4616 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4617 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4618 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4619 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4620 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4621 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4622 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4623 }
4624
4625 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4626 ahci_portp->ahciport_mop_in_progress++;
4627
4628 /* Clear status */
4629 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4630
4631 /* Firstly assume an unknown device */
4632 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4633
4634 ahci_disable_port_intrs(ahci_ctlp, port);
4635
4636 /* port reset is necessary for port multiplier port */
4637 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4638 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4639 "ahci_initialize_pmport:"
4640 "port reset failed at port %d:%d",
4641 port, pmport);
4642 goto out;
4643 }
4644
4645 /* Is port failed? */
4646 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4647 SATA_PSTATE_FAILED) {
4648 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4649 "ahci_initialize_pmport: port %d:%d failed. "
4650 "state = 0x%x", port, pmport,
4651 ahci_portp->ahciport_port_state);
4652 goto out;
4653 }
4654
4655 /* Is there any device attached? */
4656 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4657 == SATA_DTYPE_NONE) {
4658 /* Do not waste time on an empty port */
4659 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4660 "ahci_initialize_pmport: No device is found "
4661 "at port %d:%d", port, pmport);
4662 ret = AHCI_SUCCESS;
4663 goto out;
4664 }
4665
4666 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4667 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4668 "port %d:%d is ready now.", port, pmport);
4669
4670 /*
4671 * Till now we can assure a device attached to that HBA port and work
4672 * correctly. Now try to get the device signature. This is an optional
4673 * step. If failed, unknown device is assumed, then SATA module will
4674 * continue to use IDENTIFY DEVICE to get the information of the
4675 * device.
4676 */
4677 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4678
4679 ret = AHCI_SUCCESS;
4680
4681 out:
4682 /* Next try to mop the pending commands */
4683 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4684 finished_tags = ahci_portp->ahciport_pending_tags &
4685 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4686 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4687 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4688 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4689 reset_tags &= ~finished_tags;
4690
4691 ahci_mop_commands(ahci_ctlp,
4692 ahci_portp,
4693 slot_status,
4694 0, /* failed tags */
4695 0, /* timeout tags */
4696 0, /* aborted tags */
4697 reset_tags); /* reset tags */
4698
4699 /* Clear PxSNTF register if supported. */
4700 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4701 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4702 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4703 AHCI_SNOTIF_CLEAR_ALL);
4704 }
4705
4706 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4707 ahci_enable_port_intrs(ahci_ctlp, port);
4708 return (ret);
4709 }
4710
4711 /*
4712 * ahci_probe_pmult()
4713 *
4714 * This function will be called to probe a port multiplier, which will
4715 * handle hotplug events on port multiplier ports.
4716 *
4717 * NOTE: Only called from ahci_tran_probe_port()
4718 */
4719 static int
4720 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4721 ahci_addr_t *addrp)
4722 {
4723 sata_device_t sdevice;
4724 ahci_addr_t pmport_addr;
4725 uint32_t gscr32, port_hotplug_tags;
4726 uint32_t pmport_sstatus;
4727 int dev_exists_now = 0, dev_existed_previously = 0;
4728 uint8_t port = addrp->aa_port;
4729 int npmport;
4730
4731 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4732
4733 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4734 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4735 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4736
4737 do {
4738 npmport = ddi_ffs(port_hotplug_tags) - 1;
4739 if (npmport == -1)
4740 /* no pending hot plug events. */
4741 return (AHCI_SUCCESS);
4742
4743 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4744 "hot-plug event at port %d:%d", port, npmport);
4745
4746 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4747
4748 /* Check previous device at that port */
4749 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4750 != SATA_DTYPE_NONE)
4751 dev_existed_previously = 1;
4752
4753 /* PxSStatus tells the presence of device. */
4754 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4755 &pmport_sstatus, err);
4756
4757 if (SSTATUS_GET_DET(pmport_sstatus) ==
4758 SSTATUS_DET_DEVPRE_PHYCOM)
4759 dev_exists_now = 1;
4760
4761 /*
4762 * Clear PxSERR is critical. The transition from 0 to 1 will
4763 * emit a FIS which generates an asynchronous notification
4764 * event at controller. If we fail to clear the PxSERR, the
4765 * Async Notif events will no longer be activated on this
4766 * pmport.
4767 */
4768 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4769 AHCI_SERROR_CLEAR_ALL, err);
4770
4771 bzero((void *)&sdevice, sizeof (sata_device_t));
4772 sdevice.satadev_addr.cport = ahci_ctlp->
4773 ahcictl_port_to_cport[port];
4774 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4775 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4776 sdevice.satadev_state = SATA_PSTATE_PWRON;
4777
4778 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4779 "[Existence] %d -> %d", dev_existed_previously,
4780 dev_exists_now);
4781
4782 if (dev_exists_now) {
4783 if (dev_existed_previously) {
4784 /* Link (may) not change: Exist -> Exist * */
4785 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4786 "ahci_probe_pmult: port %d:%d "
4787 "device link lost/established",
4788 port, npmport);
4789
4790 mutex_exit(&ahci_portp->ahciport_mutex);
4791 sata_hba_event_notify(
4792 ahci_ctlp->ahcictl_sata_hba_tran->
4793 sata_tran_hba_dip,
4794 &sdevice,
4795 SATA_EVNT_LINK_LOST|
4796 SATA_EVNT_LINK_ESTABLISHED);
4797 mutex_enter(&ahci_portp->ahciport_mutex);
4798 } else {
4799 /* Link change: None -> Exist */
4800 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4801 "ahci_probe_pmult: port %d:%d "
4802 "device link established", port, npmport);
4803
4804 /* Clear port state */
4805 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4806 SATA_STATE_UNKNOWN);
4807 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4808 "ahci_probe_pmult: port %d "
4809 "ahciport_port_state [Cleared].", port);
4810
4811 mutex_exit(&ahci_portp->ahciport_mutex);
4812 sata_hba_event_notify(
4813 ahci_ctlp->ahcictl_sata_hba_tran->
4814 sata_tran_hba_dip,
4815 &sdevice,
4816 SATA_EVNT_LINK_ESTABLISHED);
4817 mutex_enter(&ahci_portp->ahciport_mutex);
4818 }
4819 } else { /* No device exists now */
4820 if (dev_existed_previously) {
4821
4822 /* Link change: Exist -> None */
4823 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4824 "ahci_probe_pmult: port %d:%d "
4825 "device link lost", port, npmport);
4826
4827 /* An existing device is lost. */
4828 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4829 SATA_STATE_UNKNOWN);
4830 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4831 SATA_DTYPE_NONE);
4832
4833 mutex_exit(&ahci_portp->ahciport_mutex);
4834 sata_hba_event_notify(
4835 ahci_ctlp->ahcictl_sata_hba_tran->
4836 sata_tran_hba_dip,
4837 &sdevice,
4838 SATA_EVNT_LINK_LOST);
4839 mutex_enter(&ahci_portp->ahciport_mutex);
4840 }
4841 }
4842
4843 CLEAR_BIT(port_hotplug_tags, npmport);
4844 } while (port_hotplug_tags != 0);
4845
4846 return (AHCI_SUCCESS);
4847
4848 err: /* R/W PMULT error */
4849 return (AHCI_FAILURE);
4850 }
4851
4852 /*
4853 * Probe and initialize a port multiplier port.
4854 * A port multiplier port could only be initilaizer here.
4855 */
4856 static int
4857 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4858 ahci_addr_t *addrp, sata_device_t *sd)
4859 {
4860 uint32_t port_state;
4861 uint8_t port = addrp->aa_port;
4862 ahci_addr_t addr_pmult;
4863
4864 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4865
4866 /*
4867 * Check the parent - port multiplier first.
4868 */
4869
4870 /*
4871 * Parent port multiplier might have been removed. This event will be
4872 * ignored and failure.
4873 */
4874 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4875 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4876 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4877 "ahci_tran_probe_port: "
4878 "parent device removed, ignore event.", NULL);
4879
4880 return (AHCI_FAILURE);
4881 }
4882
4883 /* The port is ready? */
4884 port_state = ahci_portp->ahciport_port_state;
4885 if (!(port_state & SATA_STATE_READY)) {
4886 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4887 "ahci_tran_probe_port: "
4888 "parent port-mult is NOT ready.", NULL);
4889
4890 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4891 ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4892 AHCI_SUCCESS) {
4893 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4894 "ahci_tran_probe_port: "
4895 "restart port-mult failed.", NULL);
4896 return (AHCI_FAILURE);
4897 }
4898 }
4899
4900 /*
4901 * If port-mult is restarted due to some reason, we need
4902 * re-initialized the PMult.
4903 */
4904 if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
4905 /* Initialize registers on a port multiplier */
4906 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
4907 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
4908 &addr_pmult, sd) != AHCI_SUCCESS)
4909 return (AHCI_FAILURE);
4910 }
4911
4912 /*
4913 * Then we check the port-mult port
4914 */
4915 /* Is this pmport initialized? */
4916 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
4917 if (!(port_state & SATA_STATE_READY)) {
4918
4919 /* ahci_initialize_pmport() will set READY state */
4920 if (ahci_initialize_pmport(ahci_ctlp,
4921 ahci_portp, addrp) != AHCI_SUCCESS)
4922 return (AHCI_FAILURE);
4923 }
4924
4925 return (AHCI_SUCCESS);
4926 }
4927
4928 /*
4929 * AHCI device reset ...; a single device on one of the ports is reset,
4930 * but the HBA and physical communication remain intact. This is the
4931 * least intrusive.
4932 *
4933 * When issuing a software reset sequence, there should not be other
4934 * commands in the command list, so we will first clear and then re-set
4935 * PxCMD.ST to clear PxCI. And before issuing the software reset,
4936 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
4937 * cleared unless command list override (PxCMD.CLO) is supported.
4938 */
4939 static int
4940 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4941 ahci_addr_t *addrp)
4942 {
4943 ahci_fis_h2d_register_t *h2d_register_fisp;
4944 ahci_cmd_table_t *cmd_table;
4945 ahci_cmd_header_t *cmd_header;
4946 uint32_t port_cmd_status, port_cmd_issue, port_task_file;
4947 int slot, loop_count;
4948 uint8_t port = addrp->aa_port;
4949 uint8_t pmport = addrp->aa_pmport;
4950 int rval = AHCI_FAILURE;
4951
4952 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4953
4954 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
4955 "port %d:%d device software resetting (FIS)", port, pmport);
4956
4957 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
4958 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
4959 port) != AHCI_SUCCESS) {
4960 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4961 "ahci_software_reset: cannot stop HBA port %d.", port);
4962 goto out;
4963 }
4964
4965 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
4966 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4967 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4968
4969 if (port_task_file & AHCI_TFD_STS_BSY ||
4970 port_task_file & AHCI_TFD_STS_DRQ) {
4971 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
4972 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4973 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
4974 "cannot issue a software reset.", port_task_file);
4975 goto out;
4976 }
4977
4978 /*
4979 * If HBA Support CLO, as Command List Override (CAP.SCLO is
4980 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
4981 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
4982 */
4983 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4984 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
4985
4986 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4987 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4988 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4989 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
4990 port_cmd_status|AHCI_CMD_STATUS_CLO);
4991
4992 /* Waiting till PxCMD.SCLO bit is cleared */
4993 loop_count = 0;
4994 do {
4995 /* Wait for 10 millisec */
4996 drv_usecwait(AHCI_10MS_USECS);
4997
4998 /* We are effectively timing out after 1 sec. */
4999 if (loop_count++ > 100) {
5000 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5001 "SCLO time out. port %d is busy.", port);
5002 goto out;
5003 }
5004
5005 port_cmd_status =
5006 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5007 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5008 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5009
5010 /* Re-check */
5011 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5012 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5013 if (port_task_file & AHCI_TFD_STS_BSY ||
5014 port_task_file & AHCI_TFD_STS_DRQ) {
5015 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5016 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5017 port_task_file);
5018 goto out;
5019 }
5020 }
5021
5022 /* Then start port */
5023 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5024 != AHCI_SUCCESS) {
5025 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5026 "ahci_software_reset: cannot start AHCI port %d.", port);
5027 goto out;
5028 }
5029
5030 /*
5031 * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5032 * ahci_port.ahciport_pending_ncq_tags may fail
5033 * ahci_claim_free_slot(). Actually according to spec, by clearing
5034 * PxCMD.ST there is no command outstanding while executing software
5035 * reseting. Hence we directly use slot 0 instead of
5036 * ahci_claim_free_slot().
5037 */
5038 slot = 0;
5039
5040 /* Now send the first H2D Register FIS with SRST set to 1 */
5041 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5042 bzero((void *)cmd_table, ahci_cmd_table_size);
5043
5044 h2d_register_fisp =
5045 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5046
5047 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5048 SET_FIS_PMP(h2d_register_fisp, pmport);
5049 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5050
5051 /* Set Command Header in Command List */
5052 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5053 BZERO_DESCR_INFO(cmd_header);
5054 BZERO_PRD_BYTE_COUNT(cmd_header);
5055 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5056 SET_PORT_MULTI_PORT(cmd_header, pmport);
5057
5058 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5059 SET_RESET(cmd_header, 1);
5060 SET_WRITE(cmd_header, 1);
5061
5062 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5063 0,
5064 ahci_cmd_table_size,
5065 DDI_DMA_SYNC_FORDEV);
5066
5067 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5068 slot * sizeof (ahci_cmd_header_t),
5069 sizeof (ahci_cmd_header_t),
5070 DDI_DMA_SYNC_FORDEV);
5071
5072 /* Indicate to the HBA that a command is active. */
5073 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5074 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5075 (0x1 << slot));
5076
5077 loop_count = 0;
5078
5079 /* Loop till the first command is finished */
5080 do {
5081 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5082 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5083
5084 /* We are effectively timing out after 1 sec. */
5085 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5086 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5087 "the first SRST FIS is timed out, "
5088 "loop_count = %d", loop_count);
5089 goto out;
5090 }
5091 /* Wait for 10 millisec */
5092 drv_usecwait(AHCI_10MS_USECS);
5093 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5094
5095 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5096 "ahci_software_reset: 1st loop count: %d, "
5097 "port_cmd_issue = 0x%x, slot = 0x%x",
5098 loop_count, port_cmd_issue, slot);
5099
5100 /* According to ATA spec, we need wait at least 5 microsecs here. */
5101 drv_usecwait(AHCI_1MS_USECS);
5102
5103 /* Now send the second H2D Register FIS with SRST cleard to zero */
5104 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5105 bzero((void *)cmd_table, ahci_cmd_table_size);
5106
5107 h2d_register_fisp =
5108 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5109
5110 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5111 SET_FIS_PMP(h2d_register_fisp, pmport);
5112
5113 /* Set Command Header in Command List */
5114 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5115 BZERO_DESCR_INFO(cmd_header);
5116 BZERO_PRD_BYTE_COUNT(cmd_header);
5117 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5118 SET_PORT_MULTI_PORT(cmd_header, pmport);
5119
5120 SET_WRITE(cmd_header, 1);
5121
5122 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5123 0,
5124 ahci_cmd_table_size,
5125 DDI_DMA_SYNC_FORDEV);
5126
5127 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5128 slot * sizeof (ahci_cmd_header_t),
5129 sizeof (ahci_cmd_header_t),
5130 DDI_DMA_SYNC_FORDEV);
5131
5132 /* Indicate to the HBA that a command is active. */
5133 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5134 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5135 (0x1 << slot));
5136
5137 loop_count = 0;
5138
5139 /* Loop till the second command is finished */
5140 do {
5141 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5142 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5143
5144 /* We are effectively timing out after 1 sec. */
5145 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5146 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5147 "the second SRST FIS is timed out, "
5148 "loop_count = %d", loop_count);
5149 goto out;
5150 }
5151
5152 /* Wait for 10 millisec */
5153 drv_usecwait(AHCI_10MS_USECS);
5154 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5155
5156 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5157 "ahci_software_reset: 2nd loop count: %d, "
5158 "port_cmd_issue = 0x%x, slot = 0x%x",
5159 loop_count, port_cmd_issue, slot);
5160
5161 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5162 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5163 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5164 DDI_SERVICE_UNAFFECTED);
5165 goto out;
5166 }
5167
5168 rval = AHCI_SUCCESS;
5169 out:
5170 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5171 "ahci_software_reset: %s at port %d:%d",
5172 rval == AHCI_SUCCESS ? "succeed" : "failed",
5173 port, pmport);
5174
5175 return (rval);
5176 }
5177
5178 /*
5179 * AHCI port reset ...; the physical communication between the HBA and device
5180 * on a port are disabled. This is more intrusive.
5181 *
5182 * When an HBA or port reset occurs, Phy communication is going to
5183 * be re-established with the device through a COMRESET followed by the
5184 * normal out-of-band communication sequence defined in Serial ATA. At
5185 * the end of reset, the device, if working properly, will send a D2H
5186 * Register FIS, which contains the device signature. When the HBA receives
5187 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5188 * the PxSIG register with the signature.
5189 *
5190 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5191 * function is called. If not, it is assumed the interface is in hung
5192 * condition.
5193 */
5194 static int
5195 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5196 ahci_addr_t *addrp)
5197 {
5198 ahci_addr_t pmult_addr;
5199 uint32_t port_cmd_status;
5200 uint32_t port_scontrol, port_sstatus;
5201 uint32_t port_task_file;
5202 uint32_t port_state;
5203 uint8_t port = addrp->aa_port;
5204
5205 int loop_count;
5206 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5207
5208 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5209
5210 /* Target is a port multiplier port? */
5211 if (AHCI_ADDR_IS_PMPORT(addrp))
5212 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5213
5214 /* Otherwise it must be an HBA port. */
5215 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5216
5217 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5218 "Port %d port resetting...", port);
5219 ahci_portp->ahciport_port_state = 0;
5220
5221 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5222 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5223
5224 /*
5225 * According to the spec, SUD bit should be set here,
5226 * but JMicron JMB363 doesn't follow it, so print
5227 * a debug message.
5228 */
5229 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5230 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5231 "ahci_port_reset: port %d SUD bit not set", port);
5232
5233 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5234 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5235 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5236
5237 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5238 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5239 port_scontrol);
5240
5241 /* Enable PxCMD.FRE to read device */
5242 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5243 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5244 port_cmd_status|AHCI_CMD_STATUS_FRE);
5245
5246 /*
5247 * The port enters P:StartComm state, and the HBA tells the link layer
5248 * to start communication, which involves sending COMRESET to the
5249 * device. And the HBA resets PxTFD.STS to 7Fh.
5250 *
5251 * Give time for COMRESET to percolate, according to the AHCI
5252 * spec, software shall wait at least 1 millisecond before
5253 * clearing PxSCTL.DET
5254 */
5255 drv_usecwait(AHCI_1MS_USECS * 2);
5256
5257 /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5258 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5259 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5260 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5261 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5262 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5263 port_scontrol);
5264
5265 /*
5266 * When a COMINIT is received from the device, then the port enters
5267 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5268 * PxSSTS.DET to 1h to indicate a device is detected but communication
5269 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5270 * a COMINIT has been received.
5271 */
5272 /*
5273 * The DET field is valid only if IPM field indicates
5274 * that the interface is in active state.
5275 */
5276 loop_count = 0;
5277 for (;;) {
5278 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5279 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5280
5281 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5282 /*
5283 * If the interface is not active, the DET field
5284 * is considered not accurate. So we want to
5285 * continue looping.
5286 */
5287 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5288 }
5289
5290 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5291 break;
5292
5293 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5294 /*
5295 * We are effectively timing out after 0.1 sec.
5296 */
5297 break;
5298 }
5299
5300 /* Wait for 10 millisec */
5301 drv_usecwait(AHCI_10MS_USECS);
5302 }
5303
5304 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5305 "ahci_port_reset: 1st loop count: %d, "
5306 "port_sstatus = 0x%x port %d",
5307 loop_count, port_sstatus, port);
5308
5309 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5310 /*
5311 * Either the port is not active or there
5312 * is no device present.
5313 */
5314 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5315 return (AHCI_SUCCESS);
5316 }
5317
5318 /* Clear port serror register for the port */
5319 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5320 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5321 AHCI_SERROR_CLEAR_ALL);
5322
5323 /*
5324 * Devices should return a FIS contains its signature to HBA after
5325 * COMINIT signal. Check whether a D2H Register FIS is received by
5326 * polling PxTFD.STS.
5327 */
5328 loop_count = 0;
5329 for (;;) {
5330 port_task_file =
5331 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5332 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5333
5334 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5335 AHCI_TFD_STS_ERR)) == 0)
5336 break;
5337
5338 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5339 /*
5340 * We are effectively timing out after 11 sec.
5341 */
5342 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5343 "the device hardware has been initialized and "
5344 "the power-up diagnostics failed",
5345 instance, port);
5346
5347 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5348 "port %d: some or all of BSY, DRQ and ERR in "
5349 "PxTFD.STS are not clear. We need another "
5350 "software reset.", port);
5351
5352 /* Clear port serror register for the port */
5353 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5354 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5355 AHCI_SERROR_CLEAR_ALL);
5356
5357 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5358
5359 /* Try another software reset. */
5360 if (ahci_software_reset(ahci_ctlp, ahci_portp,
5361 &pmult_addr) != AHCI_SUCCESS) {
5362 AHCIPORT_SET_STATE(ahci_portp, addrp,
5363 SATA_PSTATE_FAILED);
5364 return (AHCI_FAILURE);
5365 }
5366 break;
5367 }
5368
5369 /* Wait for 10 millisec */
5370 drv_usecwait(AHCI_10MS_USECS);
5371 }
5372
5373 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5374 "ahci_port_reset: 2nd loop count: %d, "
5375 "port_task_file = 0x%x port %d",
5376 loop_count, port_task_file, port);
5377
5378 /* Clear port serror register for the port */
5379 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5380 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5381 AHCI_SERROR_CLEAR_ALL);
5382
5383 /* Set port as ready */
5384 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5385 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5386
5387 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5388 "ahci_port_reset: succeed at port %d.", port);
5389 return (AHCI_SUCCESS);
5390 }
5391
5392 /*
5393 * COMRESET on a port multiplier port.
5394 *
5395 * NOTE: Only called in ahci_port_reset()
5396 */
5397 static int
5398 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5399 ahci_addr_t *addrp)
5400 {
5401 uint32_t port_scontrol, port_sstatus, port_serror;
5402 uint32_t port_cmd_status, port_intr_status;
5403 uint32_t port_state;
5404 uint8_t port = addrp->aa_port;
5405 uint8_t pmport = addrp->aa_pmport;
5406 int loop_count;
5407 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5408
5409 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5410 "port %d:%d: pmport resetting", port, pmport);
5411
5412 /* Initialize pmport state */
5413 AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5414
5415 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5416 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5417 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5418
5419 /* PxCMD.FRE should be set before. */
5420 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5421 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5422 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5423 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5424 return (AHCI_FAILURE);
5425
5426 /*
5427 * Give time for COMRESET to percolate, according to the AHCI
5428 * spec, software shall wait at least 1 millisecond before
5429 * clearing PxSCTL.DET
5430 */
5431 drv_usecwait(AHCI_1MS_USECS*2);
5432
5433 /*
5434 * Fetch the SCONTROL again and rewrite the DET part with 0
5435 * This will generate an Asychronous Notification events.
5436 */
5437 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5438 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5439 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5440
5441 /*
5442 * The port enters P:StartComm state, and HBA tells link layer to
5443 * start communication, which involves sending COMRESET to device.
5444 * And the HBA resets PxTFD.STS to 7Fh.
5445 *
5446 * When a COMINIT is received from the device, then the port enters
5447 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5448 * PxSSTS.DET to 1h to indicate a device is detected but communication
5449 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5450 * a COMINIT has been received.
5451 */
5452 /*
5453 * The DET field is valid only if IPM field indicates
5454 * that the interface is in active state.
5455 */
5456 loop_count = 0;
5457 do {
5458 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5459
5460 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5461 /*
5462 * If the interface is not active, the DET field
5463 * is considered not accurate. So we want to
5464 * continue looping.
5465 */
5466 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5467 }
5468
5469 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5470 /*
5471 * We are effectively timing out after 0.1 sec.
5472 */
5473 break;
5474 }
5475
5476 /* Wait for 10 millisec */
5477 drv_usecwait(AHCI_10MS_USECS);
5478
5479 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5480
5481 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5482 "ahci_pmport_reset: 1st loop count: %d, "
5483 "port_sstatus = 0x%x port %d:%d",
5484 loop_count, port_sstatus, port, pmport);
5485
5486 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5487 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5488 /*
5489 * Either the port is not active or there
5490 * is no device present.
5491 */
5492 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5493 "ahci_pmport_reset: "
5494 "no device attached to port %d:%d",
5495 port, pmport);
5496 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5497 return (AHCI_SUCCESS);
5498 }
5499
5500 /* Now we can make sure there is a device connected to the port */
5501 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5502 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5503
5504 if (!(port_serror & (1 << 26))) {
5505 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5506 "COMINIT signal from the device not received port %d:%d",
5507 instance, port, pmport);
5508
5509 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5510 return (AHCI_FAILURE);
5511 }
5512
5513 /*
5514 * After clear PxSERR register, we will receive a D2H FIS.
5515 * Normally this FIS will cause a IPMS error according to AHCI spec
5516 * v1.2 because there is no command outstanding for it. So we need
5517 * to ignore this error.
5518 */
5519 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5520 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5521
5522 /* Now we need to check the D2H FIS by checking IPMS error. */
5523 loop_count = 0;
5524 do {
5525 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5526 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5527
5528 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5529 /*
5530 * No D2H FIS received. This is possible according
5531 * to SATA 2.6 spec.
5532 */
5533 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5534 "PxIS.IPMS is not set, we need another "
5535 "software reset.", port, pmport);
5536
5537 break;
5538 }
5539
5540 /* Wait for 10 millisec */
5541 mutex_exit(&ahci_portp->ahciport_mutex);
5542 delay(AHCI_10MS_TICKS);
5543 mutex_enter(&ahci_portp->ahciport_mutex);
5544
5545 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5546
5547 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5548 "ahci_pmport_reset: 2st loop count: %d, "
5549 "port_sstatus = 0x%x port %d:%d",
5550 loop_count, port_sstatus, port, pmport);
5551
5552 /* Clear IPMS */
5553 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5554 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5555 AHCI_INTR_STATUS_IPMS);
5556 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5557
5558 /* This pmport is now ready for ahci_tran_start() */
5559 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5560 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5561
5562 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5563 "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5564 return (AHCI_SUCCESS);
5565
5566 err: /* R/W PMULT error */
5567 /* IPMS flags might be set before. */
5568 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5569 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5570 "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5571
5572 return (AHCI_FAILURE);
5573 }
5574
5575 /*
5576 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5577 * This is the most intrusive.
5578 *
5579 * When an HBA reset occurs, Phy communication will be re-established with
5580 * the device through a COMRESET followed by the normal out-of-band
5581 * communication sequence defined in Serial ATA. At the end of reset, the
5582 * device, if working properly, will send a D2H Register FIS, which contains
5583 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5584 * and PxTFD.ERR register fields, and updates the PxSIG register with the
5585 * signature.
5586 *
5587 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5588 */
5589 static int
5590 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5591 {
5592 ahci_port_t *ahci_portp;
5593 uint32_t ghc_control;
5594 uint8_t port;
5595 int loop_count;
5596 int rval = AHCI_SUCCESS;
5597
5598 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5599 NULL);
5600
5601 mutex_enter(&ahci_ctlp->ahcictl_mutex);
5602
5603 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5604 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5605
5606 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5607 ghc_control |= AHCI_HBA_GHC_HR;
5608 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5609 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5610
5611 /*
5612 * Wait until HBA Reset complete or timeout
5613 */
5614 loop_count = 0;
5615 do {
5616 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5617 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5618
5619 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5620 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5621 "ahci hba reset is timing out, "
5622 "ghc_control = 0x%x", ghc_control);
5623 /* We are effectively timing out after 1 sec. */
5624 break;
5625 }
5626
5627 /* Wait for 10 millisec */
5628 drv_usecwait(AHCI_10MS_USECS);
5629 } while (ghc_control & AHCI_HBA_GHC_HR);
5630
5631 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5632 "ahci_hba_reset: 1st loop count: %d, "
5633 "ghc_control = 0x%x", loop_count, ghc_control);
5634
5635 if (ghc_control & AHCI_HBA_GHC_HR) {
5636 /* The hba is not reset for some reasons */
5637 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5638 "hba reset failed: HBA in a hung or locked state", NULL);
5639 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5640 return (AHCI_FAILURE);
5641 }
5642
5643 /*
5644 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5645 */
5646 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5647 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5648 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5649 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5650 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5651
5652 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5653
5654 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5655 /* Only check implemented ports */
5656 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5657 continue;
5658 }
5659
5660 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5661 mutex_enter(&ahci_portp->ahciport_mutex);
5662
5663 /* Make sure the drive is spun-up */
5664 ahci_staggered_spin_up(ahci_ctlp, port);
5665
5666 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5667 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5668 AHCI_SUCCESS) {
5669 rval = AHCI_FAILURE;
5670 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5671 "ahci_hba_reset: port %d failed", port);
5672 /*
5673 * Set the port state to SATA_PSTATE_FAILED if
5674 * failed to initialize it.
5675 */
5676 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5677 }
5678
5679 mutex_exit(&ahci_portp->ahciport_mutex);
5680 }
5681
5682 return (rval);
5683 }
5684
5685 /*
5686 * This routine is only called from AHCI_ATTACH or phyrdy change
5687 * case. It first calls software reset, then stop the port and try to
5688 * read PxSIG register to find the type of device attached to the port.
5689 *
5690 * The caller should make sure a valid device exists on specified port and
5691 * physical communication has been established so that the signature could
5692 * be retrieved by software reset.
5693 *
5694 * NOTE: The port interrupts should be disabled before the function is called.
5695 */
5696 static void
5697 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5698 ahci_addr_t *addrp)
5699 {
5700 ahci_addr_t dev_addr;
5701 uint32_t signature;
5702 uint8_t port = addrp->aa_port;
5703 uint8_t pmport = addrp->aa_pmport;
5704 int rval;
5705
5706 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5707 ASSERT(AHCI_ADDR_IS_VALID(addrp));
5708
5709 /*
5710 * If the HBA doesn't support port multiplier, then the driver
5711 * doesn't need to bother to check port multiplier device.
5712 *
5713 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5714 * connected to Silicon Image 4723, to which the two sata drives
5715 * attached can be set with RAID1, RAID0 or Spanning mode.
5716 *
5717 * We found software reset will get failure if port multiplier address
5718 * 0xf is used by software reset, so just ignore the check since
5719 * ICH7 doesn't support port multiplier device at all.
5720 */
5721 if (AHCI_ADDR_IS_PORT(addrp) &&
5722 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5723 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5724 "ahci_find_dev_signature enter: port %d", port);
5725
5726 /*
5727 * NOTE: when the ahci address is a HBA port, we do not know
5728 * it is a device or a port multiplier that attached. we need
5729 * try a software reset at port multiplier address (0xf
5730 * pmport)
5731 */
5732 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5733 } else {
5734 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5735 "ahci_find_dev_signature enter: port %d:%d",
5736 port, pmport);
5737 dev_addr = *addrp;
5738 }
5739
5740 /* Assume it is unknown. */
5741 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5742
5743 /* Issue a software reset to get the signature */
5744 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5745 if (rval != AHCI_SUCCESS) {
5746
5747 /*
5748 * Try to do software reset again with pmport set with 0 if
5749 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5750 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5751 */
5752 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5753 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5754 dev_addr.aa_pmport = 0;
5755 rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5756 &dev_addr);
5757 }
5758
5759 if (rval != AHCI_SUCCESS) {
5760 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5761 "ahci_find_dev_signature: software reset failed "
5762 "at port %d:%d, cannot get signature.",
5763 port, pmport);
5764
5765 AHCIPORT_SET_STATE(ahci_portp, addrp,
5766 SATA_PSTATE_FAILED);
5767 return;
5768 }
5769 }
5770
5771 /*
5772 * ahci_software_reset has started the port, so we need manually stop
5773 * the port again.
5774 */
5775 if (AHCI_ADDR_IS_PORT(addrp)) {
5776 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5777 ahci_portp, port) != AHCI_SUCCESS) {
5778 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5779 "ahci_find_dev_signature: cannot stop port %d.",
5780 port);
5781 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5782 return;
5783 }
5784 }
5785
5786 /* Now we can make sure that a valid signature is received. */
5787 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5788 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5789
5790 if (AHCI_ADDR_IS_PMPORT(addrp)) {
5791 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5792 "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5793 signature, port, pmport);
5794 } else {
5795 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5796 "ahci_find_dev_signature: signature = 0x%x at port %d",
5797 signature, port);
5798 }
5799
5800 /* NOTE: Only support ATAPI device at controller port. */
5801 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5802 signature = SATA_DTYPE_UNKNOWN;
5803
5804 switch (signature) {
5805
5806 case AHCI_SIGNATURE_DISK:
5807 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5808 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5809 "Disk is found at port: %d", port);
5810 break;
5811
5812 case AHCI_SIGNATURE_ATAPI:
5813 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5814 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5815 "ATAPI device is found at port: %d", port);
5816 break;
5817
5818 case AHCI_SIGNATURE_PORT_MULTIPLIER:
5819 /* Port Multiplier cannot recursively attached. */
5820 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5821 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5822 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5823 "Port Multiplier is found at port: %d", port);
5824 break;
5825
5826 default:
5827 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5828 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5829 "Unknown device is found at port: %d", port);
5830 }
5831 }
5832
5833 /*
5834 * According to the spec, to reliably detect hot plug removals, software
5835 * must disable interface power management. Software should perform the
5836 * following initialization on a port after a device is attached:
5837 * Set PxSCTL.IPM to 3h to disable interface state transitions
5838 * Set PxCMD.ALPE to '0' to disable aggressive power management
5839 * Disable device initiated interface power management by SET FEATURE
5840 *
5841 * We can ignore the last item because by default the feature is disabled
5842 */
5843 static void
5844 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5845 {
5846 uint32_t port_scontrol, port_cmd_status;
5847
5848 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5849 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5850 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5851 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5852 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5853
5854 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5855 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5856 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5857 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5858 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5859 }
5860
5861 /*
5862 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5863 * to 1, then set it firstly.
5864 *
5865 * Each port contains two major DMA engines. One DMA engine walks through
5866 * the command list, and is controlled by PxCMD.ST. The second DMA engine
5867 * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5868 *
5869 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5870 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5871 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5872 *
5873 * Software shall not set PxCMD.ST to '1' unless a functional device is
5874 * present on the port(as determined by PxTFD.STS.BSY = '0',
5875 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5876 */
5877 static int
5878 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5879 {
5880 uint32_t port_cmd_status;
5881
5882 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5883
5884 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5885
5886 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5887 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5888 "the state for port %d is 0x%x",
5889 port, ahci_portp->ahciport_port_state);
5890 return (AHCI_FAILURE);
5891 }
5892
5893 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5894 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5895 "no device is attached at port %d", port);
5896 return (AHCI_FAILURE);
5897 }
5898
5899 /* First to set PxCMD.FRE before setting PxCMD.ST. */
5900 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5901 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5902
5903 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
5904 port_cmd_status |= AHCI_CMD_STATUS_FRE;
5905 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5906 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5907 port_cmd_status);
5908 }
5909
5910 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5911 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5912
5913 port_cmd_status |= AHCI_CMD_STATUS_ST;
5914
5915 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5916 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5917 port_cmd_status);
5918
5919 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
5920
5921 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
5922 "PxCMD.ST set to '1' at port %d", port);
5923
5924 return (AHCI_SUCCESS);
5925 }
5926
5927 /*
5928 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
5929 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
5930 * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
5931 */
5932 static int
5933 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
5934 {
5935 uint8_t port = ahci_portp->ahciport_port_num;
5936 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5937 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5938
5939 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5940
5941 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
5942 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
5943 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5944 port) != AHCI_SUCCESS)
5945 return (AHCI_FAILURE);
5946
5947 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5948 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5949 }
5950
5951 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
5952 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
5953 int loop_count = 0;
5954
5955 /* Clear PxCMD.FRE */
5956 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
5957 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5958 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5959 port_cmd_status);
5960
5961 /* Wait until PxCMD.FR is cleared */
5962 for (;;) {
5963 port_cmd_status =
5964 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5965 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5966
5967 if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
5968 break;
5969
5970 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
5971 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
5972 "ahci_setup_port_base_addresses: cannot "
5973 "clear PxCMD.FR for port %d.", port);
5974
5975 /*
5976 * We are effectively timing out after 0.5 sec.
5977 * This value is specified in AHCI spec.
5978 */
5979 return (AHCI_FAILURE);
5980 }
5981
5982 /* Wait for 1 millisec */
5983 drv_usecwait(AHCI_1MS_USECS);
5984 }
5985 }
5986
5987 /* Step 3: Config Port Command List Base Address */
5988 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5989 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
5990 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
5991
5992 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5993 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
5994 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
5995
5996 /* Step 4: Config Port Received FIS Base Address */
5997 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5998 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
5999 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6000
6001 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6002 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6003 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6004
6005 return (AHCI_SUCCESS);
6006 }
6007
6008 /*
6009 * Allocate the ahci_port_t including Received FIS and Command List.
6010 * The argument - port is the physical port number, and not logical
6011 * port number seen by the SATA framework.
6012 */
6013 static int
6014 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6015 {
6016 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6017 ahci_port_t *ahci_portp;
6018 char taskq_name[64] = "event_handle_taskq";
6019
6020 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6021
6022 ahci_portp =
6023 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6024
6025 ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6026 ahci_portp->ahciport_port_num = port;
6027
6028 /* Initialize the port condition variable */
6029 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6030
6031 /* Initialize the port mutex */
6032 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6033 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6034
6035 mutex_enter(&ahci_portp->ahciport_mutex);
6036
6037 /*
6038 * Allocate memory for received FIS structure and
6039 * command list for this port
6040 */
6041 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6042 goto err_case1;
6043 }
6044
6045 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6046 goto err_case2;
6047 }
6048
6049 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6050 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6051 AHCI_SUCCESS) {
6052 goto err_case3;
6053 }
6054
6055 (void) snprintf(taskq_name + strlen(taskq_name),
6056 sizeof (taskq_name) - strlen(taskq_name),
6057 "_port%d", port);
6058
6059 /* Create the taskq for the port */
6060 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6061 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6062 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6063 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6064 goto err_case3;
6065 }
6066
6067 /* Allocate the argument for the taskq */
6068 ahci_portp->ahciport_event_args =
6069 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6070
6071 ahci_portp->ahciport_event_args->ahciea_addrp =
6072 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6073
6074 if (ahci_portp->ahciport_event_args == NULL)
6075 goto err_case4;
6076
6077 /* Initialize the done queue */
6078 ahci_portp->ahciport_doneq = NULL;
6079 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6080 ahci_portp->ahciport_doneq_len = 0;
6081
6082 mutex_exit(&ahci_portp->ahciport_mutex);
6083
6084 return (AHCI_SUCCESS);
6085
6086 err_case4:
6087 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6088
6089 err_case3:
6090 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6091
6092 err_case2:
6093 ahci_dealloc_rcvd_fis(ahci_portp);
6094
6095 err_case1:
6096 mutex_exit(&ahci_portp->ahciport_mutex);
6097 mutex_destroy(&ahci_portp->ahciport_mutex);
6098 cv_destroy(&ahci_portp->ahciport_cv);
6099
6100 kmem_free(ahci_portp, sizeof (ahci_port_t));
6101
6102 return (AHCI_FAILURE);
6103 }
6104
6105 /*
6106 * Reverse of ahci_alloc_port_state().
6107 */
6108 static void
6109 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6110 {
6111 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6112
6113 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6114 ASSERT(ahci_portp != NULL);
6115
6116 mutex_enter(&ahci_portp->ahciport_mutex);
6117 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6118 sizeof (ahci_addr_t));
6119 ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6120 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6121 ahci_portp->ahciport_event_args = NULL;
6122 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6123 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6124 ahci_dealloc_rcvd_fis(ahci_portp);
6125 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6126 mutex_exit(&ahci_portp->ahciport_mutex);
6127
6128 mutex_destroy(&ahci_portp->ahciport_mutex);
6129 cv_destroy(&ahci_portp->ahciport_cv);
6130
6131 kmem_free(ahci_portp, sizeof (ahci_port_t));
6132
6133 ahci_ctlp->ahcictl_ports[port] = NULL;
6134 }
6135
6136 /*
6137 * Allocates memory for the Received FIS Structure
6138 */
6139 static int
6140 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6141 {
6142 size_t rcvd_fis_size;
6143 size_t ret_len;
6144 uint_t cookie_count;
6145
6146 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6147
6148 rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6149
6150 /* allocate rcvd FIS dma handle. */
6151 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6152 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6153 DDI_DMA_SLEEP,
6154 NULL,
6155 &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6156 DDI_SUCCESS) {
6157 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6158 "rcvd FIS dma handle alloc failed", NULL);
6159
6160 return (AHCI_FAILURE);
6161 }
6162
6163 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6164 rcvd_fis_size,
6165 &accattr,
6166 DDI_DMA_CONSISTENT,
6167 DDI_DMA_SLEEP,
6168 NULL,
6169 (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6170 &ret_len,
6171 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6172
6173 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6174 "rcvd FIS dma mem alloc fail", NULL);
6175 /* error.. free the dma handle. */
6176 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6177 return (AHCI_FAILURE);
6178 }
6179
6180 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6181 NULL,
6182 (caddr_t)ahci_portp->ahciport_rcvd_fis,
6183 rcvd_fis_size,
6184 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6185 DDI_DMA_SLEEP,
6186 NULL,
6187 &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6188 &cookie_count) != DDI_DMA_MAPPED) {
6189
6190 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6191 "rcvd FIS dma handle bind fail", NULL);
6192 /* error.. free the dma handle & free the memory. */
6193 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6194 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6195 return (AHCI_FAILURE);
6196 }
6197
6198 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6199
6200 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6201 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6202 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6203 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6204
6205 return (AHCI_SUCCESS);
6206 }
6207
6208 /*
6209 * Deallocates the Received FIS Structure
6210 */
6211 static void
6212 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6213 {
6214 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6215
6216 /* Unbind the cmd list dma handle first. */
6217 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6218
6219 /* Then free the underlying memory. */
6220 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6221
6222 /* Now free the handle itself. */
6223 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6224 }
6225
6226 /*
6227 * Allocates memory for the Command List, which contains up to 32 entries.
6228 * Each entry contains a command header, which is a 32-byte structure that
6229 * includes the pointer to the command table.
6230 */
6231 static int
6232 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6233 {
6234 size_t cmd_list_size;
6235 size_t ret_len;
6236 uint_t cookie_count;
6237
6238 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6239
6240 cmd_list_size =
6241 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6242
6243 /* allocate cmd list dma handle. */
6244 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6245 &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6246 DDI_DMA_SLEEP,
6247 NULL,
6248 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6249
6250 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6251 "cmd list dma handle alloc failed", NULL);
6252 return (AHCI_FAILURE);
6253 }
6254
6255 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6256 cmd_list_size,
6257 &accattr,
6258 DDI_DMA_CONSISTENT,
6259 DDI_DMA_SLEEP,
6260 NULL,
6261 (caddr_t *)&ahci_portp->ahciport_cmd_list,
6262 &ret_len,
6263 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6264
6265 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6266 "cmd list dma mem alloc fail", NULL);
6267 /* error.. free the dma handle. */
6268 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6269 return (AHCI_FAILURE);
6270 }
6271
6272 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6273 NULL,
6274 (caddr_t)ahci_portp->ahciport_cmd_list,
6275 cmd_list_size,
6276 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6277 DDI_DMA_SLEEP,
6278 NULL,
6279 &ahci_portp->ahciport_cmd_list_dma_cookie,
6280 &cookie_count) != DDI_DMA_MAPPED) {
6281
6282 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6283 "cmd list dma handle bind fail", NULL);
6284 /* error.. free the dma handle & free the memory. */
6285 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6286 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6287 return (AHCI_FAILURE);
6288 }
6289
6290 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6291
6292 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6293 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6294
6295 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6296 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6297
6298 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6299 goto err_out;
6300 }
6301
6302 return (AHCI_SUCCESS);
6303
6304 err_out:
6305 /* Unbind the cmd list dma handle first. */
6306 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6307
6308 /* Then free the underlying memory. */
6309 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6310
6311 /* Now free the handle itself. */
6312 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6313
6314 return (AHCI_FAILURE);
6315 }
6316
6317 /*
6318 * Deallocates the Command List
6319 */
6320 static void
6321 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6322 {
6323 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6324
6325 /* First dealloc command table */
6326 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6327
6328 /* Unbind the cmd list dma handle first. */
6329 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6330
6331 /* Then free the underlying memory. */
6332 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6333
6334 /* Now free the handle itself. */
6335 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6336 }
6337
6338 /*
6339 * Allocates memory for all Command Tables, which contains Command FIS,
6340 * ATAPI Command and Physical Region Descriptor Table.
6341 */
6342 static int
6343 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6344 {
6345 size_t ret_len;
6346 ddi_dma_cookie_t cmd_table_dma_cookie;
6347 uint_t cookie_count;
6348 int slot;
6349
6350 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6351
6352 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6353 "ahci_alloc_cmd_tables: port %d enter",
6354 ahci_portp->ahciport_port_num);
6355
6356 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6357 /* Allocate cmd table dma handle. */
6358 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6359 &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6360 DDI_DMA_SLEEP,
6361 NULL,
6362 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6363 DDI_SUCCESS) {
6364
6365 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6366 "cmd table dma handle alloc failed", NULL);
6367
6368 goto err_out;
6369 }
6370
6371 if (ddi_dma_mem_alloc(
6372 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6373 ahci_cmd_table_size,
6374 &accattr,
6375 DDI_DMA_CONSISTENT,
6376 DDI_DMA_SLEEP,
6377 NULL,
6378 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6379 &ret_len,
6380 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6381 NULL) {
6382
6383 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6384 "cmd table dma mem alloc fail", NULL);
6385
6386 /* error.. free the dma handle. */
6387 ddi_dma_free_handle(
6388 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6389 goto err_out;
6390 }
6391
6392 if (ddi_dma_addr_bind_handle(
6393 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6394 NULL,
6395 (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6396 ahci_cmd_table_size,
6397 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6398 DDI_DMA_SLEEP,
6399 NULL,
6400 &cmd_table_dma_cookie,
6401 &cookie_count) != DDI_DMA_MAPPED) {
6402
6403 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6404 "cmd table dma handle bind fail", NULL);
6405 /* error.. free the dma handle & free the memory. */
6406 ddi_dma_mem_free(
6407 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6408 ddi_dma_free_handle(
6409 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6410 goto err_out;
6411 }
6412
6413 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6414 ahci_cmd_table_size);
6415
6416 /* Config Port Command Table Base Address */
6417 SET_COMMAND_TABLE_BASE_ADDR(
6418 (&ahci_portp->ahciport_cmd_list[slot]),
6419 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6420
6421 #ifndef __lock_lint
6422 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6423 (&ahci_portp->ahciport_cmd_list[slot]),
6424 cmd_table_dma_cookie.dmac_laddress >> 32);
6425 #endif
6426 }
6427
6428 return (AHCI_SUCCESS);
6429 err_out:
6430
6431 for (slot--; slot >= 0; slot--) {
6432 /* Unbind the cmd table dma handle first */
6433 (void) ddi_dma_unbind_handle(
6434 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6435
6436 /* Then free the underlying memory */
6437 ddi_dma_mem_free(
6438 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6439
6440 /* Now free the handle itself */
6441 ddi_dma_free_handle(
6442 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6443 }
6444
6445 return (AHCI_FAILURE);
6446 }
6447
6448 /*
6449 * Deallocates memory for all Command Tables.
6450 */
6451 static void
6452 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6453 {
6454 int slot;
6455
6456 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6457
6458 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6459 "ahci_dealloc_cmd_tables: %d enter",
6460 ahci_portp->ahciport_port_num);
6461
6462 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6463 /* Unbind the cmd table dma handle first. */
6464 (void) ddi_dma_unbind_handle(
6465 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6466
6467 /* Then free the underlying memory. */
6468 ddi_dma_mem_free(
6469 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6470
6471 /* Now free the handle itself. */
6472 ddi_dma_free_handle(
6473 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6474 }
6475 }
6476
6477 /*
6478 * Update SATA registers at controller ports
6479 */
6480 static void
6481 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6482 sata_device_t *sd)
6483 {
6484 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6485
6486 sd->satadev_scr.sstatus =
6487 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6488 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6489 sd->satadev_scr.serror =
6490 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6491 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6492 sd->satadev_scr.scontrol =
6493 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6494 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6495 sd->satadev_scr.sactive =
6496 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6497 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6498 }
6499
6500 /*
6501 * For poll mode, ahci_port_intr will be called to emulate the interrupt
6502 */
6503 static void
6504 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6505 {
6506 uint32_t port_intr_status;
6507 uint32_t port_intr_enable;
6508
6509 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6510 "ahci_port_intr enter: port %d", port);
6511
6512 mutex_enter(&ahci_portp->ahciport_mutex);
6513 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6514 /* For SATA_OPMODE_POLLING commands */
6515 port_intr_enable =
6516 (AHCI_INTR_STATUS_DHRS |
6517 AHCI_INTR_STATUS_PSS |
6518 AHCI_INTR_STATUS_SDBS |
6519 AHCI_INTR_STATUS_UFS |
6520 AHCI_INTR_STATUS_PCS |
6521 AHCI_INTR_STATUS_PRCS |
6522 AHCI_INTR_STATUS_OFS |
6523 AHCI_INTR_STATUS_INFS |
6524 AHCI_INTR_STATUS_IFS |
6525 AHCI_INTR_STATUS_HBDS |
6526 AHCI_INTR_STATUS_HBFS |
6527 AHCI_INTR_STATUS_TFES);
6528 } else {
6529 /*
6530 * port_intr_enable indicates that the corresponding interrrupt
6531 * reporting is enabled.
6532 */
6533 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6534 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6535 }
6536
6537 /* IPMS error in port reset should be ignored according AHCI spec. */
6538 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6539 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6540 mutex_exit(&ahci_portp->ahciport_mutex);
6541
6542 /*
6543 * port_intr_stats indicates that the corresponding interrupt
6544 * condition is active.
6545 */
6546 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6547 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6548
6549 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6550 "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6551 "port_intr_enable = 0x%x",
6552 port, port_intr_status, port_intr_enable);
6553
6554 port_intr_status &= port_intr_enable;
6555
6556 /*
6557 * Pending interrupt events are indicated by the PxIS register.
6558 * Make sure we don't miss any event.
6559 */
6560 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6561 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6562 DDI_SERVICE_UNAFFECTED);
6563 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6564 DDI_FME_VERSION);
6565 return;
6566 }
6567
6568 /* First clear the port interrupts status */
6569 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6570 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6571 port_intr_status);
6572
6573 /* Check the completed non-queued commands */
6574 if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6575 AHCI_INTR_STATUS_PSS)) {
6576 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6577 ahci_portp, port);
6578 }
6579
6580 /* Check the completed queued commands */
6581 if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6582 (void) ahci_intr_set_device_bits(ahci_ctlp,
6583 ahci_portp, port);
6584 }
6585
6586 /* Check the port connect change status interrupt bit */
6587 if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6588 (void) ahci_intr_port_connect_change(ahci_ctlp,
6589 ahci_portp, port);
6590 }
6591
6592 /* Check the device mechanical presence status interrupt bit */
6593 if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6594 (void) ahci_intr_device_mechanical_presence_status(
6595 ahci_ctlp, ahci_portp, port);
6596 }
6597
6598 /* Check the PhyRdy change status interrupt bit */
6599 if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6600 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6601 port);
6602 }
6603
6604 /*
6605 * Check the non-fatal error interrupt bits, there are four
6606 * kinds of non-fatal errors at the time being:
6607 *
6608 * PxIS.UFS - Unknown FIS Error
6609 * PxIS.OFS - Overflow Error
6610 * PxIS.INFS - Interface Non-Fatal Error
6611 * PxIS.IPMS - Incorrect Port Multiplier Status Error
6612 *
6613 * For these non-fatal errors, the HBA can continue to operate,
6614 * so the driver just log the error messages.
6615 */
6616 if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6617 AHCI_INTR_STATUS_OFS |
6618 AHCI_INTR_STATUS_IPMS |
6619 AHCI_INTR_STATUS_INFS)) {
6620 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6621 port, port_intr_status);
6622 }
6623
6624 /*
6625 * Check the fatal error interrupt bits, there are four kinds
6626 * of fatal errors for AHCI controllers:
6627 *
6628 * PxIS.HBFS - Host Bus Fatal Error
6629 * PxIS.HBDS - Host Bus Data Error
6630 * PxIS.IFS - Interface Fatal Error
6631 * PxIS.TFES - Task File Error
6632 *
6633 * The fatal error means the HBA can not recover from it by
6634 * itself, and it will try to abort the transfer, and the software
6635 * must intervene to restart the port.
6636 */
6637 if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6638 AHCI_INTR_STATUS_HBDS |
6639 AHCI_INTR_STATUS_HBFS |
6640 AHCI_INTR_STATUS_TFES))
6641 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6642 port, port_intr_status);
6643
6644 /* Check the cold port detect interrupt bit */
6645 if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6646 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6647 }
6648
6649 /* Second clear the corresponding bit in IS.IPS */
6650 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6651 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6652
6653 /* Try to recover at the end of the interrupt handler. */
6654 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6655 DDI_FM_OK) {
6656 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6657 DDI_SERVICE_UNAFFECTED);
6658 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6659 DDI_FME_VERSION);
6660 }
6661 }
6662
6663 /*
6664 * Interrupt service handler
6665 */
6666 static uint_t
6667 ahci_intr(caddr_t arg1, caddr_t arg2)
6668 {
6669 #ifndef __lock_lint
6670 _NOTE(ARGUNUSED(arg2))
6671 #endif
6672 /* LINTED */
6673 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6674 ahci_port_t *ahci_portp;
6675 int32_t global_intr_status;
6676 uint8_t port;
6677
6678 /*
6679 * global_intr_status indicates that the corresponding port has
6680 * an interrupt pending.
6681 */
6682 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6683 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6684
6685 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6686 /* The interrupt is not ours */
6687 return (DDI_INTR_UNCLAIMED);
6688 }
6689
6690 /*
6691 * Check the handle after reading global_intr_status - we don't want
6692 * to miss any port with pending interrupts.
6693 */
6694 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6695 DDI_FM_OK) {
6696 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6697 DDI_SERVICE_UNAFFECTED);
6698 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6699 DDI_FME_VERSION);
6700 return (DDI_INTR_UNCLAIMED);
6701 }
6702
6703 /* Loop for all the ports */
6704 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6705 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6706 continue;
6707 }
6708 if (!((0x1 << port) & global_intr_status)) {
6709 continue;
6710 }
6711
6712 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6713
6714 /* Call ahci_port_intr */
6715 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6716 }
6717
6718 return (DDI_INTR_CLAIMED);
6719 }
6720
6721 /*
6722 * For non-queued commands, when the corresponding bit in the PxCI register
6723 * is cleared, it means the command is completed successfully. And according
6724 * to the HBA state machine, there are three conditions which possibly will
6725 * try to clear the PxCI register bit.
6726 * 1. Receive one D2H Register FIS which is with 'I' bit set
6727 * 2. Update PIO Setup FIS
6728 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6729 *
6730 * Process completed non-queued commands when the interrupt status bit -
6731 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6732 *
6733 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6734 * with the 'I' bit set. And the following commands will send thus
6735 * FIS with 'I' bit set upon the successful completion:
6736 * 1. Non-data commands
6737 * 2. DMA data-in command
6738 * 3. DMA data-out command
6739 * 4. PIO data-out command
6740 * 5. PACKET non-data commands
6741 * 6. PACKET PIO data-in command
6742 * 7. PACKET PIO data-out command
6743 * 8. PACKET DMA data-in command
6744 * 9. PACKET DMA data-out command
6745 *
6746 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6747 * with the 'I' bit set. And the following commands will send this
6748 * FIS upon the successful completion:
6749 * 1. PIO data-in command
6750 */
6751 static int
6752 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6753 ahci_port_t *ahci_portp, uint8_t port)
6754 {
6755 uint32_t port_cmd_issue = 0;
6756 uint32_t finished_tags;
6757 int finished_slot;
6758 sata_pkt_t *satapkt;
6759 ahci_fis_d2h_register_t *rcvd_fisp;
6760 #if AHCI_DEBUG
6761 ahci_cmd_header_t *cmd_header;
6762 uint32_t cmd_dmacount;
6763 #endif
6764
6765 mutex_enter(&ahci_portp->ahciport_mutex);
6766
6767 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6768 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6769 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6770 /*
6771 * Spurious interrupt. Nothing to be done.
6772 */
6773 mutex_exit(&ahci_portp->ahciport_mutex);
6774 return (AHCI_SUCCESS);
6775 }
6776
6777 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6778 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6779
6780 /* If the PxCI corrupts, don't complete the commmands. */
6781 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6782 != DDI_FM_OK) {
6783 mutex_exit(&ahci_portp->ahciport_mutex);
6784 return (AHCI_FAILURE);
6785 }
6786
6787 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6788 /* Slot 0 is always used during error recovery */
6789 finished_tags = 0x1 & ~port_cmd_issue;
6790 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6791 "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6792 "retrieval is finished, and finished_tags = 0x%x",
6793 port, finished_tags);
6794 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6795 finished_tags = 0x1 & ~port_cmd_issue;
6796 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6797 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6798 "port multiplier is finished, and finished_tags = 0x%x",
6799 port, finished_tags);
6800
6801 } else {
6802
6803 finished_tags = ahci_portp->ahciport_pending_tags &
6804 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6805 }
6806
6807 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6808 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6809 "port_cmd_issue = 0x%x finished_tags = 0x%x",
6810 ahci_portp->ahciport_pending_tags, port_cmd_issue,
6811 finished_tags);
6812
6813 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6814 (finished_tags == 0x1)) {
6815 satapkt = ahci_portp->ahciport_err_retri_pkt;
6816 ASSERT(satapkt != NULL);
6817
6818 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6819 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6820 "with SATA_PKT_COMPLETED", (void *)satapkt);
6821
6822 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6823 goto out;
6824 }
6825
6826 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6827 (finished_tags == 0x1)) {
6828 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6829 ASSERT(satapkt != NULL);
6830
6831 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6832 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6833 "with SATA_PKT_COMPLETED", (void *)satapkt);
6834
6835 /* READ PORTMULT need copy out FIS content. */
6836 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6837 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6838 ahcirf_d2h_register_fis);
6839 satapkt->satapkt_cmd.satacmd_status_reg =
6840 GET_RFIS_STATUS(rcvd_fisp);
6841 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6842 }
6843
6844 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6845 goto out;
6846 }
6847
6848 while (finished_tags) {
6849 finished_slot = ddi_ffs(finished_tags) - 1;
6850 if (finished_slot == -1) {
6851 goto out;
6852 }
6853
6854 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6855 ASSERT(satapkt != NULL);
6856 #if AHCI_DEBUG
6857 /*
6858 * For non-native queued commands, the PRD byte count field
6859 * shall contain an accurate count of the number of bytes
6860 * transferred for the command before the PxCI bit is cleared
6861 * to '0' for the command.
6862 *
6863 * The purpose of this field is to let software know how many
6864 * bytes transferred for a given operation in order to
6865 * determine if underflow occurred. When issuing native command
6866 * queuing commands, this field should not be used and is not
6867 * required to be valid since in this case underflow is always
6868 * illegal.
6869 *
6870 * For data reads, the HBA will update its PRD byte count with
6871 * the total number of bytes received from the last FIS, and
6872 * may be able to continue normally. For data writes, the
6873 * device will detect an error, and HBA most likely will get
6874 * a fatal error.
6875 *
6876 * Therefore, here just put code to debug part. And please
6877 * refer to the comment above ahci_intr_fatal_error for the
6878 * definition of underflow error.
6879 */
6880 cmd_dmacount =
6881 ahci_portp->ahciport_prd_bytecounts[finished_slot];
6882 if (cmd_dmacount) {
6883 cmd_header =
6884 &ahci_portp->ahciport_cmd_list[finished_slot];
6885 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6886 "ahci_intr_cmd_cmplt: port %d, "
6887 "PRD Byte Count = 0x%x, "
6888 "ahciport_prd_bytecounts = 0x%x", port,
6889 cmd_header->ahcich_prd_byte_count,
6890 cmd_dmacount);
6891
6892 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6893 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6894 "ahci_intr_cmd_cmplt: port %d, "
6895 "an underflow occurred", port);
6896 }
6897 }
6898 #endif
6899
6900 /*
6901 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6902 * feature, sata_special_regs flag will be set, and the
6903 * driver should copy the status and the other corresponding
6904 * register values in the D2H Register FIS received (It's
6905 * working on Non-data protocol) from the device back to
6906 * the sata_cmd.
6907 *
6908 * For every AHCI port, there is only one Received FIS
6909 * structure, which contains the FISes received from the
6910 * device, So we're trying to copy the content of D2H
6911 * Register FIS in the Received FIS structure back to
6912 * the sata_cmd.
6913 */
6914 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6915 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6916 ahcirf_d2h_register_fis);
6917 satapkt->satapkt_cmd.satacmd_status_reg =
6918 GET_RFIS_STATUS(rcvd_fisp);
6919 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6920 }
6921
6922 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6923 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6924 "with SATA_PKT_COMPLETED", (void *)satapkt);
6925
6926 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
6927 CLEAR_BIT(finished_tags, finished_slot);
6928 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
6929
6930 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6931 }
6932 out:
6933 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
6934 "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
6935 ahci_portp->ahciport_pending_tags);
6936
6937 ahci_flush_doneq(ahci_portp);
6938
6939 mutex_exit(&ahci_portp->ahciport_mutex);
6940
6941 return (AHCI_SUCCESS);
6942 }
6943
6944 /*
6945 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
6946 * with the 'I' bit set and has been copied into system memory. It will
6947 * be sent under the following situations:
6948 *
6949 * 1. NCQ command is completed
6950 *
6951 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
6952 * via the Set Device Bits FIS. When such event is generated, the software
6953 * needs to read PxSACT register and compares the current value to the
6954 * list of commands previously issue by software. ahciport_pending_ncq_tags
6955 * keeps the tags of previously issued commands.
6956 *
6957 * 2. Asynchronous Notification
6958 *
6959 * Asynchronous Notification is a feature in SATA spec 2.6.
6960 *
6961 * 1) ATAPI device will send a signal to the host when media is inserted or
6962 * removed and avoids polling the device for media changes. The signal
6963 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
6964 * set to '1'. At the moment, it's not supported yet.
6965 *
6966 * 2) Port multiplier will send a signal to the host when a hot plug event
6967 * has occured on a port multiplier port. It is used when command based
6968 * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
6969 */
6970 static int
6971 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
6972 ahci_port_t *ahci_portp, uint8_t port)
6973 {
6974 ahci_addr_t addr;
6975
6976 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
6977 "ahci_intr_set_device_bits enter: port %d", port);
6978
6979 /* Initialize HBA port address */
6980 AHCI_ADDR_SET_PORT(&addr, port);
6981
6982 /* NCQ plug handler */
6983 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
6984
6985 /* Check port multiplier's asynchronous notification events */
6986 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
6987 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
6988 ahci_portp, port);
6989 }
6990
6991 /* ATAPI events is not supported yet */
6992
6993 return (AHCI_SUCCESS);
6994 }
6995 /*
6996 * NCQ interrupt handler. Called upon a NCQ command is completed.
6997 * Only be called from ahci_intr_set_device_bits().
6998 */
6999 static int
7000 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7001 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7002 {
7003 uint32_t port_sactive;
7004 uint32_t port_cmd_issue;
7005 uint32_t issued_tags;
7006 int issued_slot;
7007 uint32_t finished_tags;
7008 int finished_slot;
7009 uint8_t port = addrp->aa_port;
7010 sata_pkt_t *satapkt;
7011
7012 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7013 "ahci_intr_set_device_bits enter: port %d", port);
7014
7015 mutex_enter(&ahci_portp->ahciport_mutex);
7016 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7017 mutex_exit(&ahci_portp->ahciport_mutex);
7018 return (AHCI_SUCCESS);
7019 }
7020
7021 /*
7022 * First the handler got which commands are finished by checking
7023 * PxSACT register
7024 */
7025 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7026 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7027
7028 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7029 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7030
7031 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7032 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7033 "port_sactive = 0x%x", port,
7034 ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7035
7036 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7037 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7038
7039 /*
7040 * For NCQ commands, the software can determine which command has
7041 * already been transmitted to the device by checking PxCI register.
7042 */
7043 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7044 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7045
7046 issued_tags = ahci_portp->ahciport_pending_tags &
7047 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7048
7049 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7050 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7051 != DDI_FM_OK) {
7052 mutex_exit(&ahci_portp->ahciport_mutex);
7053 return (AHCI_FAILURE);
7054 }
7055
7056 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7057 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7058 "port_cmd_issue = 0x%x", port,
7059 ahci_portp->ahciport_pending_tags, port_cmd_issue);
7060
7061 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7062 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7063
7064 /*
7065 * Clear ahciport_pending_tags bit when the corresponding command
7066 * is already sent down to the device.
7067 */
7068 while (issued_tags) {
7069 issued_slot = ddi_ffs(issued_tags) - 1;
7070 if (issued_slot == -1) {
7071 goto next;
7072 }
7073 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7074 CLEAR_BIT(issued_tags, issued_slot);
7075 }
7076
7077 next:
7078 while (finished_tags) {
7079 finished_slot = ddi_ffs(finished_tags) - 1;
7080 if (finished_slot == -1) {
7081 goto out;
7082 }
7083
7084 /* The command is certainly transmitted to the device */
7085 ASSERT(!(ahci_portp->ahciport_pending_tags &
7086 (0x1 << finished_slot)));
7087
7088 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7089 ASSERT(satapkt != NULL);
7090
7091 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7092 "ahci_intr_set_device_bits: sending up pkt 0x%p "
7093 "with SATA_PKT_COMPLETED", (void *)satapkt);
7094
7095 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7096 CLEAR_BIT(finished_tags, finished_slot);
7097 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7098
7099 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7100 }
7101 out:
7102 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7103 "ahci_intr_set_device_bits: port %d "
7104 "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7105 port, ahci_portp->ahciport_pending_ncq_tags,
7106 ahci_portp->ahciport_pending_tags);
7107
7108 ahci_flush_doneq(ahci_portp);
7109
7110 mutex_exit(&ahci_portp->ahciport_mutex);
7111
7112 return (AHCI_SUCCESS);
7113 }
7114
7115 /*
7116 * Port multiplier asynchronous notification event handler. Called upon a
7117 * device is hot plugged/pulled.
7118 *
7119 * The async-notification event will only be recorded by ahcipmi_snotif_tags
7120 * here and will be handled by ahci_probe_pmult().
7121 *
7122 * NOTE: called only from ahci_port_intr().
7123 */
7124 static int
7125 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7126 uint8_t port)
7127 {
7128 sata_device_t sdevice;
7129
7130 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7131 "ahci_intr_pmult_sntf_events enter: port %d ", port);
7132
7133 /* no hot-plug while attaching process */
7134 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7135 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7136 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7137 return (AHCI_SUCCESS);
7138 }
7139 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7140
7141 mutex_enter(&ahci_portp->ahciport_mutex);
7142 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7143 mutex_exit(&ahci_portp->ahciport_mutex);
7144 return (AHCI_SUCCESS);
7145 }
7146
7147 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7148
7149 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7150 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7151 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7152 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7153 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7154 AHCI_SNOTIF_CLEAR_ALL);
7155
7156 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7157 mutex_exit(&ahci_portp->ahciport_mutex);
7158 return (AHCI_SUCCESS);
7159 }
7160
7161 /* Port Multiplier sub-device hot-plug handler */
7162 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7163 mutex_exit(&ahci_portp->ahciport_mutex);
7164 return (AHCI_SUCCESS);
7165 }
7166
7167 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7168 /* Not allowed to re-enter. */
7169 mutex_exit(&ahci_portp->ahciport_mutex);
7170 return (AHCI_SUCCESS);
7171 }
7172
7173 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7174
7175 /*
7176 * NOTE:
7177 * Even if Asynchronous Notification is supported (and enabled) by
7178 * both controller and the port multiplier, the content of PxSNTF
7179 * register is always set to 0x8000 by async notification event. We
7180 * need to check GSCR[32] on the port multiplier to find out the
7181 * owner of this event.
7182 * This is not accord with SATA spec 2.6 and needs further
7183 * clarification.
7184 */
7185 /* hot-plug will not reported while reseting. */
7186 if (ahci_portp->ahciport_reset_in_progress == 1) {
7187 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7188 "port %d snotif event ignored", port);
7189 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7190 mutex_exit(&ahci_portp->ahciport_mutex);
7191 return (AHCI_SUCCESS);
7192 }
7193
7194 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7195 "PxSNTF is set to 0x%x by port multiplier",
7196 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7197
7198 /*
7199 * Now we need do some necessary operation and inform SATA framework
7200 * that link/device events has happened.
7201 */
7202 bzero((void *)&sdevice, sizeof (sata_device_t));
7203 sdevice.satadev_addr.cport = ahci_ctlp->
7204 ahcictl_port_to_cport[port];
7205 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7206 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7207 sdevice.satadev_state = SATA_PSTATE_PWRON;
7208
7209 /* Just reject packets, do not stop that port. */
7210 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7211
7212 mutex_exit(&ahci_portp->ahciport_mutex);
7213 sata_hba_event_notify(
7214 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7215 &sdevice,
7216 SATA_EVNT_PMULT_LINK_CHANGED);
7217 mutex_enter(&ahci_portp->ahciport_mutex);
7218
7219 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7220 mutex_exit(&ahci_portp->ahciport_mutex);
7221
7222 return (AHCI_SUCCESS);
7223 }
7224
7225 /*
7226 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7227 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7228 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7229 * indicates a COMINIT signal was received.
7230 *
7231 * Hot plug insertion is detected by reception of a COMINIT signal from the
7232 * device. On reception of unsolicited COMINIT, the HBA shall generate a
7233 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7234 * begin the normal communication negotiation sequence as outlined in the
7235 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7236 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7237 * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7238 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7239 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7240 * to handle hot plug insertion. In this interrupt handler, just do nothing
7241 * but print some log message and clear the bit.
7242 */
7243 static int
7244 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7245 ahci_port_t *ahci_portp, uint8_t port)
7246 {
7247 #if AHCI_DEBUG
7248 uint32_t port_serror;
7249 #endif
7250
7251 mutex_enter(&ahci_portp->ahciport_mutex);
7252
7253 #if AHCI_DEBUG
7254 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7255 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7256
7257 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7258 "ahci_intr_port_connect_change: port %d, "
7259 "port_serror = 0x%x", port, port_serror);
7260 #endif
7261
7262 /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7263 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7264 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7265 SERROR_EXCHANGED_ERR);
7266
7267 mutex_exit(&ahci_portp->ahciport_mutex);
7268
7269 return (AHCI_SUCCESS);
7270 }
7271
7272 /*
7273 * Hot Plug Operation for platforms that support Mechanical Presence
7274 * Switches.
7275 *
7276 * When set, it indicates that a mechanical presence switch attached to this
7277 * port has been opened or closed, which may lead to a change in the connection
7278 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7279 * are set to '1'.
7280 *
7281 * At the moment, this interrupt is not needed and disabled and we just log
7282 * the debug message.
7283 */
7284 static int
7285 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7286 ahci_port_t *ahci_portp, uint8_t port)
7287 {
7288 uint32_t cap_status, port_cmd_status;
7289
7290 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7291 "ahci_intr_device_mechanical_presence_status enter, "
7292 "port %d", port);
7293
7294 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7295 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7296
7297 mutex_enter(&ahci_portp->ahciport_mutex);
7298 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7299 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7300
7301 if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7302 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7303 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7304 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7305 "the interrupt: cap_status = 0x%x, "
7306 "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7307 mutex_exit(&ahci_portp->ahciport_mutex);
7308
7309 return (AHCI_SUCCESS);
7310 }
7311
7312 #if AHCI_DEBUG
7313 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7314 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7315 "The mechanical presence switch is open: "
7316 "port %d, port_cmd_status = 0x%x",
7317 port, port_cmd_status);
7318 } else {
7319 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7320 "The mechanical presence switch is close: "
7321 "port %d, port_cmd_status = 0x%x",
7322 port, port_cmd_status);
7323 }
7324 #endif
7325
7326 mutex_exit(&ahci_portp->ahciport_mutex);
7327
7328 return (AHCI_SUCCESS);
7329 }
7330
7331 /*
7332 * Native Hot Plug Support.
7333 *
7334 * When set, it indicates that the internal PHYRDY signal changed state.
7335 * This bit reflects the state of PxSERR.DIAG.N.
7336 *
7337 * There are three kinds of conditions to generate this interrupt event:
7338 * 1. a device is inserted
7339 * 2. a device is disconnected
7340 * 3. when the link enters/exits a Partial or Slumber interface power
7341 * management state
7342 *
7343 * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7344 * bit may be set due to the link entering the Partial or Slumber power
7345 * management state, rather than due to a hot plug insertion or removal
7346 * event. So far, the interface power management is disabled, so the
7347 * driver can reliably get removal detection notification via the
7348 * PxSERR.DIAG.N bit.
7349 */
7350 static int
7351 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7352 ahci_port_t *ahci_portp, uint8_t port)
7353 {
7354 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7355 sata_device_t sdevice;
7356 int dev_exists_now = 0;
7357 int dev_existed_previously = 0;
7358 ahci_addr_t port_addr;
7359
7360 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7361 "ahci_intr_phyrdy_change enter, port %d", port);
7362
7363 /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7364 mutex_enter(&ahci_portp->ahciport_mutex);
7365 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7366 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7367 SERROR_PHY_RDY_CHG);
7368 mutex_exit(&ahci_portp->ahciport_mutex);
7369
7370 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7371 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7372 (ahci_portp == NULL)) {
7373 /* The whole controller setup is not yet done. */
7374 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7375 return (AHCI_SUCCESS);
7376 }
7377 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7378
7379 mutex_enter(&ahci_portp->ahciport_mutex);
7380
7381 /* SStatus tells the presence of device. */
7382 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7383 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7384
7385 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7386 dev_exists_now = 1;
7387 }
7388
7389 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7390 dev_existed_previously = 1;
7391 }
7392
7393 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7394 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7395 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7396 "ahci_intr_phyrdy_change: port %d "
7397 "AHCI_PORT_FLAG_NODEV is cleared", port);
7398 if (dev_exists_now == 0)
7399 dev_existed_previously = 1;
7400 }
7401
7402 bzero((void *)&sdevice, sizeof (sata_device_t));
7403 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7404 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7405 sdevice.satadev_addr.pmport = 0;
7406 sdevice.satadev_state = SATA_PSTATE_PWRON;
7407 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7408
7409 AHCI_ADDR_SET_PORT(&port_addr, port);
7410
7411 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7412 if (dev_exists_now) {
7413 if (dev_existed_previously) { /* 1 -> 1 */
7414 /* Things are fine now. The loss was temporary. */
7415 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7416 "ahci_intr_phyrdy_change port %d "
7417 "device link lost/established", port);
7418
7419 mutex_exit(&ahci_portp->ahciport_mutex);
7420 sata_hba_event_notify(
7421 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7422 &sdevice,
7423 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7424 mutex_enter(&ahci_portp->ahciport_mutex);
7425
7426 } else { /* 0 -> 1 */
7427 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7428 "ahci_intr_phyrdy_change: port %d "
7429 "device link established", port);
7430
7431 /*
7432 * A new device has been detected. The new device
7433 * might be a port multiplier instead of a drive, so
7434 * we cannot update the signature directly.
7435 */
7436 (void) ahci_initialize_port(ahci_ctlp,
7437 ahci_portp, &port_addr);
7438
7439 /* Try to start the port */
7440 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7441 != AHCI_SUCCESS) {
7442 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7443 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7444 "ahci_intr_phyrdy_change: port %d failed "
7445 "at start port", port);
7446 }
7447
7448 /* Clear the max queue depth for inserted device */
7449 ahci_portp->ahciport_max_ncq_tags = 0;
7450
7451 mutex_exit(&ahci_portp->ahciport_mutex);
7452 sata_hba_event_notify(
7453 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7454 &sdevice,
7455 SATA_EVNT_LINK_ESTABLISHED);
7456 mutex_enter(&ahci_portp->ahciport_mutex);
7457
7458 }
7459 } else { /* No device exists now */
7460
7461 if (dev_existed_previously) { /* 1 -> 0 */
7462 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7463 "ahci_intr_phyrdy_change: port %d "
7464 "device link lost", port);
7465
7466 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7467 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7468 ahci_portp, port);
7469
7470 if (ahci_portp->ahciport_device_type ==
7471 SATA_DTYPE_PMULT) {
7472 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7473 }
7474
7475 /* An existing device is lost. */
7476 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7477 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7478
7479 mutex_exit(&ahci_portp->ahciport_mutex);
7480 sata_hba_event_notify(
7481 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7482 &sdevice,
7483 SATA_EVNT_LINK_LOST);
7484 mutex_enter(&ahci_portp->ahciport_mutex);
7485 }
7486 }
7487 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7488
7489 mutex_exit(&ahci_portp->ahciport_mutex);
7490
7491 return (AHCI_SUCCESS);
7492 }
7493
7494 /*
7495 * PxIS.UFS - Unknown FIS Error
7496 *
7497 * This interrupt event means an unknown FIS was received and has been
7498 * copied into system memory. An unknown FIS is not considered an illegal
7499 * FIS, unless the length received is more than 64 bytes. If an unknown
7500 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7501 * normal operation. If the unknown FIS is more than 64 bytes, then it
7502 * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7503 * a fatal error.
7504 *
7505 * PxIS.IPMS - Incorrect Port Multiplier Status
7506 *
7507 * IPMS Indicates that the HBA received a FIS from a device that did not
7508 * have a command outstanding. The IPMS bit may be set during enumeration
7509 * of devices on a Port Multiplier due to the normal Port Multiplier
7510 * enumeration process. It is recommended that IPMS only be used after
7511 * enumeration is complete on the Port Multiplier (copied from spec).
7512 *
7513 * PxIS.OFS - Overflow Error
7514 *
7515 * Command list overflow is defined as software building a command table
7516 * that has fewer total bytes than the transaction given to the device.
7517 * On device writes, the HBA will run out of data, and on reads, there
7518 * will be no room to put the data.
7519 *
7520 * For an overflow on data read, either PIO or DMA, the HBA will set
7521 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7522 * non-fatal error when the HBA can continues. Sometimes, it will cause
7523 * a fatal error and need the software to do something.
7524 *
7525 * For an overflow on data write, setting PxIS.OFS is optional for both
7526 * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7527 * software to clean up from this serious error.
7528 *
7529 * PxIS.INFS - Interface Non-Fatal Error
7530 *
7531 * This interrupt event indicates that the HBA encountered an error on
7532 * the Serial ATA interface but was able to continue operation. The kind
7533 * of error usually occurred during a non-Data FIS, and under this condition
7534 * the FIS will be re-transmitted by HBA automatically.
7535 *
7536 * When the FMA is implemented, there should be a stat structure to
7537 * record how many every kind of error happens.
7538 */
7539 static int
7540 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7541 uint8_t port, uint32_t intr_status)
7542 {
7543 uint32_t port_serror;
7544 #if AHCI_DEBUG
7545 uint32_t port_cmd_status;
7546 uint32_t port_cmd_issue;
7547 uint32_t port_sactive;
7548 int current_slot;
7549 uint32_t current_tags;
7550 sata_pkt_t *satapkt;
7551 ahci_cmd_header_t *cmd_header;
7552 uint32_t cmd_dmacount;
7553 #endif
7554
7555 mutex_enter(&ahci_portp->ahciport_mutex);
7556
7557 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7558 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7559
7560 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7561 "ahci_intr_non_fatal_error: port %d, "
7562 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7563
7564 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7565
7566 if (intr_status & AHCI_INTR_STATUS_UFS) {
7567 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7568 "ahci port %d has unknown FIS error", port);
7569
7570 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7571 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7572 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7573 SERROR_FIS_TYPE);
7574 }
7575
7576 #if AHCI_DEBUG
7577 if (intr_status & AHCI_INTR_STATUS_IPMS) {
7578 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7579 "has Incorrect Port Multiplier Status error", port);
7580 }
7581
7582 if (intr_status & AHCI_INTR_STATUS_OFS) {
7583 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7584 "ahci port %d has overflow error", port);
7585 }
7586
7587 if (intr_status & AHCI_INTR_STATUS_INFS) {
7588 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7589 "ahci port %d has interface non fatal error", port);
7590 }
7591
7592 /*
7593 * Record the error occurred command's slot.
7594 */
7595 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7596 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7597 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7598 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7599
7600 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7601 AHCI_CMD_STATUS_CCS_SHIFT;
7602
7603 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7604 satapkt = ahci_portp->ahciport_err_retri_pkt;
7605 ASSERT(satapkt != NULL);
7606 ASSERT(current_slot == 0);
7607 } else {
7608 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7609 }
7610
7611 if (satapkt != NULL) {
7612 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7613 "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7614 "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7615 satapkt->satapkt_cmd.satacmd_cmd_reg);
7616
7617 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7618 "ahci_intr_non_fatal_error: port %d, "
7619 "satapkt 0x%p is being processed when error occurs",
7620 port, (void *)satapkt);
7621
7622 /*
7623 * PRD Byte Count field of command header is not
7624 * required to reflect the total number of bytes
7625 * transferred when an overflow occurs, so here
7626 * just log the value.
7627 */
7628 cmd_dmacount =
7629 ahci_portp->ahciport_prd_bytecounts[current_slot];
7630 if (cmd_dmacount) {
7631 cmd_header = &ahci_portp->
7632 ahciport_cmd_list[current_slot];
7633 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7634 "ahci_intr_non_fatal_error: port %d, "
7635 "PRD Byte Count = 0x%x, "
7636 "ahciport_prd_bytecounts = 0x%x", port,
7637 cmd_header->ahcich_prd_byte_count,
7638 cmd_dmacount);
7639 }
7640 }
7641 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7642 /*
7643 * For queued command, list those command which have already
7644 * been transmitted to the device and still not completed.
7645 */
7646 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7647 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7648
7649 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7650 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7651
7652 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7653 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7654 "port_sactive = 0x%x port_cmd_issue = 0x%x",
7655 ahci_portp->ahciport_pending_ncq_tags,
7656 port_sactive, port_cmd_issue);
7657
7658 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7659 port_sactive & ~port_cmd_issue &
7660 AHCI_NCQ_SLOT_MASK(ahci_portp);
7661
7662 while (current_tags) {
7663 current_slot = ddi_ffs(current_tags) - 1;
7664 if (current_slot == -1) {
7665 goto out;
7666 }
7667
7668 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7669 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7670 ahci_ctlp, "ahci_intr_non_fatal_error: "
7671 "port %d, satapkt 0x%p is outstanding when "
7672 "error occurs", port, (void *)satapkt);
7673
7674 CLEAR_BIT(current_tags, current_slot);
7675 }
7676 }
7677 out:
7678 #endif
7679 mutex_exit(&ahci_portp->ahciport_mutex);
7680
7681 return (AHCI_SUCCESS);
7682 }
7683
7684 /*
7685 * According to the AHCI spec, the error types include system memory
7686 * errors, interface errors, port multiplier errors, device errors,
7687 * command list overflow, command list underflow, native command
7688 * queuing tag errors and pio data transfer errors.
7689 *
7690 * System memory errors such as target abort, master abort, and parity
7691 * may cause the host to stop, and they are serious errors and needed
7692 * to be recovered with software intervention. When system software
7693 * has given a pointer to the HBA that doesn't exist in physical memory,
7694 * a master/target abort error occurs, and PxIS.HBFS will be set. A
7695 * data error such as CRC or parity occurs, the HBA aborts the transfer
7696 * (if necessary) and PxIS.HBDS will be set.
7697 *
7698 * Interface errors are errors that occur due to electrical issues on
7699 * the interface, or protocol miscommunication between the device and
7700 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7701 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7702 * causes PxIS.IFS/PxIS.INFS to be set are
7703 * 1. in PxSERR.ERR, P bit is set to '1'
7704 * 2. in PxSERR.DIAG, C or H bit is set to '1'
7705 * 3. PhyRdy drop unexpectly, N bit is set to '1'
7706 * If the error occurred during a non-data FIS, the FIS must be
7707 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7708 * the error occurred during a data FIS, the transfer will stop, so
7709 * the error is fatal and PxIS.IFS is set.
7710 *
7711 * When a FIS arrives that updates the taskfile, the HBA checks to see
7712 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7713 * stops processing any more commands.
7714 *
7715 * Command list overflow is defined as software building a command table
7716 * that has fewer total bytes than the transaction given to the device.
7717 * On device writes, the HBA will run out of data, and on reads, there
7718 * will be no room to put the data. For an overflow on data read, either
7719 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7720 * For an overflow on data write, setting PxIS.OFS is optional for both
7721 * DMA and PIO, and a COMRESET is required by software to clean up from
7722 * this serious error.
7723 *
7724 * Command list underflow is defined as software building a command
7725 * table that has more total bytes than the transaction given to the
7726 * device. For data writes, both PIO and DMA, the device will detect
7727 * an error and end the transfer. And these errors are most likely going
7728 * to be fatal errors that will cause the port to be restarted. For
7729 * data reads, the HBA updates its PRD byte count, and may be
7730 * able to continue normally, but is not required to. And The HBA is
7731 * not required to detect underflow conditions for native command
7732 * queuing command.
7733 *
7734 * The HBA does not actively check incoming DMA Setup FISes to ensure
7735 * that the PxSACT register bit for that slot is set. Existing error
7736 * mechanisms, such as host bus failure, or bad protocol, are used to
7737 * recover from this case.
7738 *
7739 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7740 * DATA FIS must be an integral number of Dwords. If the HBA receives
7741 * a request which is not an integral number of Dwords, the HBA
7742 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7743 * software restarts the port. And the HBA ensures that the size
7744 * of the DATA FIS received during a PIO command matches the size in
7745 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7746 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7747 * stop running until software restarts the port.
7748 */
7749 /*
7750 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7751 *
7752 * PxIS.IFS indicates that the hba encountered an error on the serial ata
7753 * interface which caused the transfer to stop.
7754 *
7755 * PxIS.HBDS indicates that the hba encountered a data error
7756 * (uncorrectable ecc/parity) when reading from or writing to system memory.
7757 *
7758 * PxIS.HBFS indicates that the hba encountered a host bus error that it
7759 * cannot recover from, such as a bad software pointer.
7760 *
7761 * PxIS.TFES is set whenever the status register is updated by the device
7762 * and the error bit (bit 0) is set.
7763 */
7764 static int
7765 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7766 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7767 {
7768 uint32_t port_cmd_status;
7769 uint32_t port_serror;
7770 uint32_t task_file_status;
7771 int failed_slot;
7772 sata_pkt_t *spkt = NULL;
7773 uint8_t err_byte;
7774 ahci_event_arg_t *args;
7775 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7776 uint32_t failed_tags = 0;
7777 int task_fail_flag = 0, task_abort_flag = 0;
7778 uint32_t slot_status;
7779
7780 mutex_enter(&ahci_portp->ahciport_mutex);
7781
7782 /*
7783 * ahci_intr_phyrdy_change() may have rendered it to
7784 * SATA_DTYPE_NONE.
7785 */
7786 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7787 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7788 "ahci_intr_fatal_error: port %d no device attached, "
7789 "and just return without doing anything", port);
7790 goto out0;
7791 }
7792
7793 if (intr_status & AHCI_INTR_STATUS_TFES) {
7794 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7795 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7796 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7797 "ahci_intr_fatal_error: port %d "
7798 "task_file_status = 0x%x", port, task_file_status);
7799 task_fail_flag = 1;
7800
7801 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7802 >> AHCI_TFD_ERR_SHIFT;
7803 if (err_byte == SATA_ERROR_ABORT)
7804 task_abort_flag = 1;
7805 }
7806
7807 /*
7808 * Here we just log the fatal error info in interrupt context.
7809 * Misc recovery processing will be handled in task queue.
7810 */
7811 if (task_fail_flag == 1) {
7812 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7813 /*
7814 * Read PxCMD.CCS to determine the slot that the HBA
7815 * was processing when the error occurred.
7816 */
7817 port_cmd_status = ddi_get32(
7818 ahci_ctlp->ahcictl_ahci_acc_handle,
7819 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7820 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7821 AHCI_CMD_STATUS_CCS_SHIFT;
7822 failed_tags = 0x1 << failed_slot;
7823
7824 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7825 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7826 "ahci_intr_fatal_error: spkt 0x%p is being "
7827 "processed when fatal error occurred for port %d",
7828 spkt, port);
7829
7830 /*
7831 * Won't emit the error message if it is an IDENTIFY
7832 * DEVICE command sent to an ATAPI device.
7833 */
7834 if ((spkt != NULL) &&
7835 (spkt->satapkt_cmd.satacmd_cmd_reg ==
7836 SATAC_ID_DEVICE) &&
7837 (task_abort_flag == 1))
7838 goto out1;
7839
7840 /*
7841 * Won't emit the error message if it is an ATAPI PACKET
7842 * command
7843 */
7844 if ((spkt != NULL) &&
7845 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7846 goto out1;
7847
7848 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7849 slot_status = ddi_get32(
7850 ahci_ctlp->ahcictl_ahci_acc_handle,
7851 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7852 failed_tags = slot_status &
7853 AHCI_NCQ_SLOT_MASK(ahci_portp);
7854 }
7855 }
7856
7857 /* print the fatal error type */
7858 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7859 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7860
7861 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7862 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7863
7864 /* print PxSERR related error message */
7865 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7866
7867 /* print task file register value */
7868 if (task_fail_flag == 1) {
7869 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7870 "= 0x%x", instance, port, task_file_status);
7871 if (task_abort_flag == 1) {
7872 cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7873 "port %d are aborted", instance, port);
7874 ahci_dump_commands(ahci_ctlp, port, failed_tags);
7875 }
7876 }
7877
7878 out1:
7879 /* Prepare the argument for the taskq */
7880 args = ahci_portp->ahciport_event_args;
7881 args->ahciea_ctlp = (void *)ahci_ctlp;
7882 args->ahciea_portp = (void *)ahci_portp;
7883 args->ahciea_event = intr_status;
7884 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7885
7886 /* Start the taskq to handle error recovery */
7887 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7888 ahci_events_handler,
7889 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7890 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7891 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7892 "port %d failed", instance, port);
7893 }
7894 out0:
7895 mutex_exit(&ahci_portp->ahciport_mutex);
7896
7897 return (AHCI_SUCCESS);
7898 }
7899
7900 /*
7901 * Hot Plug Operation for platforms that support Cold Presence Detect.
7902 *
7903 * When set, a device status has changed as detected by the cold presence
7904 * detect logic. This bit can either be set due to a non-connected port
7905 * receiving a device, or a connected port having its device removed.
7906 * This bit is only valid if the port supports cold presence detect as
7907 * indicated by PxCMD.CPD set to '1'.
7908 *
7909 * At the moment, this interrupt is not needed and disabled and we just
7910 * log the debug message.
7911 */
7912 static int
7913 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
7914 ahci_port_t *ahci_portp, uint8_t port)
7915 {
7916 uint32_t port_cmd_status;
7917 sata_device_t sdevice;
7918
7919 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7920 "ahci_intr_cold_port_detect enter, port %d", port);
7921
7922 mutex_enter(&ahci_portp->ahciport_mutex);
7923
7924 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7925 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7926 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
7927 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7928 "port %d does not support cold presence detect, so "
7929 "we just ignore this interrupt", port);
7930 mutex_exit(&ahci_portp->ahciport_mutex);
7931 return (AHCI_SUCCESS);
7932 }
7933
7934 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7935 "port %d device status has changed", port);
7936
7937 bzero((void *)&sdevice, sizeof (sata_device_t));
7938 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7939 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7940 sdevice.satadev_addr.pmport = 0;
7941 sdevice.satadev_state = SATA_PSTATE_PWRON;
7942
7943 if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
7944 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7945 "port %d: a device is hot plugged", port);
7946 mutex_exit(&ahci_portp->ahciport_mutex);
7947 sata_hba_event_notify(
7948 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7949 &sdevice,
7950 SATA_EVNT_DEVICE_ATTACHED);
7951 mutex_enter(&ahci_portp->ahciport_mutex);
7952
7953 } else {
7954 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7955 "port %d: a device is hot unplugged", port);
7956 mutex_exit(&ahci_portp->ahciport_mutex);
7957 sata_hba_event_notify(
7958 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7959 &sdevice,
7960 SATA_EVNT_DEVICE_DETACHED);
7961 mutex_enter(&ahci_portp->ahciport_mutex);
7962 }
7963
7964 mutex_exit(&ahci_portp->ahciport_mutex);
7965
7966 return (AHCI_SUCCESS);
7967 }
7968
7969 /*
7970 * Enable the interrupts for a particular port.
7971 */
7972 static void
7973 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
7974 {
7975 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
7976
7977 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
7978 "ahci_enable_port_intrs enter, port %d", port);
7979
7980 /*
7981 * Clear port interrupt status before enabling interrupt
7982 */
7983 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7984 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
7985 AHCI_PORT_INTR_MASK);
7986
7987 /*
7988 * Clear the pending bit from IS.IPS
7989 */
7990 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7991 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
7992
7993 /*
7994 * Enable the following interrupts:
7995 * Device to Host Register FIS Interrupt (DHRS)
7996 * PIO Setup FIS Interrupt (PSS)
7997 * Set Device Bits Interrupt (SDBS)
7998 * Unknown FIS Interrupt (UFS)
7999 * Port Connect Change Status (PCS)
8000 * PhyRdy Change Status (PRCS)
8001 * Overflow Status (OFS)
8002 * Interface Non-fatal Error Status (INFS)
8003 * Interface Fatal Error Status (IFS)
8004 * Host Bus Data Error Status (HBDS)
8005 * Host Bus Fatal Error Status (HBFS)
8006 * Task File Error Status (TFES)
8007 */
8008 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8009 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8010 (AHCI_INTR_STATUS_DHRS |
8011 AHCI_INTR_STATUS_PSS |
8012 AHCI_INTR_STATUS_SDBS |
8013 AHCI_INTR_STATUS_UFS |
8014 AHCI_INTR_STATUS_DPS |
8015 AHCI_INTR_STATUS_PCS |
8016 AHCI_INTR_STATUS_PRCS |
8017 AHCI_INTR_STATUS_OFS |
8018 AHCI_INTR_STATUS_INFS |
8019 AHCI_INTR_STATUS_IFS |
8020 AHCI_INTR_STATUS_HBDS |
8021 AHCI_INTR_STATUS_HBFS |
8022 AHCI_INTR_STATUS_TFES));
8023 }
8024
8025 /*
8026 * Enable interrupts for all the ports.
8027 */
8028 static void
8029 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8030 {
8031 uint32_t ghc_control;
8032
8033 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8034
8035 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8036
8037 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8038 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8039
8040 ghc_control |= AHCI_HBA_GHC_IE;
8041
8042 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8043 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8044 }
8045
8046 /*
8047 * Disable interrupts for a particular port.
8048 */
8049 static void
8050 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8051 {
8052 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8053 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8054
8055 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8056 "ahci_disable_port_intrs enter, port %d", port);
8057
8058 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8059 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8060 }
8061
8062 /*
8063 * Disable interrupts for the whole HBA.
8064 *
8065 * The global bit is cleared, then all interrupt sources from all
8066 * ports are disabled.
8067 */
8068 static void
8069 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8070 {
8071 uint32_t ghc_control;
8072
8073 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8074 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8075
8076 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8077 NULL);
8078
8079 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8080 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8081
8082 ghc_control &= ~AHCI_HBA_GHC_IE;
8083
8084 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8085 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8086 }
8087
8088 /*
8089 * Handle FIXED or MSI interrupts.
8090 */
8091 /*
8092 * According to AHCI spec, the HBA may support several interrupt modes:
8093 * * pin based interrupts (FIXED)
8094 * * single MSI message interrupts
8095 * * multiple MSI based message interrupts
8096 *
8097 * For pin based interrupts, the software interrupt handler need to check IS
8098 * register to find out which port has pending interrupts. And then check
8099 * PxIS register to find out which interrupt events happened on that port.
8100 *
8101 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8102 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8103 * in that software interrupt handler need to check IS register to determine
8104 * which port triggered the interrupts since it uses a single message for all
8105 * port interrupts.
8106 *
8107 * HBA may optionally support multiple MSI message for better performance. In
8108 * this mode, each port may have its own interrupt message, and thus generation
8109 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8110 * represents a power-of-2 wrapper on the number of implemented ports, and
8111 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8112 * maximum number of assigned interrupts. When the number of MSI messages
8113 * allocated is less than the number requested, then hardware may have two
8114 * implementation behaviors:
8115 * * assign each ports its own interrupt and then force all additional
8116 * ports to share the last interrupt message, and this condition is
8117 * indicated by clearing GHC.MRSM to '0'
8118 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8119 * When multiple-message MSI is enabled, hardware will still set IS register
8120 * as single message case. And this IS register may be used by software when
8121 * fewer than the requested number of messages is granted in order to determine
8122 * which port had the interrupt.
8123 *
8124 * Note: The current ahci driver only supports the first two interrupt modes:
8125 * pin based interrupts and single MSI message interrupts, and the reason
8126 * is indicated in below code.
8127 */
8128 static int
8129 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8130 {
8131 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8132 int count, avail, actual;
8133 int i, rc;
8134
8135 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8136 "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8137
8138 /* get number of interrupts. */
8139 rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8140 if ((rc != DDI_SUCCESS) || (count == 0)) {
8141 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8142 "ddi_intr_get_nintrs() failed, "
8143 "rc %d count %d\n", rc, count);
8144 return (DDI_FAILURE);
8145 }
8146
8147 /* get number of available interrupts. */
8148 rc = ddi_intr_get_navail(dip, intr_type, &avail);
8149 if ((rc != DDI_SUCCESS) || (avail == 0)) {
8150 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8151 "ddi_intr_get_navail() failed, "
8152 "rc %d avail %d\n", rc, avail);
8153 return (DDI_FAILURE);
8154 }
8155
8156 #if AHCI_DEBUG
8157 if (avail < count) {
8158 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8159 "ddi_intr_get_nintrs returned %d, navail() returned %d",
8160 count, avail);
8161 }
8162 #endif
8163
8164 /*
8165 * Note: So far Solaris restricts the maximum number of messages for
8166 * x86 to 2, that is avail is 2, so here we set the count with 1 to
8167 * force the driver to use single MSI message interrupt. In future if
8168 * Solaris remove the restriction, then we need to delete the below
8169 * code and try to use multiple interrupt routine to gain better
8170 * performance.
8171 */
8172 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8173 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8174 "force to use one interrupt routine though the "
8175 "HBA supports %d interrupt", count);
8176 count = 1;
8177 }
8178
8179 /* Allocate an array of interrupt handles. */
8180 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8181 ahci_ctlp->ahcictl_intr_htable =
8182 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8183
8184 /* call ddi_intr_alloc(). */
8185 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8186 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8187
8188 if ((rc != DDI_SUCCESS) || (actual == 0)) {
8189 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8190 "ddi_intr_alloc() failed, rc %d count %d actual %d "
8191 "avail %d\n", rc, count, actual, avail);
8192 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8193 ahci_ctlp->ahcictl_intr_size);
8194 return (DDI_FAILURE);
8195 }
8196
8197 /* use interrupt count returned */
8198 #if AHCI_DEBUG
8199 if (actual < count) {
8200 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8201 "Requested: %d, Received: %d", count, actual);
8202 }
8203 #endif
8204
8205 ahci_ctlp->ahcictl_intr_cnt = actual;
8206
8207 /*
8208 * Get priority for first, assume remaining are all the same.
8209 */
8210 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8211 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8212 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8213 "ddi_intr_get_pri() failed", NULL);
8214
8215 /* Free already allocated intr. */
8216 for (i = 0; i < actual; i++) {
8217 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8218 }
8219
8220 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8221 ahci_ctlp->ahcictl_intr_size);
8222 return (DDI_FAILURE);
8223 }
8224
8225 /* Test for high level interrupt. */
8226 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8227 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8228 "ahci_add_intrs: Hi level intr not supported", NULL);
8229
8230 /* Free already allocated intr. */
8231 for (i = 0; i < actual; i++) {
8232 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8233 }
8234
8235 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8236 sizeof (ddi_intr_handle_t));
8237
8238 return (DDI_FAILURE);
8239 }
8240
8241 /* Call ddi_intr_add_handler(). */
8242 for (i = 0; i < actual; i++) {
8243 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8244 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8245 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8246 "ddi_intr_add_handler() failed", NULL);
8247
8248 /* Free already allocated intr. */
8249 for (i = 0; i < actual; i++) {
8250 (void) ddi_intr_free(
8251 ahci_ctlp->ahcictl_intr_htable[i]);
8252 }
8253
8254 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8255 ahci_ctlp->ahcictl_intr_size);
8256 return (DDI_FAILURE);
8257 }
8258 }
8259
8260 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8261 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8262 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8263 "ddi_intr_get_cap() failed", NULL);
8264
8265 /* Free already allocated intr. */
8266 for (i = 0; i < actual; i++) {
8267 (void) ddi_intr_free(
8268 ahci_ctlp->ahcictl_intr_htable[i]);
8269 }
8270
8271 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8272 ahci_ctlp->ahcictl_intr_size);
8273 return (DDI_FAILURE);
8274 }
8275
8276 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8277 /* Call ddi_intr_block_enable() for MSI. */
8278 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8279 ahci_ctlp->ahcictl_intr_cnt);
8280 } else {
8281 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8282 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8283 (void) ddi_intr_enable(
8284 ahci_ctlp->ahcictl_intr_htable[i]);
8285 }
8286 }
8287
8288 return (DDI_SUCCESS);
8289 }
8290
8291 /*
8292 * Removes the registered interrupts irrespective of whether they
8293 * were legacy or MSI.
8294 *
8295 * NOTE: The controller interrupts must be disabled before calling
8296 * this routine.
8297 */
8298 static void
8299 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8300 {
8301 int x;
8302
8303 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8304
8305 /* Disable all interrupts. */
8306 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8307 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8308 /* Call ddi_intr_block_disable(). */
8309 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8310 ahci_ctlp->ahcictl_intr_cnt);
8311 } else {
8312 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8313 (void) ddi_intr_disable(
8314 ahci_ctlp->ahcictl_intr_htable[x]);
8315 }
8316 }
8317
8318 /* Call ddi_intr_remove_handler(). */
8319 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8320 (void) ddi_intr_remove_handler(
8321 ahci_ctlp->ahcictl_intr_htable[x]);
8322 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8323 }
8324
8325 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8326 }
8327
8328 /*
8329 * This routine tries to put port into P:NotRunning state by clearing
8330 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8331 * and PxCMD.CR to '0'.
8332 */
8333 static int
8334 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8335 ahci_port_t *ahci_portp, uint8_t port)
8336 {
8337 uint32_t port_cmd_status;
8338 int loop_count;
8339
8340 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8341 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8342
8343 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8344 "ahci_put_port_into_notrunning_state enter: port %d", port);
8345
8346 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8347 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8348
8349 port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8350 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8351 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8352
8353 /* Wait until PxCMD.CR is cleared */
8354 loop_count = 0;
8355 do {
8356 port_cmd_status =
8357 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8358 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8359
8360 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8361 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8362 "clearing port %d CMD.CR timeout, "
8363 "port_cmd_status = 0x%x", port,
8364 port_cmd_status);
8365 /*
8366 * We are effectively timing out after 0.5 sec.
8367 * This value is specified in AHCI spec.
8368 */
8369 break;
8370 }
8371
8372 /* Wait for 10 millisec */
8373 drv_usecwait(AHCI_10MS_USECS);
8374 } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8375
8376 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8377
8378 if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8379 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8380 "ahci_put_port_into_notrunning_state: failed to clear "
8381 "PxCMD.CR to '0' after loop count: %d, and "
8382 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8383 return (AHCI_FAILURE);
8384 } else {
8385 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8386 "ahci_put_port_into_notrunning_state: succeeded to clear "
8387 "PxCMD.CR to '0' after loop count: %d, and "
8388 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8389 return (AHCI_SUCCESS);
8390 }
8391 }
8392
8393 /*
8394 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8395 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8396 * stable state, then set PxCMD.ST to '1' to start the port directly.
8397 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8398 * COMRESET to the device to put it in an idle state.
8399 *
8400 * The fifth argument returns whether the port reset is involved during
8401 * the process.
8402 *
8403 * The routine will be called under following scenarios:
8404 * + To reset the HBA
8405 * + To abort the packet(s)
8406 * + To reset the port
8407 * + To activate the port
8408 * + Fatal error recovery
8409 * + To abort the timeout packet(s)
8410 *
8411 * NOTES!!! During this procedure, PxSERR register will be cleared, and
8412 * according to the spec, the clearance of three bits will also clear
8413 * three interrupt status bits.
8414 * 1. PxSERR.DIAG.F will clear PxIS.UFS
8415 * 2. PxSERR.DIAG.X will clear PxIS.PCS
8416 * 3. PxSERR.DIAG.N will clear PxIS.PRCS
8417 *
8418 * Among these three interrupt events, the driver needs to take care of
8419 * PxIS.PRCS, which is the hot plug event. When the driver found out
8420 * a device was unplugged, it will call the interrupt handler.
8421 */
8422 static int
8423 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8424 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8425 {
8426 uint32_t port_sstatus;
8427 uint32_t task_file_status;
8428 sata_device_t sdevice;
8429 int rval;
8430 ahci_addr_t addr_port;
8431 ahci_pmult_info_t *pminfo = NULL;
8432 int dev_exists_begin = 0;
8433 int dev_exists_end = 0;
8434 uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8435 int npmport = 0;
8436 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8437
8438 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8439
8440 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8441 "ahci_restart_port_wait_till_ready: port %d enter", port);
8442
8443 AHCI_ADDR_SET_PORT(&addr_port, port);
8444
8445 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8446 dev_exists_begin = 1;
8447
8448 /* First clear PxCMD.ST */
8449 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8450 port);
8451 if (rval != AHCI_SUCCESS)
8452 /*
8453 * If PxCMD.CR does not clear within a reasonable time, it
8454 * may assume the interface is in a hung condition and may
8455 * continue with issuing the port reset.
8456 */
8457 goto reset;
8458
8459 /* Then clear PxSERR */
8460 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8461 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8462 AHCI_SERROR_CLEAR_ALL);
8463
8464 /* Then get PxTFD */
8465 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8466 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8467
8468 /*
8469 * Check whether the device is in a stable status, if yes,
8470 * then start the port directly. However for ahci_tran_reset_dport,
8471 * we may have to perform a port reset.
8472 */
8473 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8474 !(flag & AHCI_PORT_RESET))
8475 goto out;
8476
8477 reset:
8478 /*
8479 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8480 * a COMRESET to the device
8481 */
8482 ahci_disable_port_intrs(ahci_ctlp, port);
8483 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8484 ahci_enable_port_intrs(ahci_ctlp, port);
8485
8486 #ifdef AHCI_DEBUG
8487 if (rval != AHCI_SUCCESS)
8488 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8489 "ahci_restart_port_wait_till_ready: port %d failed",
8490 port);
8491 #endif
8492
8493 if (reset_flag != NULL)
8494 *reset_flag = 1;
8495
8496 /* Indicate to the framework that a reset has happened. */
8497 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8498 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8499 !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8500 /* Set the reset in progress flag */
8501 ahci_portp->ahciport_reset_in_progress = 1;
8502
8503 bzero((void *)&sdevice, sizeof (sata_device_t));
8504 sdevice.satadev_addr.cport =
8505 ahci_ctlp->ahcictl_port_to_cport[port];
8506 sdevice.satadev_addr.pmport = 0;
8507 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8508
8509 sdevice.satadev_state = SATA_DSTATE_RESET |
8510 SATA_DSTATE_PWR_ACTIVE;
8511 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8512 mutex_exit(&ahci_portp->ahciport_mutex);
8513 sata_hba_event_notify(
8514 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8515 &sdevice,
8516 SATA_EVNT_DEVICE_RESET);
8517 mutex_enter(&ahci_portp->ahciport_mutex);
8518 }
8519
8520 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8521 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8522 } else {
8523 ahci_portp->ahciport_reset_in_progress = 0;
8524 }
8525
8526 out:
8527 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8528
8529 /* SStatus tells the presence of device. */
8530 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8531 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8532
8533 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8534 dev_exists_end = 1;
8535 }
8536
8537 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8538 return (rval);
8539
8540 /* Check whether a hot plug event happened */
8541 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8542 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8543 "ahci_restart_port_wait_till_ready: port %d "
8544 "device is removed", port);
8545 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8546 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8547 "ahci_restart_port_wait_till_ready: port %d "
8548 "AHCI_PORT_FLAG_NODEV flag is set", port);
8549 mutex_exit(&ahci_portp->ahciport_mutex);
8550 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8551 mutex_enter(&ahci_portp->ahciport_mutex);
8552
8553 return (rval);
8554 }
8555
8556
8557 /* 0/1 -> 1 : device may change */
8558 /*
8559 * May be called by ahci_fatal_error_recovery_handler, so
8560 * don't issue software if the previous device is ATAPI.
8561 */
8562 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8563 return (rval);
8564
8565 /*
8566 * The COMRESET will make port multiplier enter legacy mode.
8567 * Issue a software reset to make it work again.
8568 */
8569 ahci_disable_port_intrs(ahci_ctlp, port);
8570 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8571 ahci_enable_port_intrs(ahci_ctlp, port);
8572
8573 /*
8574 * Following codes are specific for the port multiplier
8575 */
8576 if (previous_dev_type != SATA_DTYPE_PMULT &&
8577 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8578 /* in case previous_dev_type is corrupt */
8579 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8580 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8581 return (rval);
8582 }
8583
8584 /* Device change: PMult -> Non-PMult */
8585 if (previous_dev_type == SATA_DTYPE_PMULT &&
8586 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8587 /*
8588 * This might happen because
8589 * 1. Software reset failed. Port multiplier is not correctly
8590 * enumerated.
8591 * 2. Another non-port-multiplier device is attached. Perhaps
8592 * the port multiplier was replaced by another device by
8593 * whatever reason, but AHCI driver missed hot-plug event.
8594 *
8595 * Now that the port has been initialized, we just need to
8596 * update the port structure according new device, then report
8597 * and wait SATA framework to probe new device.
8598 */
8599
8600 /* Force to release pmult resource */
8601 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8602 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8603
8604 bzero((void *)&sdevice, sizeof (sata_device_t));
8605 sdevice.satadev_addr.cport =
8606 ahci_ctlp->ahcictl_port_to_cport[port];
8607 sdevice.satadev_addr.pmport = 0;
8608 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8609
8610 sdevice.satadev_state = SATA_DSTATE_RESET |
8611 SATA_DSTATE_PWR_ACTIVE;
8612
8613 mutex_exit(&ahci_portp->ahciport_mutex);
8614 sata_hba_event_notify(
8615 ahci_ctlp->ahcictl_dip,
8616 &sdevice,
8617 SATA_EVNT_DEVICE_RESET);
8618 mutex_enter(&ahci_portp->ahciport_mutex);
8619
8620 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8621 "Port multiplier is [Gone] at port %d ", port);
8622 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8623 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8624
8625 return (AHCI_SUCCESS);
8626 }
8627
8628 /* Device change: Non-PMult -> PMult */
8629 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8630
8631 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8632 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8633
8634 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8635 }
8636 pminfo = ahci_portp->ahciport_pmult_info;
8637 ASSERT(pminfo != NULL);
8638
8639 /* Device (may) change: PMult -> PMult */
8640 /*
8641 * First initialize port multiplier. Set state to READY and wait for
8642 * probe entry point to initialize it
8643 */
8644 ahci_portp->ahciport_port_state = SATA_STATE_READY;
8645
8646 /*
8647 * It's a little complicated while target is a port multiplier. we
8648 * need to COMRESET all pmports behind that PMult otherwise those
8649 * sub-links between the PMult and the sub-devices will be in an
8650 * inactive state (indicated by PSCR0/PxSSTS) and the following access
8651 * to those sub-devices will be rejected by Link-Fatal-Error.
8652 */
8653 /*
8654 * The PxSNTF will be set soon after the pmult is plugged. While the
8655 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8656 * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8657 * will initialized it.
8658 */
8659 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8660 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8661
8662 /* Report reset event. */
8663 ahci_portp->ahciport_reset_in_progress = 1;
8664
8665 bzero((void *)&sdevice, sizeof (sata_device_t));
8666 sdevice.satadev_addr.cport = cport;
8667 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8668 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8669 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8670 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8671 SATA_EVNT_DEVICE_RESET);
8672
8673 return (rval);
8674 }
8675
8676 /*
8677 * This routine may be called under four scenarios:
8678 * a) do the recovery from fatal error
8679 * b) or we need to timeout some commands
8680 * c) or we need to abort some commands
8681 * d) or we need reset device/port/controller
8682 *
8683 * In all these scenarios, we need to send any pending unfinished
8684 * commands up to sata framework.
8685 */
8686 static void
8687 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8688 ahci_port_t *ahci_portp,
8689 uint32_t slot_status,
8690 uint32_t failed_tags,
8691 uint32_t timeout_tags,
8692 uint32_t aborted_tags,
8693 uint32_t reset_tags)
8694 {
8695 uint32_t finished_tags = 0;
8696 uint32_t unfinished_tags = 0;
8697 int tmp_slot;
8698 sata_pkt_t *satapkt;
8699 int ncq_cmd_in_progress = 0;
8700 int err_retri_cmd_in_progress = 0;
8701 int rdwr_pmult_cmd_in_progress = 0;
8702
8703 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8704
8705 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8706 "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8707 ahci_portp->ahciport_port_num, slot_status);
8708
8709 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8710 "ahci_mop_commands: failed_tags: 0x%x, "
8711 "timeout_tags: 0x%x aborted_tags: 0x%x, "
8712 "reset_tags: 0x%x", failed_tags,
8713 timeout_tags, aborted_tags, reset_tags);
8714
8715 #ifdef AHCI_DEBUG
8716 if (ahci_debug_flags & AHCIDBG_ERRS) {
8717 int i;
8718 char msg_buf[200] = {0, };
8719 for (i = 0x1f; i >= 0; i--) {
8720 if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8721 msg_buf[i] = 'X';
8722 else
8723 msg_buf[i] = '.';
8724 }
8725 msg_buf[0x20] = '\0';
8726 cmn_err(CE_NOTE, "port[%d] slots: %s",
8727 ahci_portp->ahciport_port_num, msg_buf);
8728 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8729 (void *)ahci_portp->ahciport_err_retri_pkt,
8730 (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8731 }
8732 #endif
8733
8734 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8735 finished_tags = ahci_portp->ahciport_pending_tags &
8736 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8737
8738 unfinished_tags = slot_status &
8739 AHCI_SLOT_MASK(ahci_ctlp) &
8740 ~failed_tags &
8741 ~aborted_tags &
8742 ~reset_tags &
8743 ~timeout_tags;
8744 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8745 ncq_cmd_in_progress = 1;
8746 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8747 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8748
8749 unfinished_tags = slot_status &
8750 AHCI_NCQ_SLOT_MASK(ahci_portp) &
8751 ~failed_tags &
8752 ~aborted_tags &
8753 ~reset_tags &
8754 ~timeout_tags;
8755 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8756
8757 /*
8758 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8759 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8760 * complete successfully due to one of the following three
8761 * conditions:
8762 *
8763 * 1. Fatal error - failed_tags includes its slot
8764 * 2. Timed out - timeout_tags includes its slot
8765 * 3. Aborted when hot unplug - aborted_tags includes its
8766 * slot
8767 *
8768 * Please note that the command is always sent down in Slot 0
8769 */
8770 err_retri_cmd_in_progress = 1;
8771 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8772 "ahci_mop_commands is called for port %d while "
8773 "REQUEST SENSE or READ LOG EXT for error retrieval "
8774 "is being executed slot_status = 0x%x",
8775 ahci_portp->ahciport_port_num, slot_status);
8776 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8777 ASSERT(slot_status == 0x1);
8778 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8779 rdwr_pmult_cmd_in_progress = 1;
8780 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8781 "ahci_mop_commands is called for port %d while "
8782 "READ/WRITE PORTMULT command is being executed",
8783 ahci_portp->ahciport_port_num);
8784
8785 ASSERT(slot_status == 0x1);
8786 }
8787
8788 #ifdef AHCI_DEBUG
8789 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8790 "ahci_mop_commands: finished_tags: 0x%x, "
8791 "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8792 #endif
8793
8794 /* Send up finished packets with SATA_PKT_COMPLETED */
8795 while (finished_tags) {
8796 tmp_slot = ddi_ffs(finished_tags) - 1;
8797 if (tmp_slot == -1) {
8798 break;
8799 }
8800
8801 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8802 ASSERT(satapkt != NULL);
8803
8804 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8805 "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8806 (void *)satapkt);
8807
8808 /*
8809 * Cannot fetch the return register content since the port
8810 * was restarted, so the corresponding tag will be set to
8811 * aborted tags.
8812 */
8813 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8814 CLEAR_BIT(finished_tags, tmp_slot);
8815 aborted_tags |= tmp_slot;
8816 continue;
8817 }
8818
8819 if (ncq_cmd_in_progress)
8820 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8821 tmp_slot);
8822 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8823 CLEAR_BIT(finished_tags, tmp_slot);
8824 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8825
8826 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8827 }
8828
8829 /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8830 while (failed_tags) {
8831 if (err_retri_cmd_in_progress) {
8832 satapkt = ahci_portp->ahciport_err_retri_pkt;
8833 ASSERT(satapkt != NULL);
8834 ASSERT(failed_tags == 0x1);
8835
8836 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8837 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8838 (void *)satapkt);
8839 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8840 break;
8841 }
8842 if (rdwr_pmult_cmd_in_progress) {
8843 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8844 ASSERT(satapkt != NULL);
8845 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8846 "ahci_mop_commands: sending up "
8847 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8848 (void *)satapkt);
8849 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8850 break;
8851 }
8852
8853 tmp_slot = ddi_ffs(failed_tags) - 1;
8854 if (tmp_slot == -1) {
8855 break;
8856 }
8857
8858 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8859 ASSERT(satapkt != NULL);
8860
8861 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8862 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8863 (void *)satapkt);
8864
8865 if (ncq_cmd_in_progress)
8866 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8867 tmp_slot);
8868 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8869 CLEAR_BIT(failed_tags, tmp_slot);
8870 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8871
8872 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8873 }
8874
8875 /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8876 while (timeout_tags) {
8877 if (err_retri_cmd_in_progress) {
8878 satapkt = ahci_portp->ahciport_err_retri_pkt;
8879 ASSERT(satapkt != NULL);
8880 ASSERT(timeout_tags == 0x1);
8881
8882 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8883 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8884 (void *)satapkt);
8885 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8886 break;
8887 }
8888 if (rdwr_pmult_cmd_in_progress) {
8889 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8890 ASSERT(satapkt != NULL);
8891 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8892 "ahci_mop_commands: sending up "
8893 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8894 (void *)satapkt);
8895 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8896 break;
8897 }
8898
8899 tmp_slot = ddi_ffs(timeout_tags) - 1;
8900 if (tmp_slot == -1) {
8901 break;
8902 }
8903
8904 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8905 ASSERT(satapkt != NULL);
8906
8907 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8908 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8909 (void *)satapkt);
8910
8911 if (ncq_cmd_in_progress)
8912 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8913 tmp_slot);
8914 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8915 CLEAR_BIT(timeout_tags, tmp_slot);
8916 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8917
8918 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8919 }
8920
8921 /* Send up aborted packets with SATA_PKT_ABORTED */
8922 while (aborted_tags) {
8923 if (err_retri_cmd_in_progress) {
8924 satapkt = ahci_portp->ahciport_err_retri_pkt;
8925 ASSERT(satapkt != NULL);
8926 ASSERT(aborted_tags == 0x1);
8927
8928 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8929 "sending up pkt 0x%p with SATA_PKT_ABORTED",
8930 (void *)satapkt);
8931 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8932 break;
8933 }
8934 if (rdwr_pmult_cmd_in_progress) {
8935 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8936 ASSERT(satapkt != NULL);
8937 ASSERT(aborted_tags == 0x1);
8938 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8939 "ahci_mop_commands: sending up "
8940 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
8941 (void *)satapkt);
8942 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8943 break;
8944 }
8945
8946 tmp_slot = ddi_ffs(aborted_tags) - 1;
8947 if (tmp_slot == -1) {
8948 break;
8949 }
8950
8951 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8952 ASSERT(satapkt != NULL);
8953
8954 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8955 "sending up pkt 0x%p with SATA_PKT_ABORTED",
8956 (void *)satapkt);
8957
8958 if (ncq_cmd_in_progress)
8959 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8960 tmp_slot);
8961 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8962 CLEAR_BIT(aborted_tags, tmp_slot);
8963 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8964
8965 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8966 }
8967
8968 /* Send up reset packets with SATA_PKT_RESET. */
8969 while (reset_tags) {
8970 if (rdwr_pmult_cmd_in_progress) {
8971 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8972 ASSERT(satapkt != NULL);
8973 ASSERT(aborted_tags == 0x1);
8974 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8975 "ahci_mop_commands: sending up "
8976 "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
8977 (void *)satapkt);
8978 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
8979 break;
8980 }
8981
8982 tmp_slot = ddi_ffs(reset_tags) - 1;
8983 if (tmp_slot == -1) {
8984 break;
8985 }
8986
8987 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8988 ASSERT(satapkt != NULL);
8989
8990 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8991 "sending up pkt 0x%p with SATA_PKT_RESET",
8992 (void *)satapkt);
8993
8994 if (ncq_cmd_in_progress)
8995 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8996 tmp_slot);
8997 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8998 CLEAR_BIT(reset_tags, tmp_slot);
8999 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9000
9001 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9002 }
9003
9004 /* Send up unfinished packets with SATA_PKT_RESET */
9005 while (unfinished_tags) {
9006 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9007 if (tmp_slot == -1) {
9008 break;
9009 }
9010
9011 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9012 ASSERT(satapkt != NULL);
9013
9014 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9015 "sending up pkt 0x%p with SATA_PKT_RESET",
9016 (void *)satapkt);
9017
9018 if (ncq_cmd_in_progress)
9019 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9020 tmp_slot);
9021 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9022 CLEAR_BIT(unfinished_tags, tmp_slot);
9023 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9024
9025 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9026 }
9027
9028 ahci_portp->ahciport_mop_in_progress--;
9029 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9030
9031 if (ahci_portp->ahciport_mop_in_progress == 0)
9032 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9033
9034 ahci_flush_doneq(ahci_portp);
9035 }
9036
9037 /*
9038 * This routine is going to first request a READ LOG EXT sata pkt from sata
9039 * module, and then deliver it to the HBA to get the ncq failure context.
9040 * The return value is the exactly failed tags.
9041 */
9042 static uint32_t
9043 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9044 uint8_t port)
9045 {
9046 sata_device_t sdevice;
9047 sata_pkt_t *rdlog_spkt, *spkt;
9048 ddi_dma_handle_t buf_dma_handle;
9049 ahci_addr_t addr;
9050 int loop_count;
9051 int rval;
9052 int failed_slot;
9053 uint32_t failed_tags = 0;
9054 struct sata_ncq_error_recovery_page *ncq_err_page;
9055
9056 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9057 "ahci_get_rdlogext_data enter: port %d", port);
9058
9059 /* Prepare the sdevice data */
9060 bzero((void *)&sdevice, sizeof (sata_device_t));
9061 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9062
9063 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9064 sdevice.satadev_addr.pmport = 0;
9065
9066 /* Translate sata_device.satadev_addr -> ahci_addr */
9067 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9068
9069 /*
9070 * Call the sata hba interface to get a rdlog spkt
9071 */
9072 loop_count = 0;
9073 loop:
9074 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9075 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9076 if (rdlog_spkt == NULL) {
9077 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9078 /* Sleep for a while */
9079 drv_usecwait(AHCI_10MS_USECS);
9080 goto loop;
9081 }
9082 /* Timed out after 1s */
9083 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9084 "failed to get rdlog spkt for port %d", port);
9085 return (failed_tags);
9086 }
9087
9088 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9089
9090 /*
9091 * This flag is used to handle the specific error recovery when the
9092 * READ LOG EXT command gets a failure (fatal error or time-out).
9093 */
9094 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9095
9096 /*
9097 * This start is not supposed to fail because after port is restarted,
9098 * the whole command list is empty.
9099 */
9100 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9101 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9102 ahci_portp->ahciport_err_retri_pkt = NULL;
9103
9104 /* Remove the flag after READ LOG EXT command is completed */
9105 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9106
9107 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9108 /* Update the request log data */
9109 buf_dma_handle = *(ddi_dma_handle_t *)
9110 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9111 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9112 DDI_DMA_SYNC_FORKERNEL);
9113 if (rval == DDI_SUCCESS) {
9114 ncq_err_page =
9115 (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9116 satapkt_cmd.satacmd_bp->b_un.b_addr;
9117
9118 /* Get the failed tag */
9119 failed_slot = ncq_err_page->ncq_tag;
9120 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9121 "ahci_get_rdlogext_data: port %d "
9122 "failed slot %d", port, failed_slot);
9123 if (failed_slot & NQ) {
9124 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9125 "the failed slot is not a valid tag", NULL);
9126 goto out;
9127 }
9128
9129 failed_slot &= NCQ_TAG_MASK;
9130 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9131 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9132 "ahci_get_rdlogext_data: failed spkt 0x%p",
9133 (void *)spkt);
9134 if (spkt == NULL) {
9135 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9136 "the failed slot spkt is NULL", NULL);
9137 goto out;
9138 }
9139
9140 failed_tags = 0x1 << failed_slot;
9141
9142 /* Fill out the error context */
9143 ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9144 ncq_err_page);
9145 ahci_update_sata_registers(ahci_ctlp, port,
9146 &spkt->satapkt_device);
9147 }
9148 }
9149 out:
9150 sata_free_error_retrieval_pkt(rdlog_spkt);
9151
9152 return (failed_tags);
9153 }
9154
9155 /*
9156 * This routine is going to first request a REQUEST SENSE sata pkt from sata
9157 * module, and then deliver it to the HBA to get the sense data and copy
9158 * the sense data back to the orignal failed sata pkt, and free the REQUEST
9159 * SENSE sata pkt later.
9160 */
9161 static void
9162 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9163 uint8_t port, sata_pkt_t *spkt)
9164 {
9165 sata_device_t sdevice;
9166 sata_pkt_t *rs_spkt;
9167 sata_cmd_t *sata_cmd;
9168 ddi_dma_handle_t buf_dma_handle;
9169 ahci_addr_t addr;
9170 int loop_count;
9171 #if AHCI_DEBUG
9172 struct scsi_extended_sense *rqsense;
9173 #endif
9174
9175 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9176 "ahci_get_rqsense_data enter: port %d", port);
9177
9178 /* Prepare the sdevice data */
9179 bzero((void *)&sdevice, sizeof (sata_device_t));
9180 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9181
9182 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9183 sdevice.satadev_addr.pmport = 0;
9184
9185 /* Translate sata_device.satadev_addr -> ahci_addr */
9186 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9187
9188 sata_cmd = &spkt->satapkt_cmd;
9189
9190 /*
9191 * Call the sata hba interface to get a rs spkt
9192 */
9193 loop_count = 0;
9194 loop:
9195 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9196 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9197 if (rs_spkt == NULL) {
9198 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9199 /* Sleep for a while */
9200 drv_usecwait(AHCI_10MS_USECS);
9201 goto loop;
9202
9203 }
9204 /* Timed out after 1s */
9205 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9206 "failed to get rs spkt for port %d", port);
9207 return;
9208 }
9209
9210 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9211
9212 /*
9213 * This flag is used to handle the specific error recovery when the
9214 * REQUEST SENSE command gets a faiure (fatal error or time-out).
9215 */
9216 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9217
9218 /*
9219 * This start is not supposed to fail because after port is restarted,
9220 * the whole command list is empty.
9221 */
9222 ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9223 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9224 ahci_portp->ahciport_err_retri_pkt = NULL;
9225
9226 /* Remove the flag after REQUEST SENSE command is completed */
9227 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9228
9229 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9230 /* Update the request sense data */
9231 buf_dma_handle = *(ddi_dma_handle_t *)
9232 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9233 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9234 DDI_DMA_SYNC_FORKERNEL);
9235 /* Copy the request sense data */
9236 bcopy(rs_spkt->
9237 satapkt_cmd.satacmd_bp->b_un.b_addr,
9238 &sata_cmd->satacmd_rqsense,
9239 SATA_ATAPI_MIN_RQSENSE_LEN);
9240 #if AHCI_DEBUG
9241 rqsense = (struct scsi_extended_sense *)
9242 sata_cmd->satacmd_rqsense;
9243
9244 /* Dump the sense data */
9245 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9246 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9247 "Sense data for satapkt %p ATAPI cmd 0x%x",
9248 spkt, sata_cmd->satacmd_acdb[0]);
9249 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9250 " es_code 0x%x es_class 0x%x "
9251 "es_key 0x%x es_add_code 0x%x "
9252 "es_qual_code 0x%x",
9253 rqsense->es_code, rqsense->es_class,
9254 rqsense->es_key, rqsense->es_add_code,
9255 rqsense->es_qual_code);
9256 #endif
9257 }
9258
9259 sata_free_error_retrieval_pkt(rs_spkt);
9260 }
9261
9262 /*
9263 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9264 * the port must be restarted. When the HBA detects thus error, it may try
9265 * to abort a transfer. And if the transfer was aborted, the device is
9266 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9267 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9268 * that the device is in a stable status and transfers may be restarted without
9269 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9270 * then the software will send the COMRESET to do the port reset.
9271 *
9272 * Software should perform the appropriate error recovery actions based on
9273 * whether non-queued commands were being issued or natived command queuing
9274 * commands were being issued.
9275 *
9276 * And software will complete the command that had the error with error mark
9277 * to higher level software.
9278 *
9279 * Fatal errors include the following:
9280 * PxIS.IFS - Interface Fatal Error Status
9281 * PxIS.HBDS - Host Bus Data Error Status
9282 * PxIS.HBFS - Host Bus Fatal Error Status
9283 * PxIS.TFES - Task File Error Status
9284 */
9285 static void
9286 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9287 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9288 {
9289 uint32_t port_cmd_status;
9290 uint32_t slot_status = 0;
9291 uint32_t failed_tags = 0;
9292 int failed_slot;
9293 int reset_flag = 0, flag = 0;
9294 ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9295 sata_cmd_t *sata_cmd = NULL;
9296 sata_pkt_t *spkt = NULL;
9297 #if AHCI_DEBUG
9298 ahci_cmd_header_t *cmd_header;
9299 #endif
9300 uint8_t port = addrp->aa_port;
9301 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9302 int rval;
9303
9304 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9305
9306 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9307 "ahci_fatal_error_recovery_handler enter: port %d", port);
9308
9309 /* Port multiplier error */
9310 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9311 /* FBS code is neither completed nor tested. */
9312 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9313 port, intr_status);
9314
9315 /* Force a port reset */
9316 flag = AHCI_PORT_RESET;
9317 }
9318
9319 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9320 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9321
9322 /* Read PxCI to see which commands are still outstanding */
9323 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9324 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9325
9326 /*
9327 * Read PxCMD.CCS to determine the slot that the HBA
9328 * was processing when the error occurred.
9329 */
9330 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9331 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9332 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9333 AHCI_CMD_STATUS_CCS_SHIFT;
9334
9335 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9336 spkt = ahci_portp->ahciport_err_retri_pkt;
9337 ASSERT(spkt != NULL);
9338 } else {
9339 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9340 if (spkt == NULL) {
9341 /* May happen when interface errors occur? */
9342 goto next;
9343 }
9344 }
9345
9346 #if AHCI_DEBUG
9347 /*
9348 * Debugging purpose...
9349 */
9350 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9351 cmd_header =
9352 &ahci_portp->ahciport_cmd_list[failed_slot];
9353 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9354 "ahci_fatal_error_recovery_handler: port %d, "
9355 "PRD Byte Count = 0x%x, "
9356 "ahciport_prd_bytecounts = 0x%x", port,
9357 cmd_header->ahcich_prd_byte_count,
9358 ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9359 }
9360 #endif
9361
9362 sata_cmd = &spkt->satapkt_cmd;
9363
9364 /* Fill out the status and error registers for PxIS.TFES */
9365 if (intr_status & AHCI_INTR_STATUS_TFES) {
9366 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9367 ahcirf_d2h_register_fis);
9368
9369 /* Copy the error context back to the sata_cmd */
9370 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9371 }
9372
9373 /* The failed command must be one of the outstanding commands */
9374 failed_tags = 0x1 << failed_slot;
9375 ASSERT(failed_tags & slot_status);
9376
9377 /* Update the sata registers, especially PxSERR register */
9378 ahci_update_sata_registers(ahci_ctlp, port,
9379 &spkt->satapkt_device);
9380
9381 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9382 /* Read PxSACT to see which commands are still outstanding */
9383 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9384 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9385 }
9386 next:
9387
9388 #if AHCI_DEBUG
9389 /*
9390 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9391 * set, it means a fatal error happened after REQUEST SENSE command
9392 * or READ LOG EXT command is delivered to the HBA during the error
9393 * recovery process. At this time, the only outstanding command is
9394 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9395 */
9396 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9397 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9398 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9399 "command or READ LOG EXT command for error data retrieval "
9400 "failed", port);
9401 ASSERT(slot_status == 0x1);
9402 ASSERT(failed_slot == 0);
9403 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9404 SCMD_REQUEST_SENSE ||
9405 spkt->satapkt_cmd.satacmd_cmd_reg ==
9406 SATAC_READ_LOG_EXT);
9407 }
9408 #endif
9409
9410 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9411 ahci_portp->ahciport_mop_in_progress++;
9412
9413 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9414 port, flag, &reset_flag);
9415
9416 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9417 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9418 if (rval == AHCI_SUCCESS)
9419 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9420 "succeed", instance, port);
9421 else
9422 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9423 "failed", instance, port);
9424 }
9425
9426 /*
9427 * Won't retrieve error information:
9428 * 1. Port reset was involved to recover
9429 * 2. Device is gone
9430 * 3. IDENTIFY DEVICE command sent to ATAPI device
9431 * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9432 */
9433 if (reset_flag ||
9434 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9435 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9436 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9437 goto out;
9438
9439 /*
9440 * Deliver READ LOG EXT to gather information about the error when
9441 * a COMRESET has not been performed as part of the error recovery
9442 * during NCQ command processing.
9443 */
9444 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9445 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9446 ahci_portp, port);
9447 goto out;
9448 }
9449
9450 /*
9451 * Deliver REQUEST SENSE for ATAPI command to gather information about
9452 * the error when a COMRESET has not been performed as part of the
9453 * error recovery.
9454 */
9455 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9456 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9457 out:
9458 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9459 "ahci_fatal_error_recovery_handler: port %d fatal error "
9460 "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9461 "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9462 port, slot_status, ahci_portp->ahciport_pending_tags,
9463 ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9464
9465 ahci_mop_commands(ahci_ctlp,
9466 ahci_portp,
9467 slot_status,
9468 failed_tags, /* failed tags */
9469 0, /* timeout tags */
9470 0, /* aborted tags */
9471 0); /* reset tags */
9472 }
9473
9474 /*
9475 * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9476 * 1. device specific.PxFBS.SDE=1
9477 * 2. Non-Deivce specific.
9478 * Nothing will be done when Command-based switching is employed.
9479 *
9480 * Currently code is neither completed nor tested.
9481 */
9482 static void
9483 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9484 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9485 {
9486 #ifndef __lock_lint
9487 _NOTE(ARGUNUSED(intr_status))
9488 #endif
9489 uint32_t port_fbs_ctrl;
9490 int loop_count = 0;
9491 ahci_addr_t addr;
9492
9493 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9494
9495 /* Nothing will be done under Command-based switching. */
9496 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9497 return;
9498
9499 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9500 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9501
9502 if (!(port_fbs_ctrl & AHCI_FBS_EN))
9503 /* FBS is not enabled. */
9504 return;
9505
9506 /* Problem's getting complicated now. */
9507 /*
9508 * If FIS-based switching is used, we need to check
9509 * the PxFBS to see the error type.
9510 */
9511 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9512 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9513
9514 /* Refer to spec(v1.2) 9.3.6.1 */
9515 if (port_fbs_ctrl & AHCI_FBS_SDE) {
9516 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9517 "A Device Sepcific Error: port %d", port);
9518 /*
9519 * Controller has paused commands for all other
9520 * sub-devices until PxFBS.DEC is set.
9521 */
9522 ahci_reject_all_abort_pkts(ahci_ctlp,
9523 ahci_portp, 0);
9524
9525 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9526 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9527 port_fbs_ctrl | AHCI_FBS_DEC);
9528
9529 /*
9530 * Wait controller clear PxFBS.DEC,
9531 * then we can continue.
9532 */
9533 loop_count = 0;
9534 do {
9535 port_fbs_ctrl = ddi_get32(ahci_ctlp->
9536 ahcictl_ahci_acc_handle, (uint32_t *)
9537 AHCI_PORT_PxFBS(ahci_ctlp, port));
9538
9539 if (loop_count++ > 1000)
9540 /*
9541 * Esclate the error. Follow
9542 * non-device specific error
9543 * procedure.
9544 */
9545 return;
9546
9547 drv_usecwait(AHCI_100US_USECS);
9548 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9549
9550 /*
9551 * Issue a software reset to ensure drive is in
9552 * a known state.
9553 */
9554 (void) ahci_software_reset(ahci_ctlp,
9555 ahci_portp, &addr);
9556
9557 } else {
9558
9559 /* Process Non-Device Specific Error. */
9560 /* This will be handled later on. */
9561 cmn_err(CE_NOTE, "!FBS is not supported now.");
9562 }
9563 }
9564 /*
9565 * Handle events - fatal error recovery
9566 */
9567 static void
9568 ahci_events_handler(void *args)
9569 {
9570 ahci_event_arg_t *ahci_event_arg;
9571 ahci_ctl_t *ahci_ctlp;
9572 ahci_port_t *ahci_portp;
9573 ahci_addr_t *addrp;
9574 uint32_t event;
9575 int instance;
9576
9577 ahci_event_arg = (ahci_event_arg_t *)args;
9578
9579 ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9580 ahci_portp = ahci_event_arg->ahciea_portp;
9581 addrp = ahci_event_arg->ahciea_addrp;
9582 event = ahci_event_arg->ahciea_event;
9583 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9584
9585 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9586 "ahci_events_handler enter: port %d intr_status = 0x%x",
9587 ahci_portp->ahciport_port_num, event);
9588
9589 mutex_enter(&ahci_portp->ahciport_mutex);
9590
9591 /*
9592 * ahci_intr_phyrdy_change() may have rendered it to
9593 * SATA_DTYPE_NONE.
9594 */
9595 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9596 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9597 "ahci_events_handler: port %d no device attached, "
9598 "and just return without doing anything",
9599 ahci_portp->ahciport_port_num);
9600
9601 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9602 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9603 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9604 "succeed", instance, ahci_portp->ahciport_port_num);
9605 }
9606
9607 goto out;
9608 }
9609
9610 if (event & (AHCI_INTR_STATUS_IFS |
9611 AHCI_INTR_STATUS_HBDS |
9612 AHCI_INTR_STATUS_HBFS |
9613 AHCI_INTR_STATUS_TFES))
9614 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9615 addrp, event);
9616
9617 out:
9618 mutex_exit(&ahci_portp->ahciport_mutex);
9619 }
9620
9621 /*
9622 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9623 * detect there are some commands which are timed out.
9624 */
9625 static void
9626 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9627 uint8_t port, uint32_t tmp_timeout_tags)
9628 {
9629 uint32_t slot_status = 0;
9630 uint32_t finished_tags = 0;
9631 uint32_t timeout_tags = 0;
9632
9633 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9634 "ahci_timeout_pkts enter: port %d", port);
9635
9636 mutex_enter(&ahci_portp->ahciport_mutex);
9637
9638 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9639 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9640 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9641 /* Read PxCI to see which commands are still outstanding */
9642 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9643 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9644 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9645 /* Read PxSACT to see which commands are still outstanding */
9646 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9647 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9648 }
9649
9650 #if AHCI_DEBUG
9651 /*
9652 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9653 * set, it means a fatal error happened after REQUEST SENSE command
9654 * or READ LOG EXT command is delivered to the HBA during the error
9655 * recovery process. At this time, the only outstanding command is
9656 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9657 */
9658 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9659 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9660 "ahci_timeout_pkts called while REQUEST SENSE "
9661 "command or READ LOG EXT command for error recovery "
9662 "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9663 "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9664 tmp_timeout_tags, slot_status,
9665 ahci_portp->ahciport_pending_tags,
9666 ahci_portp->ahciport_pending_ncq_tags);
9667 ASSERT(slot_status == 0x1);
9668 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9669 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9670 "ahci_timeout_pkts called while executing R/W PMULT "
9671 "command timeout_tags = 0x%x, slot_status = 0x%x",
9672 tmp_timeout_tags, slot_status);
9673 ASSERT(slot_status == 0x1);
9674 }
9675 #endif
9676
9677 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9678 ahci_portp->ahciport_mop_in_progress++;
9679
9680 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9681 port, AHCI_PORT_RESET, NULL);
9682
9683 /*
9684 * Re-identify timeout tags because some previously checked commands
9685 * could already complete.
9686 */
9687 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9688 finished_tags = ahci_portp->ahciport_pending_tags &
9689 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9690 timeout_tags = tmp_timeout_tags & ~finished_tags;
9691
9692 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9693 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9694 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9695 "pending_tags = 0x%x ",
9696 port, finished_tags, timeout_tags,
9697 slot_status, ahci_portp->ahciport_pending_tags);
9698 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9699 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9700 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9701 timeout_tags = tmp_timeout_tags & ~finished_tags;
9702
9703 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9704 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9705 "timeout_tags = 0x%x, port_sactive = 0x%x, "
9706 "pending_ncq_tags = 0x%x ",
9707 port, finished_tags, timeout_tags,
9708 slot_status, ahci_portp->ahciport_pending_ncq_tags);
9709 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9710 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9711 timeout_tags = tmp_timeout_tags;
9712 }
9713
9714 ahci_mop_commands(ahci_ctlp,
9715 ahci_portp,
9716 slot_status,
9717 0, /* failed tags */
9718 timeout_tags, /* timeout tags */
9719 0, /* aborted tags */
9720 0); /* reset tags */
9721
9722 mutex_exit(&ahci_portp->ahciport_mutex);
9723 }
9724
9725 /*
9726 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9727 * for long time.
9728 */
9729 static void
9730 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9731 {
9732 ahci_port_t *ahci_portp;
9733 sata_pkt_t *spkt;
9734 uint32_t pending_tags;
9735 uint32_t timeout_tags;
9736 uint32_t port_cmd_status;
9737 uint32_t port_sactive;
9738 uint8_t port;
9739 int tmp_slot;
9740 int current_slot;
9741 uint32_t current_tags;
9742 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9743
9744 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9745
9746 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9747 "ahci_watchdog_handler entered", NULL);
9748
9749 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9750 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9751 continue;
9752 }
9753
9754 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9755
9756 mutex_enter(&ahci_portp->ahciport_mutex);
9757 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9758 mutex_exit(&ahci_portp->ahciport_mutex);
9759 continue;
9760 }
9761
9762 /* Skip the check for those ports in error recovery */
9763 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9764 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9765 mutex_exit(&ahci_portp->ahciport_mutex);
9766 continue;
9767 }
9768
9769 pending_tags = 0;
9770 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9771 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9772
9773 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9774 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9775 current_slot = 0;
9776 pending_tags = 0x1;
9777 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9778 current_slot =
9779 (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9780 AHCI_CMD_STATUS_CCS_SHIFT;
9781 pending_tags = ahci_portp->ahciport_pending_tags;
9782 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9783 port_sactive = ddi_get32(
9784 ahci_ctlp->ahcictl_ahci_acc_handle,
9785 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9786 current_tags = port_sactive &
9787 ~port_cmd_status &
9788 AHCI_NCQ_SLOT_MASK(ahci_portp);
9789 pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9790 }
9791
9792 timeout_tags = 0;
9793 while (pending_tags) {
9794 tmp_slot = ddi_ffs(pending_tags) - 1;
9795 if (tmp_slot == -1) {
9796 break;
9797 }
9798
9799 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9800 spkt = ahci_portp->ahciport_err_retri_pkt;
9801 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9802 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9803 else
9804 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9805
9806 if ((spkt != NULL) && spkt->satapkt_time &&
9807 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9808 /*
9809 * If a packet has survived for more than it's
9810 * max life cycles, it is a candidate for time
9811 * out.
9812 */
9813 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9814 ahci_watchdog_timeout;
9815
9816 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9817 > 0)
9818 goto next;
9819
9820 #if AHCI_DEBUG
9821 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9822 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9823 ahci_ctlp, "watchdog: the current "
9824 "tags is 0x%x", current_tags);
9825 } else {
9826 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9827 ahci_ctlp, "watchdog: the current "
9828 "slot is %d", current_slot);
9829 }
9830 #endif
9831
9832 /*
9833 * We need to check whether the HBA has
9834 * begun to execute the command, if not,
9835 * then re-set the timer of the command.
9836 */
9837 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9838 (tmp_slot != current_slot) ||
9839 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9840 ((0x1 << tmp_slot) & current_tags)) {
9841 ahci_portp->ahciport_slot_timeout \
9842 [tmp_slot] = spkt->satapkt_time;
9843 } else {
9844 timeout_tags |= (0x1 << tmp_slot);
9845 cmn_err(CE_WARN, "!ahci%d: watchdog "
9846 "port %d satapkt 0x%p timed out\n",
9847 instance, port, (void *)spkt);
9848 }
9849 }
9850 next:
9851 CLEAR_BIT(pending_tags, tmp_slot);
9852 }
9853
9854 if (timeout_tags) {
9855 mutex_exit(&ahci_portp->ahciport_mutex);
9856 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9857 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9858 port, timeout_tags);
9859 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9860 mutex_enter(&ahci_portp->ahciport_mutex);
9861 }
9862
9863 mutex_exit(&ahci_portp->ahciport_mutex);
9864 }
9865
9866 /* Re-install the watchdog timeout handler */
9867 if (ahci_ctlp->ahcictl_timeout_id != 0) {
9868 ahci_ctlp->ahcictl_timeout_id =
9869 timeout((void (*)(void *))ahci_watchdog_handler,
9870 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9871 }
9872
9873 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9874 }
9875
9876 /*
9877 * Fill the error context into sata_cmd for non-queued command error.
9878 */
9879 static void
9880 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9881 {
9882 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9883 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9884 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9885 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9886 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9887 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9888 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9889
9890 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9891 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9892 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9893 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9894 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9895 }
9896 }
9897
9898 /*
9899 * Fill the ncq error page into sata_cmd for queued command error.
9900 */
9901 static void
9902 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9903 struct sata_ncq_error_recovery_page *ncq_err_page)
9904 {
9905 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9906 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9907 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
9908 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
9909 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
9910 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
9911 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
9912 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
9913 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
9914 scmd->satacmd_status_reg = ncq_err_page->ncq_status;
9915 scmd->satacmd_error_reg = ncq_err_page->ncq_error;
9916 }
9917
9918 /*
9919 * Put the respective register value to sata_cmd_t for satacmd_flags.
9920 */
9921 static void
9922 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9923 {
9924 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
9925 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9926 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
9927 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9928 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
9929 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9930 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
9931 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9932 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
9933 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9934 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
9935 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9936 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
9937 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9938 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
9939 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9940 if (scmd->satacmd_flags.sata_copy_out_device_reg)
9941 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9942 if (scmd->satacmd_flags.sata_copy_out_error_reg)
9943 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9944 }
9945
9946 static void
9947 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
9948 uint32_t intr_status)
9949 {
9950 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9951
9952 if (intr_status & AHCI_INTR_STATUS_IFS)
9953 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
9954 "error", instance, port);
9955
9956 if (intr_status & AHCI_INTR_STATUS_HBDS)
9957 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
9958 instance, port);
9959
9960 if (intr_status & AHCI_INTR_STATUS_HBFS)
9961 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
9962 instance, port);
9963
9964 if (intr_status & AHCI_INTR_STATUS_TFES)
9965 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
9966 instance, port);
9967
9968 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
9969 "recovery", instance, port);
9970 }
9971
9972 static void
9973 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
9974 uint32_t slot_tags)
9975 {
9976 ahci_port_t *ahci_portp;
9977 int tmp_slot;
9978 sata_pkt_t *spkt;
9979 sata_cmd_t cmd;
9980
9981 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9982 ASSERT(ahci_portp != NULL);
9983
9984 while (slot_tags) {
9985 tmp_slot = ddi_ffs(slot_tags) - 1;
9986 if (tmp_slot == -1) {
9987 break;
9988 }
9989
9990 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9991 ASSERT(spkt != NULL);
9992 cmd = spkt->satapkt_cmd;
9993
9994 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
9995 "features_reg = 0x%x sec_count_msb = 0x%x "
9996 "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
9997 "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
9998 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
9999 "lba_high_lsb = 0x%x device_reg = 0x%x "
10000 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10001 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10002 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10003 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10004 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10005 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10006 cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10007 *((uint32_t *)&(cmd.satacmd_flags)));
10008
10009 CLEAR_BIT(slot_tags, tmp_slot);
10010 }
10011 }
10012
10013 /*
10014 * Dump the serror message to the log.
10015 */
10016 static void
10017 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10018 uint32_t port_serror, int debug_only)
10019 {
10020 static char err_buf[512];
10021 static char err_msg_header[16];
10022 char *err_msg = err_buf;
10023
10024 *err_buf = '\0';
10025 *err_msg_header = '\0';
10026
10027 if (port_serror & SERROR_DATA_ERR_FIXED) {
10028 err_msg = strcat(err_msg,
10029 "\tRecovered Data Integrity Error (I)\n");
10030 }
10031
10032 if (port_serror & SERROR_COMM_ERR_FIXED) {
10033 err_msg = strcat(err_msg,
10034 "\tRecovered Communication Error (M)\n");
10035 }
10036
10037 if (port_serror & SERROR_DATA_ERR) {
10038 err_msg = strcat(err_msg,
10039 "\tTransient Data Integrity Error (T)\n");
10040 }
10041
10042 if (port_serror & SERROR_PERSISTENT_ERR) {
10043 err_msg = strcat(err_msg,
10044 "\tPersistent Communication or Data Integrity Error (C)\n");
10045 }
10046
10047 if (port_serror & SERROR_PROTOCOL_ERR) {
10048 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10049 }
10050
10051 if (port_serror & SERROR_INT_ERR) {
10052 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10053 }
10054
10055 if (port_serror & SERROR_PHY_RDY_CHG) {
10056 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10057 }
10058
10059 if (port_serror & SERROR_PHY_INT_ERR) {
10060 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10061 }
10062
10063 if (port_serror & SERROR_COMM_WAKE) {
10064 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10065 }
10066
10067 if (port_serror & SERROR_10B_TO_8B_ERR) {
10068 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10069 }
10070
10071 if (port_serror & SERROR_DISPARITY_ERR) {
10072 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10073 }
10074
10075 if (port_serror & SERROR_CRC_ERR) {
10076 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10077 }
10078
10079 if (port_serror & SERROR_HANDSHAKE_ERR) {
10080 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10081 }
10082
10083 if (port_serror & SERROR_LINK_SEQ_ERR) {
10084 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10085 }
10086
10087 if (port_serror & SERROR_TRANS_ERR) {
10088 err_msg = strcat(err_msg,
10089 "\tTransport state transition error (T)\n");
10090 }
10091
10092 if (port_serror & SERROR_FIS_TYPE) {
10093 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10094 }
10095
10096 if (port_serror & SERROR_EXCHANGED_ERR) {
10097 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10098 }
10099
10100 if (*err_msg == '\0')
10101 return;
10102
10103 if (debug_only) {
10104 (void) sprintf(err_msg_header, "port %d", port);
10105 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10106 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10107 } else if (ahci_ctlp) {
10108 cmn_err(CE_WARN, "!ahci%d: %s %s",
10109 ddi_get_instance(ahci_ctlp->ahcictl_dip),
10110 err_msg_header, err_msg);
10111
10112 /* sata trace debug */
10113 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10114 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10115 err_msg_header, err_msg);
10116 } else {
10117 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10118
10119 /* sata trace debug */
10120 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10121 }
10122 }
10123
10124 /*
10125 * Translate the sata_address_t type into the ahci_addr_t type.
10126 * sata_device.satadev_addr structure is used as source.
10127 */
10128 static void
10129 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10130 ahci_addr_t *ahci_addrp)
10131 {
10132 sata_address_t *sata_addrp = &sd->satadev_addr;
10133 ahci_addrp->aa_port =
10134 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10135 ahci_addrp->aa_pmport = sata_addrp->pmport;
10136
10137 switch (sata_addrp->qual) {
10138 case SATA_ADDR_DCPORT:
10139 case SATA_ADDR_CPORT:
10140 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10141 break;
10142 case SATA_ADDR_PMULT:
10143 case SATA_ADDR_PMULT_SPEC:
10144 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10145 break;
10146 case SATA_ADDR_DPMPORT:
10147 case SATA_ADDR_PMPORT:
10148 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10149 break;
10150 case SATA_ADDR_NULL:
10151 default:
10152 /* something went wrong */
10153 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10154 break;
10155 }
10156 }
10157
10158 /*
10159 * This routine is to calculate the total number of ports implemented
10160 * by the HBA.
10161 */
10162 static int
10163 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10164 {
10165 uint8_t i;
10166 int num = 0;
10167
10168 for (i = 0; i < AHCI_MAX_PORTS; i++) {
10169 if (((uint32_t)0x1 << i) & ports_implemented)
10170 num++;
10171 }
10172
10173 return (num);
10174 }
10175
10176 #if AHCI_DEBUG
10177 static void
10178 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10179 {
10180 static char name[16];
10181 va_list ap;
10182
10183 mutex_enter(&ahci_log_mutex);
10184
10185 va_start(ap, fmt);
10186 if (ahci_ctlp) {
10187 (void) sprintf(name, "ahci%d: ",
10188 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10189 } else {
10190 (void) sprintf(name, "ahci: ");
10191 }
10192
10193 (void) vsprintf(ahci_log_buf, fmt, ap);
10194 va_end(ap);
10195
10196 cmn_err(level, "%s%s", name, ahci_log_buf);
10197
10198 mutex_exit(&ahci_log_mutex);
10199 }
10200 #endif
10201
10202 /*
10203 * quiesce(9E) entry point.
10204 *
10205 * This function is called when the system is single-threaded at high
10206 * PIL with preemption disabled. Therefore, this function must not be
10207 * blocked.
10208 *
10209 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10210 * DDI_FAILURE indicates an error condition and should almost never happen.
10211 */
10212 static int
10213 ahci_quiesce(dev_info_t *dip)
10214 {
10215 ahci_ctl_t *ahci_ctlp;
10216 ahci_port_t *ahci_portp;
10217 int instance, port;
10218
10219 instance = ddi_get_instance(dip);
10220 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10221
10222 if (ahci_ctlp == NULL)
10223 return (DDI_FAILURE);
10224
10225 #if AHCI_DEBUG
10226 ahci_debug_flags = 0;
10227 #endif
10228
10229 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10230
10231 /* disable all the interrupts. */
10232 ahci_disable_all_intrs(ahci_ctlp);
10233
10234 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10235 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10236 continue;
10237 }
10238
10239 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10240
10241 /*
10242 * Stop the port by clearing PxCMD.ST
10243 *
10244 * Here we must disable the port interrupt because
10245 * ahci_disable_all_intrs only clear GHC.IE, and IS
10246 * register will be still set if PxIE is enabled.
10247 * When ahci shares one IRQ with other drivers, the
10248 * intr handler may claim the intr mistakenly.
10249 */
10250 ahci_disable_port_intrs(ahci_ctlp, port);
10251 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10252 ahci_portp, port);
10253 }
10254
10255 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10256
10257 return (DDI_SUCCESS);
10258 }
10259
10260 /*
10261 * The function will add a sata packet to the done queue.
10262 */
10263 static void
10264 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10265 {
10266 ASSERT(satapkt != NULL);
10267 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10268
10269 /* set the reason for all packets */
10270 satapkt->satapkt_reason = reason;
10271 satapkt->satapkt_hba_driver_private = NULL;
10272
10273 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10274 satapkt->satapkt_comp) {
10275 /*
10276 * only add to queue when mode is not synch and there is
10277 * completion callback
10278 */
10279 *ahci_portp->ahciport_doneqtail = satapkt;
10280 ahci_portp->ahciport_doneqtail =
10281 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10282 ahci_portp->ahciport_doneq_len++;
10283
10284 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10285 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10286 /*
10287 * for sync/non-poll mode, just call cv_broadcast
10288 */
10289 cv_broadcast(&ahci_portp->ahciport_cv);
10290 }
10291
10292 /*
10293 * The function will call completion callback of sata packet on the
10294 * completed queue
10295 */
10296 static void
10297 ahci_flush_doneq(ahci_port_t *ahci_portp)
10298 {
10299 sata_pkt_t *satapkt, *next;
10300
10301 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10302
10303 if (ahci_portp->ahciport_doneq) {
10304 satapkt = ahci_portp->ahciport_doneq;
10305
10306 ahci_portp->ahciport_doneq = NULL;
10307 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10308 ahci_portp->ahciport_doneq_len = 0;
10309
10310 mutex_exit(&ahci_portp->ahciport_mutex);
10311
10312 while (satapkt != NULL) {
10313 next = satapkt->satapkt_hba_driver_private;
10314 satapkt->satapkt_hba_driver_private = NULL;
10315
10316 /* Call the callback */
10317 (*satapkt->satapkt_comp)(satapkt);
10318
10319 satapkt = next;
10320 }
10321
10322 mutex_enter(&ahci_portp->ahciport_mutex);
10323 }
10324 }