1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2018, Joyent, Inc.
26 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
27 */
28
29 /*
30 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
31 *
32 * Power Management Support
33 * ------------------------
34 *
35 * At the moment, the ahci driver only implements suspend/resume to
36 * support Suspend to RAM on X86 feature. Device power management isn't
37 * implemented, link power management is disabled, and hot plug isn't
38 * allowed during the period from suspend to resume.
39 *
40 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
41 * and DDI_RESUME entries, and don't need to take care of new requests
42 * sent down after suspend because the target driver (sd) has already
43 * handled these conditions, and blocked these requests. For the detailed
44 * information, please check with sdopen, sdclose and sdioctl routines.
45 *
46 *
47 * Enclosure Management Support
48 * ----------------------------
49 *
50 * The ahci driver has basic support for AHCI Enclosure Management (EM)
51 * services. The AHCI specification provides an area in the primary ahci BAR for
52 * posting data to send out to the enclosure management and provides a register
53 * that provides both information and control about this. While the
54 * specification allows for multiple forms of enclosure management, the only
55 * supported, and commonly found form, is the AHCI specified LED format. The LED
56 * format is often implemented as a one-way communication mechanism. Software
57 * can write out what it cares about into the aforementioned data buffer and
58 * then we wait for the transmission to be sent.
59 *
60 * This has some drawbacks. It means that we cannot know whether or not it has
61 * succeeded. This means we cannot ask hardware what it thinks the LEDs are
62 * set to. There's also the added unfortunate reality that firmware on the
63 * microcontroller driving this will often not show the LEDs if no drive is
64 * present and that actions taken may potentially cause this to get out of sync
65 * with what we expect it to be. For example, the specification does not
66 * describe what should happen if a drive is removed from the enclosure while
67 * this is set and what should happen when it returns. We can only infer that it
68 * should be the same.
69 *
70 * Because only a single command can be sent at any time and we don't want to
71 * interfere with controller I/O, we create a taskq dedicated to this that has a
72 * single thread. Both resets (which occur on attach and resume) and normal
73 * changes to the LED state will be driven through this taskq. Because the taskq
74 * has a single thread, this guarantees serial processing.
75 *
76 * Each userland-submitted task (basically not resets) has a reference counted
77 * task structure. This allows the thread that called it to be cancelled and
78 * have the system clean itself up. The user thread in ioctl blocks on a CV that
79 * can receive signals as it waits for completion. Note, there is no guarantee
80 * provided by the kernel that the first thread to enter the kernel will be the
81 * first one to change state.
82 */
83
84 #include <sys/note.h>
85 #include <sys/scsi/scsi.h>
86 #include <sys/pci.h>
87 #include <sys/disp.h>
88 #include <sys/sata/sata_hba.h>
89 #include <sys/sata/adapters/ahci/ahcireg.h>
90 #include <sys/sata/adapters/ahci/ahcivar.h>
91
92 /*
93 * FMA header files
94 */
95 #include <sys/ddifm.h>
96 #include <sys/fm/protocol.h>
97 #include <sys/fm/util.h>
98 #include <sys/fm/io/ddi.h>
99
100 /*
101 * EM Control header files
102 */
103 #include <sys/types.h>
104 #include <sys/file.h>
105 #include <sys/errno.h>
106 #include <sys/open.h>
107 #include <sys/cred.h>
108 #include <sys/ddi.h>
109 #include <sys/sunddi.h>
110
111 /*
112 * This is the string displayed by modinfo, etc.
113 */
114 static char ahci_ident[] = "ahci driver";
115
116 /*
117 * Function prototypes for driver entry points
118 */
119 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
120 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
121 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
122 static int ahci_quiesce(dev_info_t *);
123
124 /*
125 * Function prototypes for SATA Framework interfaces
126 */
127 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
128 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
129
130 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
131 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
132 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
133 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
134 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
135 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
136 #if defined(__lock_lint)
137 static int ahci_selftest(dev_info_t *, sata_device_t *);
138 #endif
139
140 /*
141 * FMA Prototypes
142 */
143 static void ahci_fm_init(ahci_ctl_t *);
144 static void ahci_fm_fini(ahci_ctl_t *);
145 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
146 int ahci_check_acc_handle(ddi_acc_handle_t);
147 int ahci_check_dma_handle(ddi_dma_handle_t);
148 void ahci_fm_ereport(ahci_ctl_t *, char *);
149 static int ahci_check_all_handle(ahci_ctl_t *);
150 static int ahci_check_ctl_handle(ahci_ctl_t *);
151 static int ahci_check_port_handle(ahci_ctl_t *, int);
152 static int ahci_check_slot_handle(ahci_port_t *, int);
153
154 /*
155 * Local function prototypes
156 */
157 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
158 static int ahci_alloc_ports_state(ahci_ctl_t *);
159 static void ahci_dealloc_ports_state(ahci_ctl_t *);
160 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
161 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
162 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
163 static void ahci_dealloc_rcvd_fis(ahci_port_t *);
164 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
165 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
166 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
167 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
168 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
169 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
170
171 static int ahci_initialize_controller(ahci_ctl_t *);
172 static void ahci_uninitialize_controller(ahci_ctl_t *);
173 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
174 static int ahci_config_space_init(ahci_ctl_t *);
175 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
176
177 static void ahci_drain_ports_taskq(ahci_ctl_t *);
178 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
179 uint8_t);
180 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
181 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
182 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
183 sata_device_t *);
184 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
185 sata_pmult_gscr_t *);
186 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
187 sata_device_t *);
188 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
189 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
190 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
191 sata_device_t *);
192
193 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
194 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
195 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
196 ahci_addr_t *);
197 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
198 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
199 ahci_addr_t *, sata_pkt_t *);
200 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
201 ahci_addr_t *, sata_pkt_t *);
202 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
203 ahci_addr_t *, int);
204 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
205 static void ahci_copy_ncq_err_page(sata_cmd_t *,
206 struct sata_ncq_error_recovery_page *);
207 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
208 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
209 static void ahci_flush_doneq(ahci_port_t *);
210
211 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
212 static int ahci_hba_reset(ahci_ctl_t *);
213 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
214 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
215 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
216 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
217 ahci_addr_t *);
218 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
219 ahci_addr_t *);
220 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
221 ahci_addr_t *);
222 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
223 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
224 uint8_t);
225 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
226 uint8_t, int, int *);
227 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
228 uint32_t, uint32_t, uint32_t, uint32_t);
229 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
230 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
231 uint8_t, sata_pkt_t *);
232 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
233 ahci_addr_t *, uint32_t);
234 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
235 uint8_t, uint32_t);
236 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
237 uint8_t, uint32_t);
238 static void ahci_events_handler(void *);
239 static void ahci_watchdog_handler(ahci_ctl_t *);
240
241 static uint_t ahci_intr(caddr_t, caddr_t);
242 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
243 static int ahci_add_intrs(ahci_ctl_t *, int);
244 static void ahci_rem_intrs(ahci_ctl_t *);
245 static void ahci_enable_all_intrs(ahci_ctl_t *);
246 static void ahci_disable_all_intrs(ahci_ctl_t *);
247 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
248 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
249
250 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
251 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
252 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
253 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
254 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
255 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
256 ahci_port_t *, uint8_t);
257 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
258 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
259 uint8_t, uint32_t);
260 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
261 uint8_t, uint32_t);
262 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
263
264 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
265 static int ahci_get_num_implemented_ports(uint32_t);
266 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
267 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
268 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
269 #if AHCI_DEBUG
270 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
271 #endif
272
273 static boolean_t ahci_em_init(ahci_ctl_t *);
274 static void ahci_em_fini(ahci_ctl_t *);
275 static void ahci_em_suspend(ahci_ctl_t *);
276 static void ahci_em_resume(ahci_ctl_t *);
277 static int ahci_em_ioctl(dev_info_t *, int, intptr_t);
278
279
280 /*
281 * DMA attributes for the data buffer
282 *
283 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
284 * does not support 64-bit addressing
285 */
286 static ddi_dma_attr_t buffer_dma_attr = {
287 DMA_ATTR_V0, /* dma_attr_version */
288 0x0ull, /* dma_attr_addr_lo: lowest bus address */
289 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
290 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */
291 0x2ull, /* dma_attr_align: word aligned */
292 1, /* dma_attr_burstsizes */
293 1, /* dma_attr_minxfer */
294 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
295 0xffffffffull, /* dma_attr_seg */
296 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */
297 512, /* dma_attr_granular */
298 0, /* dma_attr_flags */
299 };
300
301 /*
302 * DMA attributes for the rcvd FIS
303 *
304 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
305 * does not support 64-bit addressing
306 */
307 static ddi_dma_attr_t rcvd_fis_dma_attr = {
308 DMA_ATTR_V0, /* dma_attr_version */
309 0x0ull, /* dma_attr_addr_lo: lowest bus address */
310 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
311 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
312 0x100ull, /* dma_attr_align: 256-byte aligned */
313 1, /* dma_attr_burstsizes */
314 1, /* dma_attr_minxfer */
315 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
316 0xffffffffull, /* dma_attr_seg */
317 1, /* dma_attr_sgllen */
318 1, /* dma_attr_granular */
319 0, /* dma_attr_flags */
320 };
321
322 /*
323 * DMA attributes for the command list
324 *
325 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
326 * does not support 64-bit addressing
327 */
328 static ddi_dma_attr_t cmd_list_dma_attr = {
329 DMA_ATTR_V0, /* dma_attr_version */
330 0x0ull, /* dma_attr_addr_lo: lowest bus address */
331 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
332 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
333 0x400ull, /* dma_attr_align: 1K-byte aligned */
334 1, /* dma_attr_burstsizes */
335 1, /* dma_attr_minxfer */
336 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
337 0xffffffffull, /* dma_attr_seg */
338 1, /* dma_attr_sgllen */
339 1, /* dma_attr_granular */
340 0, /* dma_attr_flags */
341 };
342
343 /*
344 * DMA attributes for cmd tables
345 *
346 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
347 * does not support 64-bit addressing
348 */
349 static ddi_dma_attr_t cmd_table_dma_attr = {
350 DMA_ATTR_V0, /* dma_attr_version */
351 0x0ull, /* dma_attr_addr_lo: lowest bus address */
352 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
353 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
354 0x80ull, /* dma_attr_align: 128-byte aligned */
355 1, /* dma_attr_burstsizes */
356 1, /* dma_attr_minxfer */
357 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
358 0xffffffffull, /* dma_attr_seg */
359 1, /* dma_attr_sgllen */
360 1, /* dma_attr_granular */
361 0, /* dma_attr_flags */
362 };
363
364
365 /* Device access attributes */
366 static ddi_device_acc_attr_t accattr = {
367 DDI_DEVICE_ATTR_V1,
368 DDI_STRUCTURE_LE_ACC,
369 DDI_STRICTORDER_ACC,
370 DDI_DEFAULT_ACC
371 };
372
373 static struct dev_ops ahcictl_dev_ops = {
374 DEVO_REV, /* devo_rev */
375 0, /* refcnt */
376 ahci_getinfo, /* info */
377 nulldev, /* identify */
378 nulldev, /* probe */
379 ahci_attach, /* attach */
380 ahci_detach, /* detach */
381 nodev, /* no reset */
382 NULL, /* driver operations */
383 NULL, /* bus operations */
384 NULL, /* power */
385 ahci_quiesce, /* quiesce */
386 };
387
388 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
389 SATA_TRAN_HOTPLUG_OPS_REV_1,
390 ahci_tran_hotplug_port_activate,
391 ahci_tran_hotplug_port_deactivate
392 };
393
394 extern struct mod_ops mod_driverops;
395
396 static struct modldrv modldrv = {
397 &mod_driverops, /* driverops */
398 ahci_ident, /* short description */
399 &ahcictl_dev_ops, /* driver ops */
400 };
401
402 static struct modlinkage modlinkage = {
403 MODREV_1,
404 &modldrv,
405 NULL
406 };
407
408 /* The following variables are watchdog handler related */
409 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
410 static clock_t ahci_watchdog_tick;
411
412 /*
413 * This static variable indicates the size of command table,
414 * and it's changeable with prdt number, which ahci_dma_prdt_number
415 * indicates.
416 */
417 static size_t ahci_cmd_table_size;
418
419 /*
420 * The below global variables are tunable via /etc/system
421 *
422 * ahci_dma_prdt_number
423 * ahci_msi_enabled
424 * ahci_buf_64bit_dma
425 * ahci_commu_64bit_dma
426 */
427
428 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
429 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
430
431 /* AHCI MSI is tunable */
432 boolean_t ahci_msi_enabled = B_TRUE;
433
434 /*
435 * 64-bit dma addressing for data buffer is tunable
436 *
437 * The variable controls only the below value:
438 * DBAU (upper 32-bits physical address of data block)
439 */
440 boolean_t ahci_buf_64bit_dma = B_TRUE;
441
442 /*
443 * 64-bit dma addressing for communication system descriptors is tunable
444 *
445 * The variable controls the below three values:
446 *
447 * PxCLBU (upper 32-bits for the command list base physical address)
448 * PxFBU (upper 32-bits for the received FIS base physical address)
449 * CTBAU (upper 32-bits of command table base)
450 */
451 boolean_t ahci_commu_64bit_dma = B_TRUE;
452
453 /*
454 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
455 * chipset. If the users want to have a try with 64-bit dma, please change
456 * the below variable value to enable it.
457 */
458 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
459
460 /*
461 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
462 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
463 * please change the below value to enable it.
464 */
465 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
466
467 /*
468 * These values control the default delay and default number of times to wait
469 * for an enclosure message to complete.
470 */
471 uint_t ahci_em_reset_delay_ms = 1;
472 uint_t ahci_em_reset_delay_count = 1000;
473 uint_t ahci_em_tx_delay_ms = 1;
474 uint_t ahci_em_tx_delay_count = 1000;
475
476
477 /*
478 * End of global tunable variable definition
479 */
480
481 #if AHCI_DEBUG
482 uint32_t ahci_debug_flags = 0;
483 #else
484 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
485 #endif
486
487
488 #if AHCI_DEBUG
489 /* The following is needed for ahci_log() */
490 static kmutex_t ahci_log_mutex;
491 static char ahci_log_buf[512];
492 #endif
493
494 /* Opaque state pointer initialized by ddi_soft_state_init() */
495 static void *ahci_statep = NULL;
496
497 /*
498 * ahci module initialization.
499 */
500 int
501 _init(void)
502 {
503 int ret;
504
505 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
506 if (ret != 0) {
507 goto err_out;
508 }
509
510 #if AHCI_DEBUG
511 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
512 #endif
513
514 if ((ret = sata_hba_init(&modlinkage)) != 0) {
515 #if AHCI_DEBUG
516 mutex_destroy(&ahci_log_mutex);
517 #endif
518 ddi_soft_state_fini(&ahci_statep);
519 goto err_out;
520 }
521
522 /* watchdog tick */
523 ahci_watchdog_tick = drv_usectohz(
524 (clock_t)ahci_watchdog_timeout * 1000000);
525
526 ret = mod_install(&modlinkage);
527 if (ret != 0) {
528 sata_hba_fini(&modlinkage);
529 #if AHCI_DEBUG
530 mutex_destroy(&ahci_log_mutex);
531 #endif
532 ddi_soft_state_fini(&ahci_statep);
533 goto err_out;
534 }
535
536 return (ret);
537
538 err_out:
539 cmn_err(CE_WARN, "!ahci: Module init failed");
540 return (ret);
541 }
542
543 /*
544 * ahci module uninitialize.
545 */
546 int
547 _fini(void)
548 {
549 int ret;
550
551 ret = mod_remove(&modlinkage);
552 if (ret != 0) {
553 return (ret);
554 }
555
556 /* Remove the resources allocated in _init(). */
557 sata_hba_fini(&modlinkage);
558 #if AHCI_DEBUG
559 mutex_destroy(&ahci_log_mutex);
560 #endif
561 ddi_soft_state_fini(&ahci_statep);
562
563 return (ret);
564 }
565
566 /*
567 * _info entry point
568 */
569 int
570 _info(struct modinfo *modinfop)
571 {
572 return (mod_info(&modlinkage, modinfop));
573 }
574
575 /*
576 * The attach entry point for dev_ops.
577 */
578 static int
579 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
580 {
581 ahci_ctl_t *ahci_ctlp = NULL;
582 int instance = ddi_get_instance(dip);
583 int status;
584 int attach_state;
585 uint32_t cap_status, ahci_version;
586 uint32_t ghc_control;
587 int intr_types;
588 int i;
589 pci_regspec_t *regs;
590 int regs_length;
591 int rnumber;
592 #if AHCI_DEBUG
593 int speed;
594 #endif
595
596 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
597 NULL);
598
599 switch (cmd) {
600 case DDI_ATTACH:
601 break;
602
603 case DDI_RESUME:
604
605 /*
606 * During DDI_RESUME, the hardware state of the device
607 * (power may have been removed from the device) must be
608 * restored, allow pending requests to continue, and
609 * service new requests.
610 */
611 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
612 mutex_enter(&ahci_ctlp->ahcictl_mutex);
613
614 /*
615 * GHC.AE must be set to 1 before any other AHCI register
616 * is accessed
617 */
618 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
619 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
620 ghc_control |= AHCI_HBA_GHC_AE;
621 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
622 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
623
624 /* Restart watch thread */
625 if (ahci_ctlp->ahcictl_timeout_id == 0)
626 ahci_ctlp->ahcictl_timeout_id = timeout(
627 (void (*)(void *))ahci_watchdog_handler,
628 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
629
630 mutex_exit(&ahci_ctlp->ahcictl_mutex);
631
632 /*
633 * Re-initialize the controller and enable the interrupts and
634 * restart all the ports.
635 *
636 * Note that so far we don't support hot-plug during
637 * suspend/resume.
638 */
639 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
640 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
641 "Failed to initialize the controller "
642 "during DDI_RESUME", NULL);
643 return (DDI_FAILURE);
644 }
645
646 /*
647 * Reset the enclosure services.
648 */
649 ahci_em_resume(ahci_ctlp);
650
651 mutex_enter(&ahci_ctlp->ahcictl_mutex);
652 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
653 mutex_exit(&ahci_ctlp->ahcictl_mutex);
654
655 return (DDI_SUCCESS);
656
657 default:
658 return (DDI_FAILURE);
659 }
660
661 attach_state = AHCI_ATTACH_STATE_NONE;
662
663 /* Allocate soft state */
664 status = ddi_soft_state_zalloc(ahci_statep, instance);
665 if (status != DDI_SUCCESS) {
666 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
667 instance);
668 goto err_out;
669 }
670
671 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
672 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
673 ahci_ctlp->ahcictl_dip = dip;
674
675 /* Initialize the cport/port mapping */
676 for (i = 0; i < AHCI_MAX_PORTS; i++) {
677 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
678 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
679 }
680
681 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
682
683 /* Initialize FMA properties */
684 ahci_fm_init(ahci_ctlp);
685
686 attach_state |= AHCI_ATTACH_STATE_FMA;
687
688 /*
689 * Now map the AHCI base address; which includes global
690 * registers and port control registers
691 *
692 * According to the spec, the AHCI Base Address is BAR5,
693 * but BAR0-BAR4 are optional, so we need to check which
694 * rnumber is used for BAR5.
695 */
696
697 /*
698 * search through DDI "reg" property for the AHCI register set
699 */
700 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
701 DDI_PROP_DONTPASS, "reg", (int **)®s,
702 (uint_t *)®s_length) != DDI_PROP_SUCCESS) {
703 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
704 instance);
705 goto err_out;
706 }
707
708 /* AHCI Base Address is located at 0x24 offset */
709 for (rnumber = 0; rnumber < regs_length; ++rnumber) {
710 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
711 == AHCI_PCI_RNUM)
712 break;
713 }
714
715 ddi_prop_free(regs);
716
717 if (rnumber == regs_length) {
718 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
719 instance);
720 goto err_out;
721 }
722
723 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
724
725 status = ddi_regs_map_setup(dip,
726 rnumber,
727 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
728 0,
729 0,
730 &accattr,
731 &ahci_ctlp->ahcictl_ahci_acc_handle);
732 if (status != DDI_SUCCESS) {
733 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
734 instance);
735 goto err_out;
736 }
737
738 attach_state |= AHCI_ATTACH_STATE_REG_MAP;
739
740 /*
741 * GHC.AE must be set to 1 before any other AHCI register
742 * is accessed
743 */
744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
746 ghc_control |= AHCI_HBA_GHC_AE;
747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
748 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
749
750 /* Get the AHCI version information */
751 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
752 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
753
754 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
755 (ahci_version & 0xffff0000) >> 16,
756 ((ahci_version & 0x0000ff00) >> 4 |
757 (ahci_version & 0x000000ff)));
758
759 /* We don't support controllers whose versions are lower than 1.0 */
760 if (!(ahci_version & 0xffff0000)) {
761 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
762 "than version 1.0", instance);
763 goto err_out;
764 }
765
766 /* Get the HBA capabilities information */
767 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
768 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
769
770 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
771 cap_status);
772
773 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
774 if (ahci_version >= 0x00010200) {
775 uint32_t cap2_status;
776
777 /* Get the HBA capabilities extended information */
778 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
779 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
780
781 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
782 "hba capabilities extended = 0x%x", cap2_status);
783 }
784
785 if (cap_status & AHCI_HBA_CAP_EMS) {
786 ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS;
787 ahci_ctlp->ahcictl_em_loc =
788 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
789 (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp));
790 ahci_ctlp->ahcictl_em_ctl =
791 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
792 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
793 }
794
795 #if AHCI_DEBUG
796 /* Get the interface speed supported by the HBA */
797 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
798 if (speed == 0x01) {
799 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
800 "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
801 } else if (speed == 0x10) {
802 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
803 "hba interface speed support: Gen 2 (3 Gbps)", NULL);
804 } else if (speed == 0x11) {
805 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
806 "hba interface speed support: Gen 3 (6 Gbps)", NULL);
807 }
808 #endif
809
810 /* Get the number of command slots supported by the HBA */
811 ahci_ctlp->ahcictl_num_cmd_slots =
812 ((cap_status & AHCI_HBA_CAP_NCS) >>
813 AHCI_HBA_CAP_NCS_SHIFT) + 1;
814
815 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
816 ahci_ctlp->ahcictl_num_cmd_slots);
817
818 /* Get the bit map which indicates ports implemented by the HBA */
819 ahci_ctlp->ahcictl_ports_implemented =
820 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
821 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
822
823 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
824 ahci_ctlp->ahcictl_ports_implemented);
825
826 /* Max port number implemented */
827 ahci_ctlp->ahcictl_num_ports =
828 ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
829
830 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
831 (cap_status & AHCI_HBA_CAP_NP) + 1);
832
833 /* Get the number of implemented ports by the HBA */
834 ahci_ctlp->ahcictl_num_implemented_ports =
835 ahci_get_num_implemented_ports(
836 ahci_ctlp->ahcictl_ports_implemented);
837
838 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
839 "hba number of implemented ports: %d",
840 ahci_ctlp->ahcictl_num_implemented_ports);
841
842 /* Check whether HBA supports 64bit DMA addressing */
843 if (!(cap_status & AHCI_HBA_CAP_S64A)) {
844 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
845 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
846 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
847 "hba does not support 64-bit addressing", NULL);
848 }
849
850 /* Checking for the support of Port Multiplier */
851 if (cap_status & AHCI_HBA_CAP_SPM) {
852 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
853 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
854 "hba supports port multiplier (CBSS)", NULL);
855
856 /* Support FIS-based switching ? */
857 if (cap_status & AHCI_HBA_CAP_FBSS) {
858 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
859 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
860 "hba supports FIS-based switching (FBSS)", NULL);
861 }
862 }
863
864 /* Checking for Support Command List Override */
865 if (cap_status & AHCI_HBA_CAP_SCLO) {
866 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
867 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
868 "hba supports command list override.", NULL);
869 }
870
871 /* Checking for Asynchronous Notification */
872 if (cap_status & AHCI_HBA_CAP_SSNTF) {
873 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
874 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
875 "hba supports asynchronous notification.", NULL);
876 }
877
878 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
879 != DDI_SUCCESS) {
880 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
881 instance);
882 goto err_out;
883 }
884
885 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
886
887 /*
888 * Check the pci configuration space, and set caps. We also
889 * handle the hardware defect in this function.
890 *
891 * For example, force ATI SB600 to use 32-bit dma addressing
892 * since it doesn't support 64-bit dma though its CAP register
893 * declares it support.
894 */
895 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
896 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
897 instance);
898 goto err_out;
899 }
900
901 /*
902 * Disable the whole controller interrupts before adding
903 * interrupt handlers(s).
904 */
905 ahci_disable_all_intrs(ahci_ctlp);
906
907 /* Get supported interrupt types */
908 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
909 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
910 instance);
911 goto err_out;
912 }
913
914 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
915 "ddi_intr_get_supported_types() returned: 0x%x",
916 intr_types);
917
918 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
919 /*
920 * Try MSI first, but fall back to FIXED if failed
921 */
922 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
923 DDI_SUCCESS) {
924 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
925 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
926 "Using MSI interrupt type", NULL);
927 goto intr_done;
928 }
929
930 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
931 "MSI registration failed, "
932 "trying FIXED interrupts", NULL);
933 }
934
935 if (intr_types & DDI_INTR_TYPE_FIXED) {
936 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
937 DDI_SUCCESS) {
938 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
939 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
940 "Using FIXED interrupt type", NULL);
941 goto intr_done;
942 }
943
944 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
945 "FIXED interrupt registration failed", NULL);
946 }
947
948 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
949
950 goto err_out;
951
952 intr_done:
953
954 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
955
956 /* Initialize the controller mutex */
957 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
958 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
959
960 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
961
962 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
963 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
964 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
965 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
966 }
967
968 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
969 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
970 sizeof (ahci_prdt_item_t));
971
972 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
973 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
974 " ahci_cmd_table_size is 0x%x",
975 ahci_dma_prdt_number, ahci_cmd_table_size);
976
977 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
978 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
979 ahci_dma_prdt_number;
980
981 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
982 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
983 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
984 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
985
986 /*
987 * enable 64bit dma for data buffer for SB600 if
988 * sb600_buf_64bit_dma_disable is B_FALSE
989 */
990 if ((ahci_buf_64bit_dma == B_FALSE) ||
991 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
992 !(sb600_buf_64bit_dma_disable == B_FALSE &&
993 ahci_ctlp->ahcictl_venid == 0x1002 &&
994 ahci_ctlp->ahcictl_devid == 0x4380))) {
995 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
996 0xffffffffull;
997 }
998
999 /*
1000 * enable 64bit dma for command buffer for SB600/700/710/800
1001 * if sbxxx_commu_64bit_dma_disable is B_FALSE
1002 */
1003 if ((ahci_commu_64bit_dma == B_FALSE) ||
1004 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
1005 !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
1006 ahci_ctlp->ahcictl_venid == 0x1002 &&
1007 (ahci_ctlp->ahcictl_devid == 0x4380 ||
1008 ahci_ctlp->ahcictl_devid == 0x4391)))) {
1009 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
1010 0xffffffffull;
1011 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
1012 0xffffffffull;
1013 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
1014 0xffffffffull;
1015 }
1016
1017 /* Allocate the ports structure */
1018 status = ahci_alloc_ports_state(ahci_ctlp);
1019 if (status != AHCI_SUCCESS) {
1020 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
1021 instance);
1022 goto err_out;
1023 }
1024
1025 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
1026
1027 /*
1028 * Initialize the controller and ports.
1029 */
1030 status = ahci_initialize_controller(ahci_ctlp);
1031 if (status != AHCI_SUCCESS) {
1032 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
1033 instance);
1034 goto err_out;
1035 }
1036
1037 attach_state |= AHCI_ATTACH_STATE_HW_INIT;
1038
1039 /* Start one thread to check packet timeouts */
1040 ahci_ctlp->ahcictl_timeout_id = timeout(
1041 (void (*)(void *))ahci_watchdog_handler,
1042 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
1043
1044 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
1045
1046 if (!ahci_em_init(ahci_ctlp)) {
1047 cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure "
1048 "services", instance);
1049 goto err_out;
1050 }
1051 attach_state |= AHCI_ATTACH_STATE_ENCLOSURE;
1052
1053 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
1054 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
1055 instance);
1056 goto err_out;
1057 }
1058
1059 /* Check all handles at the end of the attach operation. */
1060 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
1061 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
1062 instance);
1063 goto err_out;
1064 }
1065
1066 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
1067
1068 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
1069
1070 return (DDI_SUCCESS);
1071
1072 err_out:
1073 /* FMA message */
1074 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
1075 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
1076
1077 if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) {
1078 ahci_em_fini(ahci_ctlp);
1079 }
1080
1081 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
1082 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1083 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1084 ahci_ctlp->ahcictl_timeout_id = 0;
1085 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1086 }
1087
1088 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1089 ahci_uninitialize_controller(ahci_ctlp);
1090 }
1091
1092 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1093 ahci_dealloc_ports_state(ahci_ctlp);
1094 }
1095
1096 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1097 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1098 }
1099
1100 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1101 ahci_rem_intrs(ahci_ctlp);
1102 }
1103
1104 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1105 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1106 }
1107
1108 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1109 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1110 }
1111
1112 if (attach_state & AHCI_ATTACH_STATE_FMA) {
1113 ahci_fm_fini(ahci_ctlp);
1114 }
1115
1116 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1117 ddi_soft_state_free(ahci_statep, instance);
1118 }
1119
1120 return (DDI_FAILURE);
1121 }
1122
1123 /*
1124 * The detach entry point for dev_ops.
1125 */
1126 static int
1127 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1128 {
1129 ahci_ctl_t *ahci_ctlp;
1130 int instance;
1131 int ret;
1132
1133 instance = ddi_get_instance(dip);
1134 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1135
1136 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1137
1138 switch (cmd) {
1139 case DDI_DETACH:
1140
1141 /* disable the interrupts for an uninterrupted detach */
1142 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1143 ahci_disable_all_intrs(ahci_ctlp);
1144 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1145
1146 /* unregister from the sata framework. */
1147 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1148 if (ret != AHCI_SUCCESS) {
1149 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1150 ahci_enable_all_intrs(ahci_ctlp);
1151 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1152 return (DDI_FAILURE);
1153 }
1154
1155 ahci_em_fini(ahci_ctlp);
1156
1157 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1158
1159 /* stop the watchdog handler */
1160 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1161 ahci_ctlp->ahcictl_timeout_id = 0;
1162
1163 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1164
1165 /* uninitialize the controller */
1166 ahci_uninitialize_controller(ahci_ctlp);
1167
1168 /* remove the interrupts */
1169 ahci_rem_intrs(ahci_ctlp);
1170
1171 /* deallocate the ports structures */
1172 ahci_dealloc_ports_state(ahci_ctlp);
1173
1174 /* destroy mutex */
1175 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1176
1177 /* teardown the pci config */
1178 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1179
1180 /* remove the reg maps. */
1181 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1182
1183 /* release fma resource */
1184 ahci_fm_fini(ahci_ctlp);
1185
1186 /* free the soft state. */
1187 ddi_soft_state_free(ahci_statep, instance);
1188
1189 return (DDI_SUCCESS);
1190
1191 case DDI_SUSPEND:
1192
1193 /*
1194 * The steps associated with suspension must include putting
1195 * the underlying device into a quiescent state so that it
1196 * will not generate interrupts or modify or access memory.
1197 */
1198 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1199 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1200 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1201 return (DDI_SUCCESS);
1202 }
1203
1204 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1205
1206 ahci_em_suspend(ahci_ctlp);
1207
1208 /* stop the watchdog handler */
1209 if (ahci_ctlp->ahcictl_timeout_id) {
1210 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1211 ahci_ctlp->ahcictl_timeout_id = 0;
1212 }
1213
1214 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1215
1216 /*
1217 * drain the taskq
1218 */
1219 ahci_drain_ports_taskq(ahci_ctlp);
1220
1221 /*
1222 * Disable the interrupts and stop all the ports.
1223 */
1224 ahci_uninitialize_controller(ahci_ctlp);
1225
1226 return (DDI_SUCCESS);
1227
1228 default:
1229 return (DDI_FAILURE);
1230 }
1231 }
1232
1233 /*
1234 * The info entry point for dev_ops.
1235 *
1236 */
1237 static int
1238 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1239 void *arg, void **result)
1240 {
1241 #ifndef __lock_lint
1242 _NOTE(ARGUNUSED(dip))
1243 #endif /* __lock_lint */
1244
1245 ahci_ctl_t *ahci_ctlp;
1246 int instance;
1247 dev_t dev;
1248
1249 dev = (dev_t)arg;
1250 instance = getminor(dev);
1251
1252 switch (infocmd) {
1253 case DDI_INFO_DEVT2DEVINFO:
1254 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1255 if (ahci_ctlp != NULL) {
1256 *result = ahci_ctlp->ahcictl_dip;
1257 return (DDI_SUCCESS);
1258 } else {
1259 *result = NULL;
1260 return (DDI_FAILURE);
1261 }
1262 case DDI_INFO_DEVT2INSTANCE:
1263 *(int *)result = instance;
1264 break;
1265 default:
1266 break;
1267 }
1268
1269 return (DDI_SUCCESS);
1270 }
1271
1272 /*
1273 * Registers the ahci with sata framework.
1274 */
1275 static int
1276 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1277 {
1278 struct sata_hba_tran *sata_hba_tran;
1279
1280 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1281 "ahci_register_sata_hba_tran enter", NULL);
1282
1283 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1284
1285 /* Allocate memory for the sata_hba_tran */
1286 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1287
1288 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1289 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1290 sata_hba_tran->sata_tran_hba_dma_attr =
1291 &ahci_ctlp->ahcictl_buffer_dma_attr;
1292
1293 /* Report the number of implemented ports */
1294 sata_hba_tran->sata_tran_hba_num_cports =
1295 ahci_ctlp->ahcictl_num_implemented_ports;
1296
1297 /* Support ATAPI device */
1298 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1299
1300 /* Get the data transfer capability for PIO command by the HBA */
1301 if (cap_status & AHCI_HBA_CAP_PMD) {
1302 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1303 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1304 "DRQ block data transfer for PIO command protocol", NULL);
1305 }
1306
1307 /*
1308 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1309 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1310 * QUEUED (EXT), and SERVICE commands). Queued operations are
1311 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1312 * QUEUED commands when the HBA and device support native command
1313 * queuing(NCQ).
1314 *
1315 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1316 * CAP register of the HBA indicates NCQ is supported.
1317 *
1318 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1319 * set because the previous register content of PxCI can be re-written
1320 * in the register write.
1321 */
1322 if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1323 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1324 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1325 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1326 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1327 "Command Queuing", NULL);
1328 }
1329
1330 /* Support port multiplier? */
1331 if (cap_status & AHCI_HBA_CAP_SPM) {
1332 sata_hba_tran->sata_tran_hba_features_support |=
1333 SATA_CTLF_PORT_MULTIPLIER;
1334
1335 /* Support FIS-based switching for port multiplier? */
1336 if (cap_status & AHCI_HBA_CAP_FBSS) {
1337 sata_hba_tran->sata_tran_hba_features_support |=
1338 SATA_CTLF_PMULT_FBS;
1339 }
1340 }
1341
1342 /* Report the number of command slots */
1343 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1344
1345 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1346 sata_hba_tran->sata_tran_start = ahci_tran_start;
1347 sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1348 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1349 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1350 #ifdef __lock_lint
1351 sata_hba_tran->sata_tran_selftest = ahci_selftest;
1352 #endif
1353 /*
1354 * When SATA framework adds support for pwrmgt the
1355 * pwrmgt_ops needs to be updated
1356 */
1357 sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1358 sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl;
1359
1360 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1361
1362 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1363
1364 /* Attach it to SATA framework */
1365 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1366 != DDI_SUCCESS) {
1367 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1368 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1369 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1370 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1371 return (AHCI_FAILURE);
1372 }
1373
1374 return (AHCI_SUCCESS);
1375 }
1376
1377 /*
1378 * Unregisters the ahci with sata framework.
1379 */
1380 static int
1381 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1382 {
1383 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1384 "ahci_unregister_sata_hba_tran enter", NULL);
1385
1386 /* Detach from the SATA framework. */
1387 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1388 DDI_SUCCESS) {
1389 return (AHCI_FAILURE);
1390 }
1391
1392 /* Deallocate sata_hba_tran. */
1393 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1394 sizeof (sata_hba_tran_t));
1395
1396 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1397 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1398 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1399
1400 return (AHCI_SUCCESS);
1401 }
1402
1403 #define SET_PORTSTR(str, addrp) \
1404 if (AHCI_ADDR_IS_PORT(addrp)) \
1405 (void) sprintf((str), "%d", (addrp)->aa_port); \
1406 else if (AHCI_ADDR_IS_PMULT(addrp)) \
1407 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \
1408 else \
1409 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \
1410 (addrp)->aa_pmport);
1411
1412 /*
1413 * ahci_tran_probe_port is called by SATA framework. It returns port state,
1414 * port status registers and an attached device type via sata_device
1415 * structure.
1416 *
1417 * We return the cached information from a previous hardware probe. The
1418 * actual hardware probing itself was done either from within
1419 * ahci_initialize_controller() during the driver attach or from a phy
1420 * ready change interrupt handler.
1421 */
1422 static int
1423 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1424 {
1425 ahci_ctl_t *ahci_ctlp;
1426 ahci_port_t *ahci_portp;
1427 ahci_addr_t addr, pmult_addr;
1428 uint8_t cport = sd->satadev_addr.cport;
1429 char portstr[10];
1430 uint8_t device_type;
1431 uint32_t port_state;
1432 uint8_t port;
1433 int rval = SATA_SUCCESS, rval_init;
1434
1435 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1436 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1437
1438 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1439
1440 mutex_enter(&ahci_portp->ahciport_mutex);
1441
1442 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1443 ASSERT(AHCI_ADDR_IS_VALID(&addr));
1444 SET_PORTSTR(portstr, &addr);
1445
1446 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1447 "ahci_tran_probe_port enter: port %s", portstr);
1448
1449 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1450 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1451 ahci_portp->ahciport_pmult_info == NULL)) {
1452 /* port mutliplier is removed. */
1453 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1454 "ahci_tran_probe_port: "
1455 "pmult is removed from port %s", portstr);
1456 mutex_exit(&ahci_portp->ahciport_mutex);
1457 return (SATA_FAILURE);
1458 }
1459
1460 /*
1461 * The sata_device may refer to
1462 * 1. A controller port.
1463 * A controller port should be ready here.
1464 * 2. A port multiplier.
1465 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1466 * it and register the port multiplier to the framework.
1467 * SATA_ADDR_PMULT - check the status of all its device ports.
1468 * 3. A port multiplier port.
1469 * If it has not been initialized, initialized it.
1470 *
1471 * A port multiplier or a port multiplier port may require some
1472 * initialization because we cannot do these time-consuming jobs in an
1473 * interrupt context.
1474 */
1475 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1476 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1477 /* Initialize registers on a port multiplier */
1478 rval_init = ahci_initialize_pmult(ahci_ctlp,
1479 ahci_portp, &pmult_addr, sd);
1480 if (rval_init != AHCI_SUCCESS) {
1481 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1482 "ahci_tran_probe_port: "
1483 "pmult initialization failed.", NULL);
1484 mutex_exit(&ahci_portp->ahciport_mutex);
1485 return (SATA_FAILURE);
1486 }
1487 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1488 /* Check pmports hotplug events */
1489 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1490 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1491 SATA_ADDR_DPMPORT)) {
1492 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1493 &addr, sd) != AHCI_SUCCESS) {
1494 rval = SATA_FAILURE;
1495 goto out;
1496 }
1497 }
1498
1499 /* Update port state and device type */
1500 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1501
1502 switch (port_state) {
1503
1504 case SATA_PSTATE_FAILED:
1505 sd->satadev_state = SATA_PSTATE_FAILED;
1506 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1507 "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1508 goto out;
1509
1510 case SATA_PSTATE_SHUTDOWN:
1511 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1512 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1513 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1514 goto out;
1515
1516 case SATA_PSTATE_PWROFF:
1517 sd->satadev_state = SATA_PSTATE_PWROFF;
1518 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1519 "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1520 goto out;
1521
1522 case SATA_PSTATE_PWRON:
1523 sd->satadev_state = SATA_PSTATE_PWRON;
1524 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1525 "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1526 break;
1527
1528 default:
1529 sd->satadev_state = port_state;
1530 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1531 "ahci_tran_probe_port: port %s PORT NORMAL %x",
1532 portstr, port_state);
1533 break;
1534 }
1535
1536 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1537
1538 switch (device_type) {
1539
1540 case SATA_DTYPE_ATADISK:
1541 sd->satadev_type = SATA_DTYPE_ATADISK;
1542 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1543 "ahci_tran_probe_port: port %s DISK found", portstr);
1544 break;
1545
1546 case SATA_DTYPE_ATAPI:
1547 /*
1548 * HBA driver only knows it's an ATAPI device, and don't know
1549 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1550 * type need to be determined by checking IDENTIFY PACKET
1551 * DEVICE data
1552 */
1553 sd->satadev_type = SATA_DTYPE_ATAPI;
1554 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1555 "ahci_tran_probe_port: port %s ATAPI found", portstr);
1556 break;
1557
1558 case SATA_DTYPE_PMULT:
1559 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1560 sd->satadev_type = SATA_DTYPE_PMULT;
1561
1562 /* Update the number of pmports. */
1563 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1564 sd->satadev_add_info = ahci_portp->
1565 ahciport_pmult_info->ahcipmi_num_dev_ports;
1566
1567 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1568 "ahci_tran_probe_port: port %s Port Multiplier found",
1569 portstr);
1570 break;
1571
1572 case SATA_DTYPE_UNKNOWN:
1573 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1574 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1575 "ahci_tran_probe_port: port %s Unknown device found",
1576 portstr);
1577 break;
1578
1579 default:
1580 /* we don't support any other device types */
1581 sd->satadev_type = SATA_DTYPE_NONE;
1582 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1583 "ahci_tran_probe_port: port %s No device found", portstr);
1584 break;
1585 }
1586
1587 out:
1588 /* Register update only fails while probing a pmult/pmport */
1589 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1590 ahci_update_sata_registers(ahci_ctlp, port, sd);
1591 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1592 if (port_state & SATA_STATE_READY)
1593 if (ahci_update_pmult_pscr(ahci_ctlp,
1594 &addr, sd) != AHCI_SUCCESS)
1595 rval = SATA_FAILURE;
1596 }
1597
1598 /* Check handles for the sata registers access */
1599 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1600 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1601 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1602 DDI_SERVICE_UNAFFECTED);
1603 rval = SATA_FAILURE;
1604 }
1605
1606 mutex_exit(&ahci_portp->ahciport_mutex);
1607 return (rval);
1608 }
1609
1610 /*
1611 * There are four operation modes in sata framework:
1612 * SATA_OPMODE_INTERRUPTS
1613 * SATA_OPMODE_POLLING
1614 * SATA_OPMODE_ASYNCH
1615 * SATA_OPMODE_SYNCH
1616 *
1617 * Their combined meanings as following:
1618 *
1619 * SATA_OPMODE_SYNCH
1620 * The command has to be completed before sata_tran_start functions returns.
1621 * Either interrupts or polling could be used - it's up to the driver.
1622 * Mode used currently for internal, sata-module initiated operations.
1623 *
1624 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1625 * It is the same as the one above.
1626 *
1627 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1628 * The command has to be completed before sata_tran_start function returns.
1629 * No interrupt used, polling only. This should be the mode used for scsi
1630 * packets with FLAG_NOINTR.
1631 *
1632 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1633 * The command may be queued (callback function specified). Interrupts could
1634 * be used. It's normal operation mode.
1635 */
1636 /*
1637 * Called by sata framework to transport a sata packet down stream.
1638 */
1639 static int
1640 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1641 {
1642 ahci_ctl_t *ahci_ctlp;
1643 ahci_port_t *ahci_portp;
1644 ahci_addr_t addr;
1645 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1646 uint8_t port;
1647 char portstr[10];
1648
1649 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1650 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1651
1652 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1653 "ahci_tran_start enter: cport %d satapkt 0x%p",
1654 cport, (void *)spkt);
1655
1656 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1657
1658 mutex_enter(&ahci_portp->ahciport_mutex);
1659 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1660 SET_PORTSTR(portstr, &addr);
1661
1662 /* Sanity check */
1663 if (AHCI_ADDR_IS_PMPORT(&addr)) {
1664 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1665 ahci_portp->ahciport_pmult_info == NULL) {
1666
1667 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1668 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1669 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1670 ahci_update_sata_registers(ahci_ctlp, port,
1671 &spkt->satapkt_device);
1672 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1673 "ahci_tran_start returning PORT_ERROR while "
1674 "pmult removed: port: %s", portstr);
1675 mutex_exit(&ahci_portp->ahciport_mutex);
1676 return (SATA_TRAN_PORT_ERROR);
1677 }
1678
1679 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1680 SATA_STATE_READY)) {
1681 if (!ddi_in_panic() ||
1682 ahci_initialize_pmport(ahci_ctlp,
1683 ahci_portp, &addr) != AHCI_SUCCESS) {
1684 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1685 spkt->satapkt_device.satadev_type =
1686 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1687 spkt->satapkt_device.satadev_state =
1688 AHCIPORT_GET_STATE(ahci_portp, &addr);
1689 ahci_update_sata_registers(ahci_ctlp, port,
1690 &spkt->satapkt_device);
1691 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1692 "ahci_tran_start returning PORT_ERROR "
1693 "while sub-link is not initialized "
1694 "at port: %s", portstr);
1695 mutex_exit(&ahci_portp->ahciport_mutex);
1696 return (SATA_TRAN_PORT_ERROR);
1697 }
1698 }
1699 }
1700
1701 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1702 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1703 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1704 /*
1705 * In case the target driver would send the packet before
1706 * sata framework can have the opportunity to process those
1707 * event reports.
1708 */
1709 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1710 spkt->satapkt_device.satadev_state =
1711 ahci_portp->ahciport_port_state;
1712 ahci_update_sata_registers(ahci_ctlp, port,
1713 &spkt->satapkt_device);
1714 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1715 "ahci_tran_start returning PORT_ERROR while "
1716 "port in FAILED/SHUTDOWN/PWROFF state: "
1717 "port: %s", portstr);
1718 mutex_exit(&ahci_portp->ahciport_mutex);
1719 return (SATA_TRAN_PORT_ERROR);
1720 }
1721
1722 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1723 /*
1724 * ahci_intr_phyrdy_change() may have rendered it to
1725 * SATA_DTYPE_NONE.
1726 */
1727 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1728 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1729 spkt->satapkt_device.satadev_state =
1730 ahci_portp->ahciport_port_state;
1731 ahci_update_sata_registers(ahci_ctlp, port,
1732 &spkt->satapkt_device);
1733 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1734 "ahci_tran_start returning PORT_ERROR while "
1735 "no device attached: port: %s", portstr);
1736 mutex_exit(&ahci_portp->ahciport_mutex);
1737 return (SATA_TRAN_PORT_ERROR);
1738 }
1739
1740 /* R/W PMULT command will occupy the whole HBA port */
1741 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1742 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1743 "ahci_tran_start returning BUSY while "
1744 "executing READ/WRITE PORT-MULT command: "
1745 "port: %s", portstr);
1746 spkt->satapkt_reason = SATA_PKT_BUSY;
1747 mutex_exit(&ahci_portp->ahciport_mutex);
1748 return (SATA_TRAN_BUSY);
1749 }
1750
1751 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1752 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1753 "ahci_tran_start returning BUSY while "
1754 "hot-plug in progress: port: %s", portstr);
1755 spkt->satapkt_reason = SATA_PKT_BUSY;
1756 mutex_exit(&ahci_portp->ahciport_mutex);
1757 return (SATA_TRAN_BUSY);
1758 }
1759
1760 /*
1761 * SATA HBA driver should remember that a device was reset and it
1762 * is supposed to reject any packets which do not specify either
1763 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1764 *
1765 * This is to prevent a race condition when a device was arbitrarily
1766 * reset by the HBA driver (and lost it's setting) and a target
1767 * driver sending some commands to a device before the sata framework
1768 * has a chance to restore the device setting (such as cache enable/
1769 * disable or other resettable stuff).
1770 */
1771 /*
1772 * It is unnecessary to use specific flags to indicate
1773 * reset_in_progress for a pmport. While mopping, all command will be
1774 * mopped so that the entire HBA port is being dealt as a single
1775 * object.
1776 */
1777 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1778 ahci_portp->ahciport_reset_in_progress = 0;
1779 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1780 "ahci_tran_start [CLEAR] the "
1781 "reset_in_progress for port: %d", port);
1782 }
1783
1784 if (ahci_portp->ahciport_reset_in_progress &&
1785 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1786 ! ddi_in_panic()) {
1787 spkt->satapkt_reason = SATA_PKT_BUSY;
1788 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1789 "ahci_tran_start returning BUSY while "
1790 "reset in progress: port: %d", port);
1791 mutex_exit(&ahci_portp->ahciport_mutex);
1792 return (SATA_TRAN_BUSY);
1793 }
1794
1795 #ifdef AHCI_DEBUG
1796 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1797 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1798 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1799 spkt, port);
1800 }
1801 #endif
1802
1803 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1804 spkt->satapkt_reason = SATA_PKT_BUSY;
1805 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1806 "ahci_tran_start returning BUSY while "
1807 "mopping in progress: port: %d", port);
1808 mutex_exit(&ahci_portp->ahciport_mutex);
1809 return (SATA_TRAN_BUSY);
1810 }
1811
1812 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1813 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1814 DDI_SERVICE_UNAFFECTED);
1815 mutex_exit(&ahci_portp->ahciport_mutex);
1816 return (SATA_TRAN_BUSY);
1817 }
1818
1819 if (spkt->satapkt_op_mode &
1820 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1821 /*
1822 * If a SYNC command to be executed in interrupt context,
1823 * bounce it back to sata module.
1824 */
1825 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1826 servicing_interrupt()) {
1827 spkt->satapkt_reason = SATA_PKT_BUSY;
1828 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1829 "ahci_tran_start returning BUSY while "
1830 "sending SYNC mode under interrupt context: "
1831 "port : %d", port);
1832 mutex_exit(&ahci_portp->ahciport_mutex);
1833 return (SATA_TRAN_BUSY);
1834 }
1835
1836 /* We need to do the sync start now */
1837 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1838 spkt) == AHCI_FAILURE) {
1839 goto fail_out;
1840 }
1841 } else {
1842 /* Async start, using interrupt */
1843 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1844 == AHCI_FAILURE) {
1845 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1846 goto fail_out;
1847 }
1848 }
1849
1850 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1851 "sata tran accepted: port %s", portstr);
1852
1853 mutex_exit(&ahci_portp->ahciport_mutex);
1854 return (SATA_TRAN_ACCEPTED);
1855
1856 fail_out:
1857 /*
1858 * Failed to deliver packet to the controller.
1859 * Check if it's caused by invalid handles.
1860 */
1861 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1862 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1863 spkt->satapkt_device.satadev_type =
1864 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1865 spkt->satapkt_device.satadev_state =
1866 AHCIPORT_GET_STATE(ahci_portp, &addr);
1867 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1868 mutex_exit(&ahci_portp->ahciport_mutex);
1869 return (SATA_TRAN_PORT_ERROR);
1870 }
1871
1872 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1873 "return QUEUE_FULL: port %d", port);
1874 mutex_exit(&ahci_portp->ahciport_mutex);
1875 return (SATA_TRAN_QUEUE_FULL);
1876 }
1877
1878 /*
1879 * SATA_OPMODE_SYNCH flag is set
1880 *
1881 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1882 * without interrupt, otherwise we can still use the interrupt.
1883 */
1884 static int
1885 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1886 ahci_addr_t *addrp, sata_pkt_t *spkt)
1887 {
1888 int pkt_timeout_ticks;
1889 uint32_t timeout_tags;
1890 int rval;
1891 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1892 uint8_t port = addrp->aa_port;
1893
1894 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1895
1896 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1897 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1898
1899 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1900 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1901 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1902 addrp, spkt)) == AHCI_FAILURE) {
1903 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1904 return (rval);
1905 }
1906
1907 pkt_timeout_ticks =
1908 drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1909
1910 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1911 /* Simulate the interrupt */
1912 mutex_exit(&ahci_portp->ahciport_mutex);
1913 ahci_port_intr(ahci_ctlp, ahci_portp, port);
1914 mutex_enter(&ahci_portp->ahciport_mutex);
1915
1916 if (spkt->satapkt_reason != SATA_PKT_BUSY)
1917 break;
1918
1919 mutex_exit(&ahci_portp->ahciport_mutex);
1920 drv_usecwait(AHCI_1MS_USECS);
1921 mutex_enter(&ahci_portp->ahciport_mutex);
1922
1923 pkt_timeout_ticks -= AHCI_1MS_TICKS;
1924 if (pkt_timeout_ticks < 0) {
1925 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1926 "port %d satapkt 0x%p timed out\n",
1927 instance, port, (void *)spkt);
1928 timeout_tags = (0x1 << rval);
1929 mutex_exit(&ahci_portp->ahciport_mutex);
1930 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1931 port, timeout_tags);
1932 mutex_enter(&ahci_portp->ahciport_mutex);
1933 }
1934 }
1935
1936 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1937 return (AHCI_SUCCESS);
1938
1939 } else {
1940 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1941 addrp, spkt)) == AHCI_FAILURE)
1942 return (rval);
1943
1944 #if AHCI_DEBUG
1945 /*
1946 * Note that the driver always uses the slot 0 to deliver
1947 * REQUEST SENSE or READ LOG EXT command
1948 */
1949 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1950 ASSERT(rval == 0);
1951 #endif
1952
1953 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1954 cv_wait(&ahci_portp->ahciport_cv,
1955 &ahci_portp->ahciport_mutex);
1956
1957 return (AHCI_SUCCESS);
1958 }
1959 }
1960
1961 /*
1962 * Searches for and claims a free command slot.
1963 *
1964 * Returns value:
1965 *
1966 * AHCI_FAILURE returned only if
1967 * 1. No empty slot left
1968 * 2. Non-queued command requested while queued command(s) is outstanding
1969 * 3. Queued command requested while non-queued command(s) is outstanding
1970 * 4. HBA doesn't support multiple-use of command list while already a
1971 * non-queued command is oustanding
1972 * 5. Queued command requested while some queued command(s) has been
1973 * outstanding on a different port multiplier port. (AHCI spec 1.2,
1974 * 9.1.2)
1975 *
1976 * claimed slot number returned if succeeded
1977 *
1978 * NOTE: it will always return slot 0 for following commands to simplify the
1979 * algorithm.
1980 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process
1981 * 2. READ/WRITE PORTMULT command
1982 */
1983 static int
1984 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1985 ahci_addr_t *addrp, int command_type)
1986 {
1987 uint32_t port_cmd_issue;
1988 uint32_t free_slots;
1989 int slot;
1990
1991 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1992
1993 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1994 "ahciport_pending_tags = 0x%x "
1995 "ahciport_pending_ncq_tags = 0x%x",
1996 ahci_portp->ahciport_pending_tags,
1997 ahci_portp->ahciport_pending_ncq_tags);
1998
1999 /*
2000 * According to the AHCI spec, system software is responsible to
2001 * ensure that queued and non-queued commands are not mixed in
2002 * the command list.
2003 */
2004 if (command_type == AHCI_NON_NCQ_CMD) {
2005 /* Non-NCQ command request */
2006 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2007 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2008 "ahci_claim_free_slot: there is still pending "
2009 "queued command(s) in the command list, "
2010 "so no available slot for the non-queued "
2011 "command", NULL);
2012 return (AHCI_FAILURE);
2013 }
2014 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
2015 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2016 "ahci_claim_free_slot: there is still pending "
2017 "read/write port-mult command(s) in command list, "
2018 "so no available slot for the non-queued command",
2019 NULL);
2020 return (AHCI_FAILURE);
2021 }
2022 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
2023 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2024 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2025 "ahci_claim_free_slot: HBA cannot support multiple-"
2026 "use of the command list for non-queued commands",
2027 NULL);
2028 return (AHCI_FAILURE);
2029 }
2030 free_slots = (~ahci_portp->ahciport_pending_tags) &
2031 AHCI_SLOT_MASK(ahci_ctlp);
2032 } else if (command_type == AHCI_NCQ_CMD) {
2033 /* NCQ command request */
2034 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2035 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2036 "ahci_claim_free_slot: there is still pending "
2037 "non-queued command(s) in the command list, "
2038 "so no available slot for the queued command",
2039 NULL);
2040 return (AHCI_FAILURE);
2041 }
2042
2043 /*
2044 * NCQ commands cannot be sent to different port multiplier
2045 * ports in Command-Based Switching mode
2046 */
2047 /*
2048 * NOTE: In Command-Based Switching mode, AHCI controller
2049 * usually reports a 'Handshake Error' when multiple NCQ
2050 * commands are outstanding simultaneously.
2051 */
2052 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
2053 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2054 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
2055 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
2056 AHCIPORT_NCQ_PMPORT(ahci_portp) !=
2057 addrp->aa_pmport) {
2058 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2059 "ahci_claim_free_slot: there is still "
2060 "pending queued command(s) in the "
2061 "command list for another Port Multiplier "
2062 "port, so no available slot.", NULL);
2063 return (AHCI_FAILURE);
2064 }
2065 }
2066
2067 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
2068 AHCI_NCQ_SLOT_MASK(ahci_portp);
2069 } else if (command_type == AHCI_ERR_RETRI_CMD) {
2070 /* Error retrieval command request */
2071 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2072 "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
2073 "SENSE or READ LOG EXT command", NULL);
2074 slot = 0;
2075 goto out;
2076 } else if (command_type == AHCI_RDWR_PMULT_CMD) {
2077 /*
2078 * An extra check on PxCI. Sometimes PxCI bits may not be
2079 * cleared during hot-plug or error recovery process.
2080 */
2081 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2082 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
2083
2084 if (port_cmd_issue != 0) {
2085 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2086 "ahci_claim_free_slot: there is still pending "
2087 "command(s) in command list (0x%x/0x%x, PxCI %x),"
2088 "so no available slot for R/W PMULT command.",
2089 NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
2090 NCQ_CMD_IN_PROGRESS(ahci_portp),
2091 port_cmd_issue);
2092 return (AHCI_FAILURE);
2093 }
2094
2095 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2096 "ahci_claim_free_slot: slot 0 is allocated for "
2097 "READ/WRITE PORTMULT command", NULL);
2098 slot = 0;
2099 goto out;
2100 }
2101
2102 slot = ddi_ffs(free_slots) - 1;
2103 if (slot == -1) {
2104 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2105 "ahci_claim_free_slot: no empty slots", NULL);
2106 return (AHCI_FAILURE);
2107 }
2108
2109 /*
2110 * According to the AHCI spec, to allow a simple mechanism for the
2111 * HBA to map command list slots to queue entries, software must
2112 * match the tag number it uses to the slot it is placing the command
2113 * in. For example, if a queued command is placed in slot 5, the tag
2114 * for that command must be 5.
2115 */
2116 if (command_type == AHCI_NCQ_CMD) {
2117 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2118 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2119 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2120 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2121 }
2122 }
2123
2124 ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2125
2126 out:
2127 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2128 "ahci_claim_free_slot: found slot: 0x%x", slot);
2129
2130 return (slot);
2131 }
2132
2133 /*
2134 * Builds the Command Table for the sata packet and delivers it to controller.
2135 *
2136 * Returns:
2137 * slot number if we can obtain a slot successfully
2138 * otherwise, return AHCI_FAILURE
2139 */
2140 static int
2141 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2142 ahci_addr_t *addrp, sata_pkt_t *spkt)
2143 {
2144 int cmd_slot;
2145 sata_cmd_t *scmd;
2146 ahci_fis_h2d_register_t *h2d_register_fisp;
2147 ahci_cmd_table_t *cmd_table;
2148 ahci_cmd_header_t *cmd_header;
2149 int ncookies;
2150 int i;
2151 int command_type = AHCI_NON_NCQ_CMD;
2152 int ncq_qdepth;
2153 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2154 uint8_t port, pmport;
2155 #if AHCI_DEBUG
2156 uint32_t *ptr;
2157 uint8_t *ptr2;
2158 #endif
2159
2160 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2161
2162 port = addrp->aa_port;
2163 pmport = addrp->aa_pmport;
2164
2165 spkt->satapkt_reason = SATA_PKT_BUSY;
2166
2167 scmd = &spkt->satapkt_cmd;
2168
2169 /* Check if the command is a NCQ command */
2170 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2171 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2172 command_type = AHCI_NCQ_CMD;
2173
2174 /*
2175 * When NCQ is support, system software must determine the
2176 * maximum tag allowed by the device and the HBA, and it
2177 * must use a value not beyond of the lower bound of the two.
2178 *
2179 * Sata module is going to calculate the qdepth and send
2180 * down to HBA driver via sata_cmd.
2181 */
2182 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2183
2184 /*
2185 * At the moment, the driver doesn't support the dynamic
2186 * setting of the maximum ncq depth, and the value can be
2187 * set either during the attach or after hot-plug insertion.
2188 */
2189 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2190 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2191 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2192 "ahci_deliver_satapkt: port %d the max tags for "
2193 "NCQ command is %d", port, ncq_qdepth);
2194 } else {
2195 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2196 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2197 " port %d the max tag for NCQ command is "
2198 "requested to change from %d to %d, at the"
2199 " moment the driver doesn't support the "
2200 "dynamic change so it's going to "
2201 "still use the previous tag value",
2202 instance, port,
2203 ahci_portp->ahciport_max_ncq_tags,
2204 ncq_qdepth);
2205 }
2206 }
2207 }
2208
2209 /* Check if the command is an error retrieval command */
2210 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2211 command_type = AHCI_ERR_RETRI_CMD;
2212
2213 /* Check if the command is an read/write pmult command */
2214 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2215 command_type = AHCI_RDWR_PMULT_CMD;
2216
2217 /* Check if there is an empty command slot */
2218 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2219 addrp, command_type);
2220 if (cmd_slot == AHCI_FAILURE) {
2221 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2222 return (AHCI_FAILURE);
2223 }
2224
2225 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2226 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2227 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2228 cmd_slot, port, (void *)spkt);
2229
2230 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2231 bzero((void *)cmd_table, ahci_cmd_table_size);
2232
2233 /* For data transfer operations, it is the H2D Register FIS */
2234 h2d_register_fisp =
2235 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2236
2237 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2238
2239 /*
2240 * PMP field only make sense when target is a port multiplier or a
2241 * device behind a port multiplier. Otherwise should set it to 0.
2242 */
2243 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2244 SET_FIS_PMP(h2d_register_fisp, pmport);
2245
2246 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2247 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2248 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2249 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2250
2251 switch (scmd->satacmd_addr_type) {
2252
2253 case 0:
2254 /*
2255 * satacmd_addr_type will be 0 for the commands below:
2256 * ATAPI command
2257 * SATAC_IDLE_IM
2258 * SATAC_STANDBY_IM
2259 * SATAC_DOWNLOAD_MICROCODE
2260 * SATAC_FLUSH_CACHE
2261 * SATAC_SET_FEATURES
2262 * SATAC_SMART
2263 * SATAC_ID_PACKET_DEVICE
2264 * SATAC_ID_DEVICE
2265 * SATAC_READ_PORTMULT
2266 * SATAC_WRITE_PORTMULT
2267 */
2268 /* FALLTHRU */
2269
2270 case ATA_ADDR_LBA:
2271 /* FALLTHRU */
2272
2273 case ATA_ADDR_LBA28:
2274 /* LBA[7:0] */
2275 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2276
2277 /* LBA[15:8] */
2278 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2279
2280 /* LBA[23:16] */
2281 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2282
2283 /* LBA [27:24] (also called dev_head) */
2284 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2285
2286 break;
2287
2288 case ATA_ADDR_LBA48:
2289 /* LBA[7:0] */
2290 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2291
2292 /* LBA[15:8] */
2293 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2294
2295 /* LBA[23:16] */
2296 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2297
2298 /* LBA [31:24] */
2299 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2300 scmd->satacmd_lba_low_msb);
2301
2302 /* LBA [39:32] */
2303 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2304 scmd->satacmd_lba_mid_msb);
2305
2306 /* LBA [47:40] */
2307 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2308 scmd->satacmd_lba_high_msb);
2309
2310 /* Set dev_head */
2311 SET_FIS_DEV_HEAD(h2d_register_fisp,
2312 scmd->satacmd_device_reg);
2313
2314 /* Set the extended sector count and features */
2315 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2316 scmd->satacmd_sec_count_msb);
2317 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2318 scmd->satacmd_features_reg_ext);
2319 break;
2320 }
2321
2322 /*
2323 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2324 * filled into features field, and sector count 8:15 is filled into
2325 * features (exp) field. The hba driver doesn't need to anything
2326 * special with regard to this, since sata framework has already
2327 * done so.
2328 *
2329 * However the driver needs to make sure TAG is filled into sector
2330 * field.
2331 */
2332 if (command_type == AHCI_NCQ_CMD) {
2333 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2334 (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2335 }
2336
2337 ncookies = scmd->satacmd_num_dma_cookies;
2338 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2339 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2340 ncookies, ahci_dma_prdt_number);
2341
2342 ASSERT(ncookies <= ahci_dma_prdt_number);
2343 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2344
2345 /* *** now fill the scatter gather list ******* */
2346 for (i = 0; i < ncookies; i++) {
2347 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2348 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2349 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2350 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2351 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2352 scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2353 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2354 scmd->satacmd_dma_cookie_list[i].dmac_size;
2355 }
2356
2357 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2358 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2359 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2360
2361 /* The ACMD field is filled in for ATAPI command */
2362 if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2363 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2364 SATA_ATAPI_MAX_CDB_LEN);
2365 }
2366
2367 /* Set Command Header in Command List */
2368 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2369 BZERO_DESCR_INFO(cmd_header);
2370 BZERO_PRD_BYTE_COUNT(cmd_header);
2371
2372 /* Set the number of entries in the PRD table */
2373 SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2374
2375 /* Set the length of the command in the CFIS area */
2376 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2377
2378 /*
2379 * PMP field only make sense when target is a port multiplier or a
2380 * device behind a port multiplier. Otherwise should set it to 0.
2381 */
2382 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2383 SET_PORT_MULTI_PORT(cmd_header, pmport);
2384
2385 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2386 "sata_data_direction = 0x%x",
2387 scmd->satacmd_flags.sata_data_direction);
2388
2389 /* Set A bit if it is an ATAPI command */
2390 if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2391 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2392
2393 /* Set W bit if data is going to the device */
2394 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2395 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2396
2397 /*
2398 * Set the prefetchable bit - this bit is only valid if the PRDTL
2399 * field is non-zero or the ATAPI 'A' bit is set in the command
2400 * header. This bit cannot be set when using native command
2401 * queuing commands or when using FIS-based switching with a Port
2402 * multiplier.
2403 */
2404 if (command_type != AHCI_NCQ_CMD)
2405 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2406
2407 /*
2408 * Now remember the sata packet in ahciport_slot_pkts[].
2409 * Error retrieval command and r/w port multiplier command will
2410 * be stored specifically for each port.
2411 */
2412 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2413 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2414 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2415
2416 /*
2417 * Keep the timeout value
2418 */
2419 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2420
2421 /*
2422 * If the intial timout is less than 1 tick, then make it longer by
2423 * 1 tick to avoid immediate timeout
2424 */
2425 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2426 ahci_watchdog_timeout)
2427 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2428 ahci_watchdog_timeout;
2429
2430 #if AHCI_DEBUG
2431 if (ahci_debug_flags & AHCIDBG_ATACMD &&
2432 scmd->satacmd_cmd_reg != SATAC_PACKET ||
2433 ahci_debug_flags & AHCIDBG_ATAPICMD &&
2434 scmd->satacmd_cmd_reg == SATAC_PACKET) {
2435
2436 /* Dump the command header and table */
2437 ahci_log(ahci_ctlp, CE_WARN, "\n");
2438 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2439 "0x%p cmd_reg 0x%x port %d", spkt,
2440 scmd->satacmd_cmd_reg, port);
2441 ptr = (uint32_t *)cmd_header;
2442 ahci_log(ahci_ctlp, CE_WARN,
2443 " Command Header:%8x %8x %8x %8x",
2444 ptr[0], ptr[1], ptr[2], ptr[3]);
2445
2446 /* Dump the H2D register FIS */
2447 ptr = (uint32_t *)h2d_register_fisp;
2448 ahci_log(ahci_ctlp, CE_WARN,
2449 " Command FIS: %8x %8x %8x %8x",
2450 ptr[0], ptr[1], ptr[2], ptr[3]);
2451
2452 /* Dump the ACMD register FIS */
2453 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2454 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2455 if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2456 ahci_log(ahci_ctlp, CE_WARN,
2457 " ATAPI command: %2x %2x %2x %2x "
2458 "%2x %2x %2x %2x",
2459 ptr2[8 * i], ptr2[8 * i + 1],
2460 ptr2[8 * i + 2], ptr2[8 * i + 3],
2461 ptr2[8 * i + 4], ptr2[8 * i + 5],
2462 ptr2[8 * i + 6], ptr2[8 * i + 7]);
2463
2464 /* Dump the PRDT */
2465 for (i = 0; i < ncookies; i++) {
2466 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2467 ahci_log(ahci_ctlp, CE_WARN,
2468 " Cookie %d: %8x %8x %8x %8x",
2469 i, ptr[0], ptr[1], ptr[2], ptr[3]);
2470 }
2471 }
2472 #endif
2473
2474 (void) ddi_dma_sync(
2475 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2476 0,
2477 ahci_cmd_table_size,
2478 DDI_DMA_SYNC_FORDEV);
2479
2480 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2481 cmd_slot * sizeof (ahci_cmd_header_t),
2482 sizeof (ahci_cmd_header_t),
2483 DDI_DMA_SYNC_FORDEV);
2484
2485 if ((ahci_check_dma_handle(ahci_portp->
2486 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2487 ahci_check_dma_handle(ahci_portp->
2488 ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2489 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2490 DDI_SERVICE_UNAFFECTED);
2491 return (AHCI_FAILURE);
2492 }
2493
2494 /* Set the corresponding bit in the PxSACT.DS for queued command */
2495 if (command_type == AHCI_NCQ_CMD) {
2496 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2497 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2498 (0x1 << cmd_slot));
2499 }
2500
2501 /* Indicate to the HBA that a command is active. */
2502 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2503 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2504 (0x1 << cmd_slot));
2505
2506 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2507 "exit: port %d", port);
2508
2509 /* Make sure the command is started by the PxSACT/PxCI */
2510 if (ahci_check_acc_handle(ahci_ctlp->
2511 ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2512 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2513 DDI_SERVICE_UNAFFECTED);
2514 return (AHCI_FAILURE);
2515 }
2516
2517 return (cmd_slot);
2518 }
2519
2520 /*
2521 * Called by the sata framework to abort the previously sent packet(s).
2522 *
2523 * Reset device to abort commands.
2524 */
2525 static int
2526 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2527 {
2528 ahci_ctl_t *ahci_ctlp;
2529 ahci_port_t *ahci_portp;
2530 uint32_t slot_status = 0;
2531 uint32_t aborted_tags = 0;
2532 uint32_t finished_tags = 0;
2533 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2534 uint8_t port;
2535 int tmp_slot;
2536 int instance = ddi_get_instance(dip);
2537
2538 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2539 port = ahci_ctlp->ahcictl_cport_to_port[cport];
2540
2541 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2542 "ahci_tran_abort enter: port %d", port);
2543
2544 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2545 mutex_enter(&ahci_portp->ahciport_mutex);
2546
2547 /*
2548 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2549 * commands are being mopped, therefore there is nothing else to do
2550 */
2551 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2552 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2553 "ahci_tran_abort: port %d is in "
2554 "mopping process, so just return directly ", port);
2555 mutex_exit(&ahci_portp->ahciport_mutex);
2556 return (SATA_SUCCESS);
2557 }
2558
2559 /*
2560 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2561 * command is being executed so no other commands is outstanding,
2562 * nothing to do.
2563 */
2564 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2565 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2566 "ahci_tran_abort: port %d is reading/writing "
2567 "port multiplier, so just return directly ", port);
2568 mutex_exit(&ahci_portp->ahciport_mutex);
2569 return (SATA_SUCCESS);
2570 }
2571
2572 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2573 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2574 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2575 /*
2576 * In case the targer driver would send the request before
2577 * sata framework can have the opportunity to process those
2578 * event reports.
2579 */
2580 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2581 spkt->satapkt_device.satadev_state =
2582 ahci_portp->ahciport_port_state;
2583 ahci_update_sata_registers(ahci_ctlp, port,
2584 &spkt->satapkt_device);
2585 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2586 "ahci_tran_abort returning SATA_FAILURE while "
2587 "port in FAILED/SHUTDOWN/PWROFF state: "
2588 "port: %d", port);
2589 mutex_exit(&ahci_portp->ahciport_mutex);
2590 return (SATA_FAILURE);
2591 }
2592
2593 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2594 /*
2595 * ahci_intr_phyrdy_change() may have rendered it to
2596 * AHCI_PORT_TYPE_NODEV.
2597 */
2598 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2599 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2600 spkt->satapkt_device.satadev_state =
2601 ahci_portp->ahciport_port_state;
2602 ahci_update_sata_registers(ahci_ctlp, port,
2603 &spkt->satapkt_device);
2604 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2605 "ahci_tran_abort returning SATA_FAILURE while "
2606 "no device attached: port: %d", port);
2607 mutex_exit(&ahci_portp->ahciport_mutex);
2608 return (SATA_FAILURE);
2609 }
2610
2611 if (flag == SATA_ABORT_ALL_PACKETS) {
2612 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2613 aborted_tags = ahci_portp->ahciport_pending_tags;
2614 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2615 aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2616
2617 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2618 instance, port);
2619 } else {
2620 aborted_tags = 0xffffffff;
2621 /*
2622 * Aborting one specific packet, first search the
2623 * ahciport_slot_pkts[] list for matching spkt.
2624 */
2625 for (tmp_slot = 0;
2626 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2627 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2628 aborted_tags = (0x1 << tmp_slot);
2629 break;
2630 }
2631 }
2632
2633 if (aborted_tags == 0xffffffff) {
2634 /* request packet is not on the pending list */
2635 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2636 "Cannot find the aborting pkt 0x%p on the "
2637 "pending list", (void *)spkt);
2638 ahci_update_sata_registers(ahci_ctlp, port,
2639 &spkt->satapkt_device);
2640 mutex_exit(&ahci_portp->ahciport_mutex);
2641 return (SATA_FAILURE);
2642 }
2643 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2644 instance, port, (void *)spkt);
2645 }
2646
2647 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2648 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2649 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2650 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2651 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2652 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2653
2654 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2655 ahci_portp->ahciport_mop_in_progress++;
2656
2657 /*
2658 * To abort the packet(s), first we are trying to clear PxCMD.ST
2659 * to stop the port, and if the port can be stopped
2660 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2661 * then we just send back the aborted packet(s) with ABORTED flag
2662 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2663 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2664 * perform a COMRESET.
2665 */
2666 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2667 ahci_portp, port, NULL, NULL);
2668
2669 /*
2670 * Compute which have finished and which need to be retried.
2671 *
2672 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2673 * minus the slot_status. The aborted_tags has to be deducted by
2674 * finished_tags since we can't possibly abort a tag which had finished
2675 * already.
2676 */
2677 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2678 finished_tags = ahci_portp->ahciport_pending_tags &
2679 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2680 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2681 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2682 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2683
2684 aborted_tags &= ~finished_tags;
2685
2686 ahci_mop_commands(ahci_ctlp,
2687 ahci_portp,
2688 slot_status,
2689 0, /* failed tags */
2690 0, /* timeout tags */
2691 aborted_tags,
2692 0); /* reset tags */
2693
2694 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2695 mutex_exit(&ahci_portp->ahciport_mutex);
2696
2697 return (SATA_SUCCESS);
2698 }
2699
2700 /*
2701 * Used to do device reset and reject all the pending packets on a device
2702 * during the reset operation.
2703 *
2704 * NOTE: ONLY called by ahci_tran_reset_dport
2705 */
2706 static int
2707 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2708 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2709 {
2710 uint32_t slot_status = 0;
2711 uint32_t reset_tags = 0;
2712 uint32_t finished_tags = 0;
2713 uint8_t port = addrp->aa_port;
2714 sata_device_t sdevice;
2715 int ret;
2716
2717 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2718
2719 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2720 "ahci_reset_device_reject_pkts on port: %d", port);
2721
2722 /*
2723 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2724 * commands are being mopped, therefore there is nothing else to do
2725 */
2726 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2727 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2728 "ahci_reset_device_reject_pkts: port %d is in "
2729 "mopping process, so return directly ", port);
2730 return (SATA_SUCCESS);
2731 }
2732
2733 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2734 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2735 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2736 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2737 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2738 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2739 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2740 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2741 }
2742
2743 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2744 != AHCI_SUCCESS) {
2745 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2746 "Try to do a port reset after software "
2747 "reset failed", port);
2748 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2749 if (ret != AHCI_SUCCESS) {
2750 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2751 "ahci_reset_device_reject_pkts: port %d "
2752 "failed", port);
2753 return (SATA_FAILURE);
2754 }
2755 }
2756 /* Set the reset in progress flag */
2757 ahci_portp->ahciport_reset_in_progress = 1;
2758
2759 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2760 ahci_portp->ahciport_mop_in_progress++;
2761
2762 /* Indicate to the framework that a reset has happened */
2763 bzero((void *)&sdevice, sizeof (sata_device_t));
2764 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2765 sdevice.satadev_addr.pmport = 0;
2766 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2767 sdevice.satadev_state = SATA_DSTATE_RESET |
2768 SATA_DSTATE_PWR_ACTIVE;
2769 mutex_exit(&ahci_portp->ahciport_mutex);
2770 sata_hba_event_notify(
2771 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2772 &sdevice,
2773 SATA_EVNT_DEVICE_RESET);
2774 mutex_enter(&ahci_portp->ahciport_mutex);
2775
2776 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2777 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2778
2779 /* Next try to mop the pending commands */
2780 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2781 finished_tags = ahci_portp->ahciport_pending_tags &
2782 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2783 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2784 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2785 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2786
2787 reset_tags &= ~finished_tags;
2788
2789 ahci_mop_commands(ahci_ctlp,
2790 ahci_portp,
2791 slot_status,
2792 0, /* failed tags */
2793 0, /* timeout tags */
2794 0, /* aborted tags */
2795 reset_tags); /* reset tags */
2796
2797 return (SATA_SUCCESS);
2798 }
2799
2800 /*
2801 * Used to do device reset and reject all the pending packets on a device
2802 * during the reset operation.
2803 *
2804 * NOTE: ONLY called by ahci_tran_reset_dport
2805 */
2806 static int
2807 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2808 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2809 {
2810 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2811 uint8_t port = addrp->aa_port;
2812 uint8_t pmport = addrp->aa_pmport;
2813 sata_device_t sdevice;
2814
2815 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2816
2817 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2818 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2819
2820 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2821 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2822 "ahci_reset_pmdevice_reject_pkts: port %d is in "
2823 "mopping process, so return directly ", port);
2824 return (SATA_SUCCESS);
2825 }
2826
2827 /* Checking for outstanding commands */
2828 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2829 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2830 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2831 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2832 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2833 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2834 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2835 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2836 }
2837
2838 /* Issue SOFTWARE reset command. */
2839 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2840 != AHCI_SUCCESS) {
2841 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2842 "Try to do a port reset after software "
2843 "reset failed", port);
2844 return (SATA_FAILURE);
2845 }
2846
2847 /* Set the reset in progress flag */
2848 ahci_portp->ahciport_reset_in_progress = 1;
2849
2850 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2851 ahci_portp->ahciport_mop_in_progress++;
2852
2853 /* Indicate to the framework that a reset has happened */
2854 bzero((void *)&sdevice, sizeof (sata_device_t));
2855 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2856 sdevice.satadev_addr.pmport = pmport;
2857 if (AHCI_ADDR_IS_PMULT(addrp))
2858 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2859 else
2860 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2861 sdevice.satadev_state = SATA_DSTATE_RESET |
2862 SATA_DSTATE_PWR_ACTIVE;
2863 mutex_exit(&ahci_portp->ahciport_mutex);
2864 sata_hba_event_notify(
2865 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2866 &sdevice,
2867 SATA_EVNT_DEVICE_RESET);
2868 mutex_enter(&ahci_portp->ahciport_mutex);
2869
2870 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2871 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2872 port, pmport);
2873
2874 /* Next try to mop the pending commands */
2875 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2876 finished_tags = ahci_portp->ahciport_pending_tags &
2877 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2878 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2879 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2880 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2881 reset_tags &= ~finished_tags;
2882
2883 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2884 "reset_tags = %x, finished_tags = %x, slot_status = %x",
2885 reset_tags, finished_tags, slot_status);
2886
2887 /*
2888 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2889 * try to reset a single device behind a port multiplier will
2890 * terminate all the commands on that HBA port. We need mop these
2891 * commands as well.
2892 */
2893 ahci_mop_commands(ahci_ctlp,
2894 ahci_portp,
2895 slot_status,
2896 0, /* failed tags */
2897 0, /* timeout tags */
2898 0, /* aborted tags */
2899 reset_tags); /* reset tags */
2900
2901 return (SATA_SUCCESS);
2902 }
2903
2904 /*
2905 * Used to do port reset and reject all the pending packets on a port during
2906 * the reset operation.
2907 */
2908 static int
2909 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2910 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2911 {
2912 uint32_t slot_status = 0;
2913 uint32_t reset_tags = 0;
2914 uint32_t finished_tags = 0;
2915 uint8_t port = addrp->aa_port;
2916
2917 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2918
2919 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2920 "ahci_reset_port_reject_pkts at port: %d", port);
2921
2922 /*
2923 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2924 * commands are being mopped, therefore there is nothing else to do
2925 */
2926 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2927 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2928 "ahci_reset_port_reject_pkts: port %d is in "
2929 "mopping process, so return directly ", port);
2930 return (SATA_SUCCESS);
2931 }
2932
2933 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2934 ahci_portp->ahciport_mop_in_progress++;
2935
2936 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2937 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2938 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2939 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2940 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2941 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2942 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2943 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2944 }
2945
2946 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2947 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2948 NULL) != AHCI_SUCCESS) {
2949
2950 /* Clear mop flag */
2951 ahci_portp->ahciport_mop_in_progress--;
2952 if (ahci_portp->ahciport_mop_in_progress == 0)
2953 ahci_portp->ahciport_flags &=
2954 ~AHCI_PORT_FLAG_MOPPING;
2955 return (SATA_FAILURE);
2956 }
2957
2958 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2959 finished_tags = ahci_portp->ahciport_pending_tags &
2960 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2961 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2962 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2963 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2964
2965 reset_tags &= ~finished_tags;
2966
2967 ahci_mop_commands(ahci_ctlp,
2968 ahci_portp,
2969 slot_status,
2970 0, /* failed tags */
2971 0, /* timeout tags */
2972 0, /* aborted tags */
2973 reset_tags); /* reset tags */
2974
2975 return (SATA_SUCCESS);
2976 }
2977
2978 /*
2979 * Used to do hba reset and reject all the pending packets on all ports
2980 * during the reset operation.
2981 */
2982 static int
2983 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2984 {
2985 ahci_port_t *ahci_portp;
2986 uint32_t slot_status[AHCI_MAX_PORTS];
2987 uint32_t reset_tags[AHCI_MAX_PORTS];
2988 uint32_t finished_tags[AHCI_MAX_PORTS];
2989 int port;
2990 int ret = SATA_SUCCESS;
2991
2992 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2993 "ahci_reset_hba_reject_pkts enter", NULL);
2994
2995 bzero(slot_status, sizeof (slot_status));
2996 bzero(reset_tags, sizeof (reset_tags));
2997 bzero(finished_tags, sizeof (finished_tags));
2998
2999 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3000 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3001 continue;
3002 }
3003
3004 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3005
3006 mutex_enter(&ahci_portp->ahciport_mutex);
3007 ahci_portp->ahciport_reset_in_progress = 1;
3008 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3009 slot_status[port] = ddi_get32(
3010 ahci_ctlp->ahcictl_ahci_acc_handle,
3011 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3012 reset_tags[port] = slot_status[port] &
3013 AHCI_SLOT_MASK(ahci_ctlp);
3014 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3015 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3016 port, reset_tags[port],
3017 ahci_portp->ahciport_pending_tags);
3018 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3019 slot_status[port] = ddi_get32(
3020 ahci_ctlp->ahcictl_ahci_acc_handle,
3021 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3022 reset_tags[port] = slot_status[port] &
3023 AHCI_NCQ_SLOT_MASK(ahci_portp);
3024 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3025 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3026 port, reset_tags[port],
3027 ahci_portp->ahciport_pending_tags);
3028 }
3029 mutex_exit(&ahci_portp->ahciport_mutex);
3030 }
3031
3032 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
3033 ret = SATA_FAILURE;
3034 }
3035
3036 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3037 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3038 continue;
3039 }
3040
3041 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3042
3043 mutex_enter(&ahci_portp->ahciport_mutex);
3044 /*
3045 * To prevent recursive enter to ahci_mop_commands, we need
3046 * check AHCI_PORT_FLAG_MOPPING flag.
3047 */
3048 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3049 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3050 "ahci_reset_hba_reject_pkts: port %d is in "
3051 "mopping process, so return directly ", port);
3052 mutex_exit(&ahci_portp->ahciport_mutex);
3053 continue;
3054 }
3055
3056 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3057 ahci_portp->ahciport_mop_in_progress++;
3058
3059 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
3060 finished_tags[port] =
3061 ahci_portp->ahciport_pending_tags &
3062 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
3063 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
3064 finished_tags[port] =
3065 ahci_portp->ahciport_pending_ncq_tags &
3066 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
3067
3068 reset_tags[port] &= ~finished_tags[port];
3069
3070 ahci_mop_commands(ahci_ctlp,
3071 ahci_portp,
3072 slot_status[port],
3073 0, /* failed tags */
3074 0, /* timeout tags */
3075 0, /* aborted tags */
3076 reset_tags[port]); /* reset tags */
3077 mutex_exit(&ahci_portp->ahciport_mutex);
3078 }
3079 out:
3080 return (ret);
3081 }
3082
3083 /*
3084 * Called by sata framework to reset a port(s) or device.
3085 */
3086 static int
3087 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
3088 {
3089 ahci_ctl_t *ahci_ctlp;
3090 ahci_port_t *ahci_portp;
3091 ahci_addr_t addr;
3092 uint8_t cport = sd->satadev_addr.cport;
3093 uint8_t pmport = sd->satadev_addr.pmport;
3094 uint8_t port;
3095 int ret = SATA_SUCCESS;
3096 int instance = ddi_get_instance(dip);
3097
3098 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3099 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3100 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3101
3102 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3103
3104 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3105 "ahci_tran_reset_dport enter: cport %d", cport);
3106
3107 switch (sd->satadev_addr.qual) {
3108 case SATA_ADDR_PMPORT:
3109 /*
3110 * If we want to issue a COMRESET on a pmport, we need to
3111 * reject the outstanding commands on that pmport. According
3112 * to AHCI spec, PxCI register could only be cleared by
3113 * clearing PxCMD.ST, which will halt the controller port - as
3114 * well as other pmports.
3115 *
3116 * Therefore we directly reset the controller port for
3117 * simplicity. ahci_tran_probe_port() will handle reset stuff
3118 * like initializing the given pmport.
3119 */
3120 /* FALLTHRU */
3121 case SATA_ADDR_CPORT:
3122 /* Port reset */
3123 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3124 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3125 "port %d reset port", instance, port);
3126
3127 mutex_enter(&ahci_portp->ahciport_mutex);
3128 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3129 mutex_exit(&ahci_portp->ahciport_mutex);
3130
3131 break;
3132
3133 case SATA_ADDR_DPMPORT:
3134 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3135 "port %d:%d reset device", instance, port, pmport);
3136 /* FALLTHRU */
3137 case SATA_ADDR_DCPORT:
3138 /* Device reset */
3139 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3140 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3141 "port %d reset device", instance, port);
3142
3143 mutex_enter(&ahci_portp->ahciport_mutex);
3144 /*
3145 * software reset request must be sent to SATA_PMULT_HOSTPORT
3146 * if target is a port multiplier:
3147 */
3148 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3149 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3150 AHCI_ADDR_SET_PMULT(&addr, port);
3151
3152 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3153 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3154 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3155 /*
3156 * In case the targer driver would send the request
3157 * before sata framework can have the opportunity to
3158 * process those event reports.
3159 */
3160 sd->satadev_state = ahci_portp->ahciport_port_state;
3161 ahci_update_sata_registers(ahci_ctlp, port, sd);
3162 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3163 "ahci_tran_reset_dport returning SATA_FAILURE "
3164 "while port in FAILED/SHUTDOWN/PWROFF state: "
3165 "port: %d", port);
3166 mutex_exit(&ahci_portp->ahciport_mutex);
3167 ret = SATA_FAILURE;
3168 break;
3169 }
3170
3171 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3172 SATA_DTYPE_NONE) {
3173 /*
3174 * ahci_intr_phyrdy_change() may have rendered it to
3175 * AHCI_PORT_TYPE_NODEV.
3176 */
3177 sd->satadev_type = SATA_DTYPE_NONE;
3178 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3179 &addr);
3180 ahci_update_sata_registers(ahci_ctlp, port, sd);
3181 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3182 "ahci_tran_reset_dport returning SATA_FAILURE "
3183 "while no device attached: port: %d", port);
3184 mutex_exit(&ahci_portp->ahciport_mutex);
3185 ret = SATA_FAILURE;
3186 break;
3187 }
3188
3189 if (AHCI_ADDR_IS_PORT(&addr)) {
3190 ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3191 ahci_portp, &addr);
3192 } else {
3193 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3194 ahci_portp, &addr);
3195 }
3196
3197 mutex_exit(&ahci_portp->ahciport_mutex);
3198 break;
3199
3200 case SATA_ADDR_CNTRL:
3201 /* Reset the whole controller */
3202 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3203 "reset the whole hba", instance);
3204 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3205 break;
3206
3207 default:
3208 ret = SATA_FAILURE;
3209 }
3210
3211 return (ret);
3212 }
3213
3214 /*
3215 * Called by sata framework to activate a port as part of hotplug.
3216 * (cfgadm -c connect satax/y)
3217 * Support port multiplier.
3218 */
3219 static int
3220 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3221 {
3222 ahci_ctl_t *ahci_ctlp;
3223 ahci_port_t *ahci_portp;
3224 ahci_addr_t addr;
3225 uint8_t cport = satadev->satadev_addr.cport;
3226 uint8_t pmport = satadev->satadev_addr.pmport;
3227 uint8_t port;
3228 int instance = ddi_get_instance(dip);
3229
3230 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3231 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3232
3233 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3234 "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3235
3236 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3237
3238 mutex_enter(&ahci_portp->ahciport_mutex);
3239 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3240 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3241
3242 if (AHCI_ADDR_IS_PORT(&addr)) {
3243 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3244 instance, port);
3245
3246 /* Enable the interrupts on the port */
3247 ahci_enable_port_intrs(ahci_ctlp, port);
3248
3249 /*
3250 * Reset the port so that the PHY communication would be
3251 * re-established. But this reset is an internal operation
3252 * and the sata module doesn't need to know about it.
3253 * Moreover, the port with a device attached will be started
3254 * too.
3255 */
3256 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3257 ahci_portp, port,
3258 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3259 NULL);
3260
3261 /*
3262 * Need to check the link status and device status of the port
3263 * and consider raising power if the port was in D3 state
3264 */
3265 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3266 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3267 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3268 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3269 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3270 instance, port, pmport);
3271 /* AHCI_ADDR_PMPORT */
3272 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3273 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3274 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3275 }
3276
3277 satadev->satadev_state = ahci_portp->ahciport_port_state;
3278
3279 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3280
3281 mutex_exit(&ahci_portp->ahciport_mutex);
3282 return (SATA_SUCCESS);
3283 }
3284
3285 /*
3286 * Called by sata framework to deactivate a port as part of hotplug.
3287 * (cfgadm -c disconnect satax/y)
3288 * Support port multiplier.
3289 */
3290 static int
3291 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3292 {
3293 ahci_ctl_t *ahci_ctlp;
3294 ahci_port_t *ahci_portp;
3295 ahci_addr_t addr;
3296 uint8_t cport = satadev->satadev_addr.cport;
3297 uint8_t pmport = satadev->satadev_addr.pmport;
3298 uint8_t port;
3299 uint32_t port_scontrol;
3300 int instance = ddi_get_instance(dip);
3301
3302 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3303 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3304
3305 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3306 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3307
3308 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3309 mutex_enter(&ahci_portp->ahciport_mutex);
3310 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3311 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3312
3313 if (AHCI_ADDR_IS_PORT(&addr)) {
3314 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3315 instance, port);
3316
3317 /* Disable the interrupts on the port */
3318 ahci_disable_port_intrs(ahci_ctlp, port);
3319
3320 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3321
3322 /* First to abort all the pending commands */
3323 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3324
3325 /* Then stop the port */
3326 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3327 ahci_portp, port);
3328 }
3329
3330 /* Next put the PHY offline */
3331 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3332 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3333 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3334 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3335 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3336 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3337 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3338 instance, port, pmport);
3339
3340 ahci_disable_port_intrs(ahci_ctlp, port);
3341 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3342 != SATA_DTYPE_NONE)
3343 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3344
3345 /* Re-enable the interrupts for the other pmports */
3346 ahci_enable_port_intrs(ahci_ctlp, port);
3347 }
3348
3349 /* Update port state */
3350 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3351 satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3352
3353 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3354
3355 mutex_exit(&ahci_portp->ahciport_mutex);
3356 return (SATA_SUCCESS);
3357 }
3358
3359 /*
3360 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3361 * when a device is unplugged or a port is deactivated.
3362 */
3363 static void
3364 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3365 ahci_port_t *ahci_portp, uint8_t port)
3366 {
3367 uint32_t slot_status = 0;
3368 uint32_t abort_tags = 0;
3369
3370 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3371
3372 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3373 "ahci_reject_all_abort_pkts at port: %d", port);
3374
3375 /* Read/write port multiplier command takes highest priority */
3376 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3377 slot_status = 0x1;
3378 abort_tags = 0x1;
3379 goto out;
3380 }
3381
3382 /*
3383 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3384 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3385 * to get the error data, if yes when the device is removed, the
3386 * command needs to be aborted too.
3387 */
3388 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3389 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3390 slot_status = 0x1;
3391 abort_tags = 0x1;
3392 goto out;
3393 } else {
3394 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3395 "ahci_reject_all_abort_pkts return directly "
3396 "port %d no needs to reject any outstanding "
3397 "commands", port);
3398 return;
3399 }
3400 }
3401
3402 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3403 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3404 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3405 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3406 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3407 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3408 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3409 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3410 }
3411
3412 out:
3413 /* No need to do mop when there is no outstanding commands */
3414 if (slot_status != 0) {
3415 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3416 ahci_portp->ahciport_mop_in_progress++;
3417
3418 ahci_mop_commands(ahci_ctlp,
3419 ahci_portp,
3420 slot_status,
3421 0, /* failed tags */
3422 0, /* timeout tags */
3423 abort_tags, /* aborting tags */
3424 0); /* reset tags */
3425 }
3426 }
3427
3428 #if defined(__lock_lint)
3429 static int
3430 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3431 {
3432 return (SATA_SUCCESS);
3433 }
3434 #endif
3435
3436 /*
3437 * Initialize fma capabilities and register with IO fault services.
3438 */
3439 static void
3440 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3441 {
3442 /*
3443 * Need to change iblock to priority for new MSI intr
3444 */
3445 ddi_iblock_cookie_t fm_ibc;
3446
3447 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3448 ahci_ctlp->ahcictl_dip,
3449 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3450 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3451 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3452
3453 /* Only register with IO Fault Services if we have some capability */
3454 if (ahci_ctlp->ahcictl_fm_cap) {
3455 /* Adjust access and dma attributes for FMA */
3456 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3457 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3458 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3459 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3460 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3461
3462 /*
3463 * Register capabilities with IO Fault Services.
3464 * ahcictl_fm_cap will be updated to indicate
3465 * capabilities actually supported (not requested.)
3466 */
3467 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3468 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3469
3470 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3471 cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3472 ddi_get_instance(ahci_ctlp->ahcictl_dip));
3473 return;
3474 }
3475 /*
3476 * Initialize pci ereport capabilities if ereport
3477 * capable (should always be.)
3478 */
3479 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3480 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3481 pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3482 }
3483
3484 /*
3485 * Register error callback if error callback capable.
3486 */
3487 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3488 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3489 ahci_fm_error_cb, (void *) ahci_ctlp);
3490 }
3491
3492 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3493 "ahci_fm_fini: fma enabled.", NULL);
3494 }
3495 }
3496
3497 /*
3498 * Releases fma capabilities and un-registers with IO fault services.
3499 */
3500 static void
3501 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3502 {
3503 /* Only unregister FMA capabilities if registered */
3504 if (ahci_ctlp->ahcictl_fm_cap) {
3505 /*
3506 * Un-register error callback if error callback capable.
3507 */
3508 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3509 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3510 }
3511
3512 /*
3513 * Release any resources allocated by pci_ereport_setup()
3514 */
3515 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3516 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3517 pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3518 }
3519
3520 /* Unregister from IO Fault Services */
3521 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3522
3523 /* Adjust access and dma attributes for FMA */
3524 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3525 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3526 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3527 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3528 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3529
3530 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3531 "ahci_fm_fini: fma disabled.", NULL);
3532 }
3533 }
3534
3535 /*ARGSUSED*/
3536 static int
3537 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3538 {
3539 /*
3540 * as the driver can always deal with an error in any dma or
3541 * access handle, we can just return the fme_status value.
3542 */
3543 pci_ereport_post(dip, err, NULL);
3544 return (err->fme_status);
3545 }
3546
3547 int
3548 ahci_check_acc_handle(ddi_acc_handle_t handle)
3549 {
3550 ddi_fm_error_t de;
3551
3552 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3553 return (de.fme_status);
3554 }
3555
3556 int
3557 ahci_check_dma_handle(ddi_dma_handle_t handle)
3558 {
3559 ddi_fm_error_t de;
3560
3561 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3562 return (de.fme_status);
3563 }
3564
3565 /*
3566 * Generate an ereport
3567 */
3568 void
3569 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3570 {
3571 uint64_t ena;
3572 char buf[FM_MAX_CLASS];
3573
3574 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3575 ena = fm_ena_generate(0, FM_ENA_FMT1);
3576 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3577 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3578 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3579 FM_EREPORT_VERSION, NULL);
3580 }
3581 }
3582
3583 /*
3584 * Check if all handles are correctly allocated.
3585 */
3586 static int
3587 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3588 {
3589 int port;
3590
3591 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3592 return (DDI_FAILURE);
3593 }
3594
3595 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3596 ahci_port_t *ahci_portp;
3597
3598 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3599 continue;
3600
3601 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3602
3603 mutex_enter(&ahci_portp->ahciport_mutex);
3604 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3605 mutex_exit(&ahci_portp->ahciport_mutex);
3606 return (DDI_FAILURE);
3607 }
3608 mutex_exit(&ahci_portp->ahciport_mutex);
3609 }
3610
3611 return (DDI_SUCCESS);
3612 }
3613
3614 /*
3615 * Check the access handles for the controller. Note that
3616 * ahcictl_pci_conf_handle is only used in attach process.
3617 */
3618 static int
3619 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3620 {
3621 if ((ahci_check_acc_handle(ahci_ctlp->
3622 ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3623 (ahci_check_acc_handle(ahci_ctlp->
3624 ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3625 return (DDI_FAILURE);
3626 }
3627 return (DDI_SUCCESS);
3628 }
3629
3630 /*
3631 * Check the DMA handles and the access handles of a controller port.
3632 */
3633 static int
3634 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3635 {
3636 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3637 int slot;
3638
3639 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3640
3641 if ((ahci_check_dma_handle(ahci_portp->
3642 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3643 (ahci_check_dma_handle(ahci_portp->
3644 ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3645 (ahci_check_acc_handle(ahci_portp->
3646 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3647 (ahci_check_acc_handle(ahci_portp->
3648 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3649 return (DDI_FAILURE);
3650 }
3651 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3652 if (ahci_check_slot_handle(ahci_portp, slot)
3653 != DDI_SUCCESS) {
3654 return (DDI_FAILURE);
3655 }
3656 }
3657 return (DDI_SUCCESS);
3658 }
3659
3660 /*
3661 * Check the DMA handles and the access handles of a cmd table slot.
3662 */
3663 static int
3664 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3665 {
3666 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3667
3668 if ((ahci_check_acc_handle(ahci_portp->
3669 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3670 (ahci_check_dma_handle(ahci_portp->
3671 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3672 return (DDI_FAILURE);
3673 }
3674 return (DDI_SUCCESS);
3675 }
3676
3677 /*
3678 * Allocate the ports structure, only called by ahci_attach
3679 */
3680 static int
3681 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3682 {
3683 int port, cport = 0;
3684
3685 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3686 "ahci_alloc_ports_state enter", NULL);
3687
3688 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3689
3690 /* Allocate structures only for the implemented ports */
3691 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3692 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3693 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3694 "hba port %d not implemented", port);
3695 continue;
3696 }
3697
3698 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3699 ahci_ctlp->ahcictl_port_to_cport[port] =
3700 (uint8_t)cport++;
3701
3702 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3703 goto err_out;
3704 }
3705 }
3706
3707 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3708 return (AHCI_SUCCESS);
3709
3710 err_out:
3711 for (port--; port >= 0; port--) {
3712 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3713 ahci_dealloc_port_state(ahci_ctlp, port);
3714 }
3715 }
3716
3717 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3718 return (AHCI_FAILURE);
3719 }
3720
3721 /*
3722 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3723 */
3724 static void
3725 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3726 {
3727 int port;
3728
3729 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3730 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3731 /* if this port is implemented by the HBA */
3732 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3733 ahci_dealloc_port_state(ahci_ctlp, port);
3734 }
3735 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3736 }
3737
3738 /*
3739 * Drain the taskq.
3740 */
3741 static void
3742 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3743 {
3744 ahci_port_t *ahci_portp;
3745 int port;
3746
3747 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3748 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3749 continue;
3750 }
3751
3752 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3753
3754 mutex_enter(&ahci_portp->ahciport_mutex);
3755 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3756 mutex_exit(&ahci_portp->ahciport_mutex);
3757 }
3758 }
3759
3760 /*
3761 * Initialize the controller and all ports. And then try to start the ports
3762 * if there are devices attached.
3763 *
3764 * This routine can be called from three seperate cases: DDI_ATTACH,
3765 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3766 * other two cases; device signature probing are attempted only during
3767 * DDI_ATTACH case.
3768 */
3769 static int
3770 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3771 {
3772 ahci_port_t *ahci_portp;
3773 ahci_addr_t addr;
3774 int port;
3775
3776 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3777 "ahci_initialize_controller enter", NULL);
3778
3779 /* Disable the whole controller interrupts */
3780 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3781 ahci_disable_all_intrs(ahci_ctlp);
3782 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3783
3784 /* Initialize the implemented ports and structures */
3785 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3786 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3787 continue;
3788 }
3789
3790 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3791 mutex_enter(&ahci_portp->ahciport_mutex);
3792
3793 /*
3794 * Ensure that the controller is not in the running state
3795 * by checking every implemented port's PxCMD register
3796 */
3797 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3798
3799 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3800 != AHCI_SUCCESS) {
3801 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3802 "ahci_initialize_controller: failed to "
3803 "initialize port %d", port);
3804 /*
3805 * Set the port state to SATA_PSTATE_FAILED if
3806 * failed to initialize it.
3807 */
3808 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3809 }
3810
3811 mutex_exit(&ahci_portp->ahciport_mutex);
3812 }
3813
3814 /* Enable the whole controller interrupts */
3815 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3816 ahci_enable_all_intrs(ahci_ctlp);
3817 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3818
3819 return (AHCI_SUCCESS);
3820 }
3821
3822 /*
3823 * Reverse of ahci_initialize_controller()
3824 *
3825 * We only need to stop the ports and disable the interrupt.
3826 */
3827 static void
3828 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3829 {
3830 ahci_port_t *ahci_portp;
3831 int port;
3832
3833 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3834 "ahci_uninitialize_controller enter", NULL);
3835
3836 /* disable all the interrupts. */
3837 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3838 ahci_disable_all_intrs(ahci_ctlp);
3839 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3840
3841 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3842 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3843 continue;
3844 }
3845
3846 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3847
3848 /* Stop the port by clearing PxCMD.ST */
3849 mutex_enter(&ahci_portp->ahciport_mutex);
3850
3851 /*
3852 * Here we must disable the port interrupt because
3853 * ahci_disable_all_intrs only clear GHC.IE, and IS
3854 * register will be still set if PxIE is enabled.
3855 * When ahci shares one IRQ with other drivers, the
3856 * intr handler may claim the intr mistakenly.
3857 */
3858 ahci_disable_port_intrs(ahci_ctlp, port);
3859 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3860 ahci_portp, port);
3861 mutex_exit(&ahci_portp->ahciport_mutex);
3862 }
3863 }
3864
3865 /*
3866 * ahci_alloc_pmult()
3867 * 1. Setting HBA port registers which are necessary for a port multiplier.
3868 * (Set PxCMD.PMA while PxCMD.ST is '0')
3869 * 2. Allocate ahci_pmult_info structure.
3870 *
3871 * NOTE: Must stop port before the function is called.
3872 */
3873 static void
3874 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3875 {
3876 uint32_t port_cmd_status;
3877 uint8_t port = ahci_portp->ahciport_port_num;
3878
3879 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3880
3881 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3882 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3883
3884 /* The port must have been stopped before. */
3885 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3886
3887 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3888 /* set PMA bit */
3889 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3890 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3891 port_cmd_status|AHCI_CMD_STATUS_PMA);
3892
3893 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3894 "ahci_alloc_pmult: "
3895 "PxCMD.PMA bit set at port %d.", port);
3896 }
3897
3898 /* Allocate port multiplier information structure */
3899 if (ahci_portp->ahciport_pmult_info == NULL) {
3900 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3901 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3902 }
3903
3904 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3905 }
3906
3907 /*
3908 * ahci_dealloc_pmult()
3909 * 1. Clearing related registers when a port multiplier is detached.
3910 * (Clear PxCMD.PMA while PxCMD.ST is '0')
3911 * 2. Deallocate ahci_pmult_info structure.
3912 *
3913 * NOTE: Must stop port before the function is called.
3914 */
3915 static void
3916 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3917 {
3918 uint32_t port_cmd_status;
3919 uint8_t port = ahci_portp->ahciport_port_num;
3920
3921 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3922
3923 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3924 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3925
3926 if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3927 /* Clear PMA bit */
3928 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3929 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3930 (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3931
3932 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3933 "ahci_dealloc_pmult: "
3934 "PxCMD.PMA bit cleared at port %d.", port);
3935 }
3936
3937 /* Release port multiplier information structure */
3938 if (ahci_portp->ahciport_pmult_info != NULL) {
3939 kmem_free(ahci_portp->ahciport_pmult_info,
3940 sizeof (ahci_pmult_info_t));
3941 ahci_portp->ahciport_pmult_info = NULL;
3942 }
3943 }
3944
3945 /*
3946 * Staggered Spin-up.
3947 */
3948 static void
3949 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3950 {
3951 uint32_t cap_status;
3952 uint32_t port_cmd_status;
3953
3954 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3955
3956 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3957 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3958
3959 /* Check for staggered spin-up support */
3960 if (!(cap_status & AHCI_HBA_CAP_SSS))
3961 return;
3962
3963 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3964 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3965
3966 /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3967 if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3968 return;
3969
3970 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3971
3972 /* Set PxCMD.SUD */
3973 port_cmd_status |= AHCI_CMD_STATUS_SUD;
3974 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3975 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3976 port_cmd_status);
3977 }
3978
3979 /*
3980 * The routine is to initialize a port. First put the port in NotRunning
3981 * state, then enable port interrupt and clear Serror register. And under
3982 * AHCI_ATTACH case, find device signature and then try to start the port.
3983 *
3984 * Called by
3985 * 1. ahci_initialize_controller
3986 * 2. ahci_intr_phyrdy_change (hotplug)
3987 */
3988 static int
3989 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3990 ahci_addr_t *addrp)
3991 {
3992 uint32_t port_sstatus, port_task_file, port_cmd_status;
3993 uint8_t port = addrp->aa_port;
3994 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */
3995 int ret;
3996
3997 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3998
3999 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */
4000 ASSERT(AHCI_ADDR_IS_PORT(addrp));
4001
4002 /*
4003 * At the time being, only probe ports/devices and get the types of
4004 * attached devices during DDI_ATTACH. In fact, the device can be
4005 * changed during power state changes, but at the time being, we
4006 * don't support the situation.
4007 */
4008 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4009 resuming = B_FALSE;
4010 } else {
4011 /* check for DDI_RESUME case */
4012 mutex_exit(&ahci_portp->ahciport_mutex);
4013 mutex_enter(&ahci_ctlp->ahcictl_mutex);
4014 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
4015 resuming = B_FALSE;
4016 mutex_exit(&ahci_ctlp->ahcictl_mutex);
4017 mutex_enter(&ahci_portp->ahciport_mutex);
4018 }
4019
4020 if (resuming) {
4021 /*
4022 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
4023 * and PxFBU registers in case these registers were cleared
4024 * during the suspend.
4025 */
4026 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4027 "ahci_initialize_port: port %d "
4028 "set PxCLB, PxCLBU, PxFB and PxFBU "
4029 "during resume", port);
4030
4031 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
4032 AHCI_SUCCESS)
4033 return (AHCI_FAILURE);
4034 }
4035
4036 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4037 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4038
4039 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4040 "ahci_initialize_port: port %d ", port);
4041
4042 /*
4043 * Check whether the port is in NotRunning state, if not,
4044 * put the port in NotRunning state
4045 */
4046 if (port_cmd_status &
4047 (AHCI_CMD_STATUS_ST |
4048 AHCI_CMD_STATUS_CR |
4049 AHCI_CMD_STATUS_FRE |
4050 AHCI_CMD_STATUS_FR)) {
4051 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
4052 ahci_portp, port);
4053 }
4054
4055 /* Make sure the drive is spun-up */
4056 ahci_staggered_spin_up(ahci_ctlp, port);
4057
4058 /* Disable interrupt */
4059 ahci_disable_port_intrs(ahci_ctlp, port);
4060
4061 /* Device is unknown at first */
4062 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4063
4064 /* Disable the interface power management */
4065 ahci_disable_interface_pm(ahci_ctlp, port);
4066
4067 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4068 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
4069 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4070 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4071
4072 /* Check physcial link status */
4073 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
4074 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
4075
4076 /* Check interface status */
4077 port_task_file & AHCI_TFD_STS_BSY ||
4078 port_task_file & AHCI_TFD_STS_DRQ ||
4079
4080 /* Check whether port reset must be executed */
4081 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
4082
4083 /* Always reset port on RESUME */
4084 resuming != B_FALSE) {
4085
4086 /* Something went wrong, we need do some reset things */
4087 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
4088
4089 /* Does port reset succeed on HBA port? */
4090 if (ret != AHCI_SUCCESS) {
4091 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4092 "ahci_initialize_port:"
4093 "port reset failed at port %d", port);
4094 return (AHCI_FAILURE);
4095 }
4096
4097 /* Is port failed? */
4098 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4099 SATA_PSTATE_FAILED) {
4100 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4101 "ahci_initialize_port: port %d state 0x%x",
4102 port, ahci_portp->ahciport_port_state);
4103 return (AHCI_FAILURE);
4104 }
4105 }
4106
4107 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4108 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4109
4110 /*
4111 * Try to get the device signature if the port is not empty.
4112 */
4113 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4114 SATA_DTYPE_NONE)
4115 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4116
4117 /* Return directly if no device connected */
4118 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4119 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4120 "No device connected to port %d", port);
4121 goto out;
4122 }
4123
4124 /* If this is a port multiplier, we need do some initialization */
4125 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4126 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4127 "Port multiplier found at port %d", port);
4128 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4129 }
4130
4131 /* Try to start the port */
4132 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4133 != AHCI_SUCCESS) {
4134 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4135 "failed to start port %d", port);
4136 return (AHCI_FAILURE);
4137 }
4138 out:
4139 /* Enable port interrupts */
4140 ahci_enable_port_intrs(ahci_ctlp, port);
4141
4142 return (AHCI_SUCCESS);
4143 }
4144
4145 /*
4146 * Handle hardware defect, and check the capabilities. For example,
4147 * power management capabilty and MSI capability.
4148 */
4149 static int
4150 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4151 {
4152 ushort_t caps_ptr, cap_count, cap;
4153 #if AHCI_DEBUG
4154 ushort_t pmcap, pmcsr;
4155 ushort_t msimc;
4156 #endif
4157 uint8_t revision;
4158
4159 ahci_ctlp->ahcictl_venid =
4160 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4161 PCI_CONF_VENID);
4162
4163 ahci_ctlp->ahcictl_devid =
4164 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4165 PCI_CONF_DEVID);
4166
4167 /*
4168 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4169 * controllers with 0x00 revision id work on 4-byte aligned buffer,
4170 * which is a bug and was fixed after 0x00 revision id controllers.
4171 *
4172 * Moreover, VT8251 cannot use multiple command slots in the command
4173 * list for non-queued commands because the previous register content
4174 * of PxCI can be re-written in the register write, so a flag will be
4175 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4176 *
4177 * For VT8251, software reset also has the same defect as the below
4178 * AMD/ATI chipset. That is, software reset will get failed if 0xf
4179 * is filled in pmport field. Therefore, another software reset need
4180 * to be done with 0 filled in pmport field.
4181 */
4182 if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4183 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4184 PCI_CONF_REVID);
4185 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4186 "revision id = 0x%x", revision);
4187 if (revision == 0x00) {
4188 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4189 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4190 "change ddi_attr_align to 0x4", NULL);
4191 }
4192
4193 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4194 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4195 "VT8251 cannot use multiple command lists for "
4196 "non-queued commands", NULL);
4197
4198 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4199 }
4200
4201 /*
4202 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4203 * DMA addressing for communication memory descriptors though S64A bit
4204 * of CAP register declares it supports. Even though 64-bit DMA for
4205 * data buffer works on ASUS M2A-VM with newer BIOS, three other
4206 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4207 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4208 *
4209 * Due to certain hardware issue, the chipset must do port reset during
4210 * initialization, otherwise, when retrieving device signature,
4211 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4212 * need to set.
4213 *
4214 * For this chipset software reset will get failure if the pmport of
4215 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4216 * multiplier is connected to the port. In order to fix the issue,
4217 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4218 * reset got failure, the driver will try to do another software reset
4219 * with pmport 0.
4220 */
4221 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4222 ahci_ctlp->ahcictl_devid == 0x4380) {
4223 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4224 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4225 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4226 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4227
4228 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4229 "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4230 "communication memory descriptors though CAP indicates "
4231 "support, so force it to use 32-bit DMA", NULL);
4232 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4233 "ATI SB600 need to do a port reset during initialization",
4234 NULL);
4235 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4236 "ATI SB600 will get software reset failure if pmport "
4237 "is set 0xf and no port multiplier is attached", NULL);
4238 }
4239
4240 /*
4241 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4242 * vendor ID and device ID (0x1002,0x4391).
4243 *
4244 * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4245 * DMA addressing for communication memory descriptors though S64A bit
4246 * of CAP register declares the support. However, it does support
4247 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4248 * set for this controller.
4249 *
4250 * SB710 has the same initialization issue as SB600, so it also need
4251 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4252 *
4253 * SB700 also has the same issue about software reset, and thus
4254 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4255 */
4256 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4257 ahci_ctlp->ahcictl_devid == 0x4391) {
4258 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4259 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4260 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4261
4262 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4263 "ATI SB700/750 cannot do 64-bit DMA for communication "
4264 "memory descriptors though CAP indicates support, "
4265 "so force it to use 32-bit DMA", NULL);
4266 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4267 "ATI SB710 need to do a port reset during initialization",
4268 NULL);
4269 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4270 "ATI SB700 will get software reset failure if pmport "
4271 "is set 0xf and no port multiplier is attached", NULL);
4272 }
4273
4274 /*
4275 * Check if capabilities list is supported and if so,
4276 * get initial capabilities pointer and clear bits 0,1.
4277 */
4278 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4279 PCI_CONF_STAT) & PCI_STAT_CAP) {
4280 caps_ptr = P2ALIGN(pci_config_get8(
4281 ahci_ctlp->ahcictl_pci_conf_handle,
4282 PCI_CONF_CAP_PTR), 4);
4283 } else {
4284 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4285 }
4286
4287 /*
4288 * Walk capabilities if supported.
4289 */
4290 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4291
4292 /*
4293 * Check that we haven't exceeded the maximum number of
4294 * capabilities and that the pointer is in a valid range.
4295 */
4296 if (++cap_count > PCI_CAP_MAX_PTR) {
4297 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4298 "too many device capabilities", NULL);
4299 return (AHCI_FAILURE);
4300 }
4301 if (caps_ptr < PCI_CAP_PTR_OFF) {
4302 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4303 "capabilities pointer 0x%x out of range",
4304 caps_ptr);
4305 return (AHCI_FAILURE);
4306 }
4307
4308 /*
4309 * Get next capability and check that it is valid.
4310 * For now, we only support power management.
4311 */
4312 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4313 caps_ptr);
4314 switch (cap) {
4315 case PCI_CAP_ID_PM:
4316
4317 /* power management supported */
4318 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4319
4320 /* Save PMCSR offset */
4321 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4322
4323 #if AHCI_DEBUG
4324 pmcap = pci_config_get16(
4325 ahci_ctlp->ahcictl_pci_conf_handle,
4326 caps_ptr + PCI_PMCAP);
4327 pmcsr = pci_config_get16(
4328 ahci_ctlp->ahcictl_pci_conf_handle,
4329 ahci_ctlp->ahcictl_pmcsr_offset);
4330 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4331 "Power Management capability found PCI_PMCAP "
4332 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4333 if ((pmcap & 0x3) == 0x3)
4334 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4335 "PCI Power Management Interface "
4336 "spec 1.2 compliant", NULL);
4337 #endif
4338 break;
4339
4340 case PCI_CAP_ID_MSI:
4341 #if AHCI_DEBUG
4342 msimc = pci_config_get16(
4343 ahci_ctlp->ahcictl_pci_conf_handle,
4344 caps_ptr + PCI_MSI_CTRL);
4345 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4346 "Message Signaled Interrupt capability found "
4347 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4348 #endif
4349 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4350 "MSI capability found", NULL);
4351 break;
4352
4353 case PCI_CAP_ID_PCIX:
4354 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4355 "PCI-X capability found", NULL);
4356 break;
4357
4358 case PCI_CAP_ID_PCI_E:
4359 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4360 "PCI Express capability found", NULL);
4361 break;
4362
4363 case PCI_CAP_ID_MSI_X:
4364 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4365 "MSI-X capability found", NULL);
4366 break;
4367
4368 case PCI_CAP_ID_SATA:
4369 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4370 "SATA capability found", NULL);
4371 break;
4372
4373 case PCI_CAP_ID_VS:
4374 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4375 "Vendor Specific capability found", NULL);
4376 break;
4377
4378 default:
4379 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4380 "unrecognized capability 0x%x", cap);
4381 break;
4382 }
4383
4384 /*
4385 * Get next capabilities pointer and clear bits 0,1.
4386 */
4387 caps_ptr = P2ALIGN(pci_config_get8(
4388 ahci_ctlp->ahcictl_pci_conf_handle,
4389 (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4390 }
4391
4392 return (AHCI_SUCCESS);
4393 }
4394
4395 /*
4396 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4397 * PORTMULT command. SYNC & POLLING mode is used.
4398 */
4399 static int
4400 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4401 uint8_t regn, uint32_t *pregv, uint8_t type)
4402 {
4403 ahci_port_t *ahci_portp;
4404 ahci_addr_t pmult_addr;
4405 sata_pkt_t *spkt;
4406 sata_cmd_t *scmd;
4407 sata_device_t sata_device;
4408 uint8_t port = addrp->aa_port;
4409 uint8_t pmport = addrp->aa_pmport;
4410 uint8_t cport;
4411 uint32_t intr_mask;
4412 int rval;
4413 char portstr[10];
4414
4415 SET_PORTSTR(portstr, addrp);
4416 cport = ahci_ctlp->ahcictl_port_to_cport[port];
4417 ahci_portp = ahci_ctlp->ahcictl_ports[port];
4418
4419 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4420 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4421
4422 /* Check the existence of the port multiplier */
4423 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4424 return (AHCI_FAILURE);
4425
4426 /* Request a READ/WRITE PORTMULT sata packet. */
4427 bzero(&sata_device, sizeof (sata_device_t));
4428 sata_device.satadev_addr.cport = cport;
4429 sata_device.satadev_addr.pmport = pmport;
4430 sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4431 sata_device.satadev_rev = SATA_DEVICE_REV;
4432
4433 /*
4434 * Make sure no command is outstanding here. All R/W PMULT requests
4435 * come from
4436 *
4437 * 1. ahci_attach()
4438 * The port should be empty.
4439 *
4440 * 2. ahci_tran_probe_port()
4441 * Any request from SATA framework (via ahci_tran_start) should be
4442 * rejected if R/W PMULT command is outstanding.
4443 *
4444 * If we are doing mopping, do not check those flags because no
4445 * command will be actually outstanding.
4446 *
4447 * If the port has been occupied by any other commands, the probe
4448 * function will return a SATA_RETRY. SATA framework will retry
4449 * later.
4450 */
4451 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4452 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4453 "R/W PMULT failed: R/W PMULT in progress at port %d.",
4454 port, ahci_portp->ahciport_flags);
4455 return (AHCI_FAILURE);
4456 }
4457
4458 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4459 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4460 NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4461 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4462 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4463 "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4464 port, ahci_portp->ahciport_flags);
4465 return (AHCI_FAILURE);
4466 }
4467
4468 /*
4469 * The port multiplier is gone. This may happen when
4470 * 1. Cutting off the power of an enclosure. The device lose the power
4471 * before port multiplier.
4472 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4473 *
4474 * The issued command should be aborted and the following command
4475 * should not be continued.
4476 */
4477 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4478 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4479 "READ/WRITE PMULT failed: "
4480 "port-mult is removed from port %d", port);
4481 return (AHCI_FAILURE);
4482 }
4483
4484 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4485
4486 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4487 &sata_device, regn, *pregv, type);
4488
4489 /*
4490 * READ/WRITE PORTMULT command is intended to sent to the control port
4491 * of the port multiplier.
4492 */
4493 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4494
4495 ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4496
4497 /* No interrupt here. Store the interrupt enable mask. */
4498 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4499 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4500 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4501 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4502
4503 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4504
4505 if (rval == AHCI_SUCCESS &&
4506 spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4507 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4508 scmd = &spkt->satapkt_cmd;
4509 *pregv = scmd->satacmd_lba_high_lsb << 24 |
4510 scmd->satacmd_lba_mid_lsb << 16 |
4511 scmd->satacmd_lba_low_lsb << 8 |
4512 scmd->satacmd_sec_count_lsb;
4513 }
4514 } else {
4515 /* Failed or not completed. */
4516 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4517 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4518 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4519 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4520 rval = AHCI_FAILURE;
4521 }
4522 out:
4523 /* Restore the interrupt mask */
4524 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4525 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4526
4527 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4528 ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4529 sata_free_rdwr_pmult_pkt(spkt);
4530 return (rval);
4531 }
4532
4533 static int
4534 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4535 uint8_t regn, uint32_t *pregv)
4536 {
4537 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4538 SATA_RDWR_PMULT_PKT_TYPE_READ);
4539 }
4540
4541 static int
4542 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4543 uint8_t regn, uint32_t regv)
4544 {
4545 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v,
4546 SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4547 }
4548
4549 #define READ_PMULT(addrp, r, pv, out) \
4550 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \
4551 goto out;
4552
4553 #define WRITE_PMULT(addrp, r, v, out) \
4554 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \
4555 goto out;
4556
4557 /*
4558 * Update sata registers on port multiplier, including GSCR/PSCR registers.
4559 * ahci_update_pmult_gscr()
4560 * ahci_update_pmult_pscr()
4561 */
4562 static int
4563 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4564 sata_pmult_gscr_t *sg)
4565 {
4566 ASSERT(MUTEX_HELD(
4567 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4568
4569 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4570 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4571 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4572 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4573
4574 return (AHCI_SUCCESS);
4575
4576 err: /* R/W PMULT error */
4577 return (AHCI_FAILURE);
4578 }
4579
4580 static int
4581 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4582 sata_device_t *sd)
4583 {
4584 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4585 ASSERT(MUTEX_HELD(
4586 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4587
4588 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4589 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4590 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4591 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4592
4593 return (AHCI_SUCCESS);
4594
4595 err: /* R/W PMULT error */
4596 return (AHCI_FAILURE);
4597 }
4598
4599 /*
4600 * ahci_initialize_pmult()
4601 *
4602 * Initialize a port multiplier, including
4603 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4604 * 2. Redefine MASK register. (SATA Chap 16.?)
4605 */
4606 static int
4607 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4608 ahci_addr_t *addrp, sata_device_t *sd)
4609 {
4610 sata_pmult_gscr_t sg;
4611 uint32_t gscr64;
4612 uint8_t port = addrp->aa_port;
4613
4614 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4615
4616 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4617 "[Initialize] Port-multiplier at port %d.", port);
4618
4619 /*
4620 * Enable features of port multiplier. Currently only
4621 * Asynchronous Notification is enabled.
4622 */
4623 /* Check gscr64 for supported features. */
4624 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4625
4626 if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4627 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4628 "port %d: Port Multiplier supports "
4629 "Asynchronous Notification.", port);
4630
4631 /* Write to gscr96 to enabled features */
4632 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4633 SATA_PMULT_CAP_SNOTIF, err);
4634
4635 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4636 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4637 AHCI_SNOTIF_CLEAR_ALL);
4638 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4639 "port %d: PMult PxSNTF cleared.", port);
4640
4641 }
4642
4643 /*
4644 * Now we need to update gscr33 register to enable hot-plug interrupt
4645 * for sub devices behind port multiplier.
4646 */
4647 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4648 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4649 "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4650
4651 /*
4652 * Fetch the number of device ports of the port multiplier
4653 */
4654 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4655 return (AHCI_FAILURE);
4656
4657 /* Register the port multiplier to SATA Framework. */
4658 mutex_exit(&ahci_portp->ahciport_mutex);
4659 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4660 mutex_enter(&ahci_portp->ahciport_mutex);
4661
4662 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4663 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4664
4665 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4666 "port %d: pmult sub-port number updated to %x.", port,
4667 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4668
4669 /* Till now port-mult is successfully initialized */
4670 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4671 return (AHCI_SUCCESS);
4672
4673 err: /* R/W PMULT error */
4674 return (AHCI_FAILURE);
4675 }
4676
4677 /*
4678 * Initialize a port multiplier port. According to spec, firstly we need
4679 * issue a COMRESET, then a software reset to get its signature.
4680 *
4681 * NOTE: This function should only be called in ahci_probe_pmport()
4682 */
4683 static int
4684 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4685 ahci_addr_t *addrp)
4686 {
4687 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4688 uint8_t port = addrp->aa_port;
4689 uint8_t pmport = addrp->aa_pmport;
4690 int ret = AHCI_FAILURE;
4691
4692 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4693 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4694
4695 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4696 "ahci_initialize_pmport: port %d:%d", port, pmport);
4697
4698 /* Check HBA port state */
4699 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4700 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4701 "ahci_initialize_pmport:"
4702 "port %d:%d Port Multiplier is failed.",
4703 port, pmport);
4704 return (AHCI_FAILURE);
4705 }
4706
4707 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4708 return (AHCI_FAILURE);
4709 }
4710 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4711
4712 /* Checking for outstanding commands */
4713 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4714 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4715 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4716 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4717 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4718 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4719 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4720 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4721 }
4722
4723 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4724 ahci_portp->ahciport_mop_in_progress++;
4725
4726 /* Clear status */
4727 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4728
4729 /* Firstly assume an unknown device */
4730 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4731
4732 ahci_disable_port_intrs(ahci_ctlp, port);
4733
4734 /* port reset is necessary for port multiplier port */
4735 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4736 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4737 "ahci_initialize_pmport:"
4738 "port reset failed at port %d:%d",
4739 port, pmport);
4740 goto out;
4741 }
4742
4743 /* Is port failed? */
4744 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4745 SATA_PSTATE_FAILED) {
4746 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4747 "ahci_initialize_pmport: port %d:%d failed. "
4748 "state = 0x%x", port, pmport,
4749 ahci_portp->ahciport_port_state);
4750 goto out;
4751 }
4752
4753 /* Is there any device attached? */
4754 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4755 == SATA_DTYPE_NONE) {
4756 /* Do not waste time on an empty port */
4757 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4758 "ahci_initialize_pmport: No device is found "
4759 "at port %d:%d", port, pmport);
4760 ret = AHCI_SUCCESS;
4761 goto out;
4762 }
4763
4764 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4765 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4766 "port %d:%d is ready now.", port, pmport);
4767
4768 /*
4769 * Till now we can assure a device attached to that HBA port and work
4770 * correctly. Now try to get the device signature. This is an optional
4771 * step. If failed, unknown device is assumed, then SATA module will
4772 * continue to use IDENTIFY DEVICE to get the information of the
4773 * device.
4774 */
4775 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4776
4777 ret = AHCI_SUCCESS;
4778
4779 out:
4780 /* Next try to mop the pending commands */
4781 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4782 finished_tags = ahci_portp->ahciport_pending_tags &
4783 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4784 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4785 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4786 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4787 reset_tags &= ~finished_tags;
4788
4789 ahci_mop_commands(ahci_ctlp,
4790 ahci_portp,
4791 slot_status,
4792 0, /* failed tags */
4793 0, /* timeout tags */
4794 0, /* aborted tags */
4795 reset_tags); /* reset tags */
4796
4797 /* Clear PxSNTF register if supported. */
4798 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4799 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4800 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4801 AHCI_SNOTIF_CLEAR_ALL);
4802 }
4803
4804 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4805 ahci_enable_port_intrs(ahci_ctlp, port);
4806 return (ret);
4807 }
4808
4809 /*
4810 * ahci_probe_pmult()
4811 *
4812 * This function will be called to probe a port multiplier, which will
4813 * handle hotplug events on port multiplier ports.
4814 *
4815 * NOTE: Only called from ahci_tran_probe_port()
4816 */
4817 static int
4818 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4819 ahci_addr_t *addrp)
4820 {
4821 sata_device_t sdevice;
4822 ahci_addr_t pmport_addr;
4823 uint32_t gscr32, port_hotplug_tags;
4824 uint32_t pmport_sstatus;
4825 int dev_exists_now = 0, dev_existed_previously = 0;
4826 uint8_t port = addrp->aa_port;
4827 int npmport;
4828
4829 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4830
4831 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4832 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4833 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4834
4835 do {
4836 npmport = ddi_ffs(port_hotplug_tags) - 1;
4837 if (npmport == -1)
4838 /* no pending hot plug events. */
4839 return (AHCI_SUCCESS);
4840
4841 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4842 "hot-plug event at port %d:%d", port, npmport);
4843
4844 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4845
4846 /* Check previous device at that port */
4847 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4848 != SATA_DTYPE_NONE)
4849 dev_existed_previously = 1;
4850
4851 /* PxSStatus tells the presence of device. */
4852 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4853 &pmport_sstatus, err);
4854
4855 if (SSTATUS_GET_DET(pmport_sstatus) ==
4856 SSTATUS_DET_DEVPRE_PHYCOM)
4857 dev_exists_now = 1;
4858
4859 /*
4860 * Clear PxSERR is critical. The transition from 0 to 1 will
4861 * emit a FIS which generates an asynchronous notification
4862 * event at controller. If we fail to clear the PxSERR, the
4863 * Async Notif events will no longer be activated on this
4864 * pmport.
4865 */
4866 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4867 AHCI_SERROR_CLEAR_ALL, err);
4868
4869 bzero((void *)&sdevice, sizeof (sata_device_t));
4870 sdevice.satadev_addr.cport = ahci_ctlp->
4871 ahcictl_port_to_cport[port];
4872 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4873 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4874 sdevice.satadev_state = SATA_PSTATE_PWRON;
4875
4876 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4877 "[Existence] %d -> %d", dev_existed_previously,
4878 dev_exists_now);
4879
4880 if (dev_exists_now) {
4881 if (dev_existed_previously) {
4882 /* Link (may) not change: Exist -> Exist * */
4883 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4884 "ahci_probe_pmult: port %d:%d "
4885 "device link lost/established",
4886 port, npmport);
4887
4888 mutex_exit(&ahci_portp->ahciport_mutex);
4889 sata_hba_event_notify(
4890 ahci_ctlp->ahcictl_sata_hba_tran->
4891 sata_tran_hba_dip,
4892 &sdevice,
4893 SATA_EVNT_LINK_LOST|
4894 SATA_EVNT_LINK_ESTABLISHED);
4895 mutex_enter(&ahci_portp->ahciport_mutex);
4896 } else {
4897 /* Link change: None -> Exist */
4898 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4899 "ahci_probe_pmult: port %d:%d "
4900 "device link established", port, npmport);
4901
4902 /* Clear port state */
4903 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4904 SATA_STATE_UNKNOWN);
4905 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4906 "ahci_probe_pmult: port %d "
4907 "ahciport_port_state [Cleared].", port);
4908
4909 mutex_exit(&ahci_portp->ahciport_mutex);
4910 sata_hba_event_notify(
4911 ahci_ctlp->ahcictl_sata_hba_tran->
4912 sata_tran_hba_dip,
4913 &sdevice,
4914 SATA_EVNT_LINK_ESTABLISHED);
4915 mutex_enter(&ahci_portp->ahciport_mutex);
4916 }
4917 } else { /* No device exists now */
4918 if (dev_existed_previously) {
4919
4920 /* Link change: Exist -> None */
4921 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4922 "ahci_probe_pmult: port %d:%d "
4923 "device link lost", port, npmport);
4924
4925 /* An existing device is lost. */
4926 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4927 SATA_STATE_UNKNOWN);
4928 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4929 SATA_DTYPE_NONE);
4930
4931 mutex_exit(&ahci_portp->ahciport_mutex);
4932 sata_hba_event_notify(
4933 ahci_ctlp->ahcictl_sata_hba_tran->
4934 sata_tran_hba_dip,
4935 &sdevice,
4936 SATA_EVNT_LINK_LOST);
4937 mutex_enter(&ahci_portp->ahciport_mutex);
4938 }
4939 }
4940
4941 CLEAR_BIT(port_hotplug_tags, npmport);
4942 } while (port_hotplug_tags != 0);
4943
4944 return (AHCI_SUCCESS);
4945
4946 err: /* R/W PMULT error */
4947 return (AHCI_FAILURE);
4948 }
4949
4950 /*
4951 * Probe and initialize a port multiplier port.
4952 * A port multiplier port could only be initilaizer here.
4953 */
4954 static int
4955 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4956 ahci_addr_t *addrp, sata_device_t *sd)
4957 {
4958 uint32_t port_state;
4959 uint8_t port = addrp->aa_port;
4960 ahci_addr_t addr_pmult;
4961
4962 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4963
4964 /*
4965 * Check the parent - port multiplier first.
4966 */
4967
4968 /*
4969 * Parent port multiplier might have been removed. This event will be
4970 * ignored and failure.
4971 */
4972 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4973 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4974 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4975 "ahci_tran_probe_port: "
4976 "parent device removed, ignore event.", NULL);
4977
4978 return (AHCI_FAILURE);
4979 }
4980
4981 /* The port is ready? */
4982 port_state = ahci_portp->ahciport_port_state;
4983 if (!(port_state & SATA_STATE_READY)) {
4984 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4985 "ahci_tran_probe_port: "
4986 "parent port-mult is NOT ready.", NULL);
4987
4988 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4989 ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4990 AHCI_SUCCESS) {
4991 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4992 "ahci_tran_probe_port: "
4993 "restart port-mult failed.", NULL);
4994 return (AHCI_FAILURE);
4995 }
4996 }
4997
4998 /*
4999 * If port-mult is restarted due to some reason, we need
5000 * re-initialized the PMult.
5001 */
5002 if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
5003 /* Initialize registers on a port multiplier */
5004 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
5005 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
5006 &addr_pmult, sd) != AHCI_SUCCESS)
5007 return (AHCI_FAILURE);
5008 }
5009
5010 /*
5011 * Then we check the port-mult port
5012 */
5013 /* Is this pmport initialized? */
5014 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5015 if (!(port_state & SATA_STATE_READY)) {
5016
5017 /* ahci_initialize_pmport() will set READY state */
5018 if (ahci_initialize_pmport(ahci_ctlp,
5019 ahci_portp, addrp) != AHCI_SUCCESS)
5020 return (AHCI_FAILURE);
5021 }
5022
5023 return (AHCI_SUCCESS);
5024 }
5025
5026 /*
5027 * AHCI device reset ...; a single device on one of the ports is reset,
5028 * but the HBA and physical communication remain intact. This is the
5029 * least intrusive.
5030 *
5031 * When issuing a software reset sequence, there should not be other
5032 * commands in the command list, so we will first clear and then re-set
5033 * PxCMD.ST to clear PxCI. And before issuing the software reset,
5034 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
5035 * cleared unless command list override (PxCMD.CLO) is supported.
5036 */
5037 static int
5038 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5039 ahci_addr_t *addrp)
5040 {
5041 ahci_fis_h2d_register_t *h2d_register_fisp;
5042 ahci_cmd_table_t *cmd_table;
5043 ahci_cmd_header_t *cmd_header;
5044 uint32_t port_cmd_status, port_cmd_issue, port_task_file;
5045 int slot, loop_count;
5046 uint8_t port = addrp->aa_port;
5047 uint8_t pmport = addrp->aa_pmport;
5048 int rval = AHCI_FAILURE;
5049
5050 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5051
5052 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5053 "port %d:%d device software resetting (FIS)", port, pmport);
5054
5055 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
5056 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5057 port) != AHCI_SUCCESS) {
5058 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5059 "ahci_software_reset: cannot stop HBA port %d.", port);
5060 goto out;
5061 }
5062
5063 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
5064 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5065 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5066
5067 if (port_task_file & AHCI_TFD_STS_BSY ||
5068 port_task_file & AHCI_TFD_STS_DRQ) {
5069 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
5070 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5071 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
5072 "cannot issue a software reset.", port_task_file);
5073 goto out;
5074 }
5075
5076 /*
5077 * If HBA Support CLO, as Command List Override (CAP.SCLO is
5078 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
5079 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
5080 */
5081 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5082 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
5083
5084 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5085 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5086 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5087 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5088 port_cmd_status|AHCI_CMD_STATUS_CLO);
5089
5090 /* Waiting till PxCMD.SCLO bit is cleared */
5091 loop_count = 0;
5092 do {
5093 /* Wait for 10 millisec */
5094 drv_usecwait(AHCI_10MS_USECS);
5095
5096 /* We are effectively timing out after 1 sec. */
5097 if (loop_count++ > 100) {
5098 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5099 "SCLO time out. port %d is busy.", port);
5100 goto out;
5101 }
5102
5103 port_cmd_status =
5104 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5105 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5106 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5107
5108 /* Re-check */
5109 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5110 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5111 if (port_task_file & AHCI_TFD_STS_BSY ||
5112 port_task_file & AHCI_TFD_STS_DRQ) {
5113 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5114 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5115 port_task_file);
5116 goto out;
5117 }
5118 }
5119
5120 /* Then start port */
5121 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5122 != AHCI_SUCCESS) {
5123 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5124 "ahci_software_reset: cannot start AHCI port %d.", port);
5125 goto out;
5126 }
5127
5128 /*
5129 * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5130 * ahci_port.ahciport_pending_ncq_tags may fail
5131 * ahci_claim_free_slot(). Actually according to spec, by clearing
5132 * PxCMD.ST there is no command outstanding while executing software
5133 * reseting. Hence we directly use slot 0 instead of
5134 * ahci_claim_free_slot().
5135 */
5136 slot = 0;
5137
5138 /* Now send the first H2D Register FIS with SRST set to 1 */
5139 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5140 bzero((void *)cmd_table, ahci_cmd_table_size);
5141
5142 h2d_register_fisp =
5143 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5144
5145 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5146 SET_FIS_PMP(h2d_register_fisp, pmport);
5147 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5148
5149 /* Set Command Header in Command List */
5150 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5151 BZERO_DESCR_INFO(cmd_header);
5152 BZERO_PRD_BYTE_COUNT(cmd_header);
5153 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5154 SET_PORT_MULTI_PORT(cmd_header, pmport);
5155
5156 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5157 SET_RESET(cmd_header, 1);
5158 SET_WRITE(cmd_header, 1);
5159
5160 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5161 0,
5162 ahci_cmd_table_size,
5163 DDI_DMA_SYNC_FORDEV);
5164
5165 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5166 slot * sizeof (ahci_cmd_header_t),
5167 sizeof (ahci_cmd_header_t),
5168 DDI_DMA_SYNC_FORDEV);
5169
5170 /* Indicate to the HBA that a command is active. */
5171 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5172 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5173 (0x1 << slot));
5174
5175 loop_count = 0;
5176
5177 /* Loop till the first command is finished */
5178 do {
5179 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5180 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5181
5182 /* We are effectively timing out after 1 sec. */
5183 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5184 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5185 "the first SRST FIS is timed out, "
5186 "loop_count = %d", loop_count);
5187 goto out;
5188 }
5189 /* Wait for 10 millisec */
5190 drv_usecwait(AHCI_10MS_USECS);
5191 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5192
5193 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5194 "ahci_software_reset: 1st loop count: %d, "
5195 "port_cmd_issue = 0x%x, slot = 0x%x",
5196 loop_count, port_cmd_issue, slot);
5197
5198 /* According to ATA spec, we need wait at least 5 microsecs here. */
5199 drv_usecwait(AHCI_1MS_USECS);
5200
5201 /* Now send the second H2D Register FIS with SRST cleard to zero */
5202 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5203 bzero((void *)cmd_table, ahci_cmd_table_size);
5204
5205 h2d_register_fisp =
5206 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5207
5208 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5209 SET_FIS_PMP(h2d_register_fisp, pmport);
5210
5211 /* Set Command Header in Command List */
5212 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5213 BZERO_DESCR_INFO(cmd_header);
5214 BZERO_PRD_BYTE_COUNT(cmd_header);
5215 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5216 SET_PORT_MULTI_PORT(cmd_header, pmport);
5217
5218 SET_WRITE(cmd_header, 1);
5219
5220 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5221 0,
5222 ahci_cmd_table_size,
5223 DDI_DMA_SYNC_FORDEV);
5224
5225 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5226 slot * sizeof (ahci_cmd_header_t),
5227 sizeof (ahci_cmd_header_t),
5228 DDI_DMA_SYNC_FORDEV);
5229
5230 /* Indicate to the HBA that a command is active. */
5231 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5232 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5233 (0x1 << slot));
5234
5235 loop_count = 0;
5236
5237 /* Loop till the second command is finished */
5238 do {
5239 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5240 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5241
5242 /* We are effectively timing out after 1 sec. */
5243 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5244 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5245 "the second SRST FIS is timed out, "
5246 "loop_count = %d", loop_count);
5247 goto out;
5248 }
5249
5250 /* Wait for 10 millisec */
5251 drv_usecwait(AHCI_10MS_USECS);
5252 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5253
5254 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5255 "ahci_software_reset: 2nd loop count: %d, "
5256 "port_cmd_issue = 0x%x, slot = 0x%x",
5257 loop_count, port_cmd_issue, slot);
5258
5259 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5260 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5261 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5262 DDI_SERVICE_UNAFFECTED);
5263 goto out;
5264 }
5265
5266 rval = AHCI_SUCCESS;
5267 out:
5268 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5269 "ahci_software_reset: %s at port %d:%d",
5270 rval == AHCI_SUCCESS ? "succeed" : "failed",
5271 port, pmport);
5272
5273 return (rval);
5274 }
5275
5276 /*
5277 * AHCI port reset ...; the physical communication between the HBA and device
5278 * on a port are disabled. This is more intrusive.
5279 *
5280 * When an HBA or port reset occurs, Phy communication is going to
5281 * be re-established with the device through a COMRESET followed by the
5282 * normal out-of-band communication sequence defined in Serial ATA. At
5283 * the end of reset, the device, if working properly, will send a D2H
5284 * Register FIS, which contains the device signature. When the HBA receives
5285 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5286 * the PxSIG register with the signature.
5287 *
5288 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5289 * function is called. If not, it is assumed the interface is in hung
5290 * condition.
5291 */
5292 static int
5293 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5294 ahci_addr_t *addrp)
5295 {
5296 ahci_addr_t pmult_addr;
5297 uint32_t port_cmd_status;
5298 uint32_t port_scontrol, port_sstatus;
5299 uint32_t port_task_file;
5300 uint32_t port_state;
5301 uint8_t port = addrp->aa_port;
5302
5303 int loop_count;
5304 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5305
5306 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5307
5308 /* Target is a port multiplier port? */
5309 if (AHCI_ADDR_IS_PMPORT(addrp))
5310 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5311
5312 /* Otherwise it must be an HBA port. */
5313 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5314
5315 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5316 "Port %d port resetting...", port);
5317 ahci_portp->ahciport_port_state = 0;
5318
5319 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5320 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5321
5322 /*
5323 * According to the spec, SUD bit should be set here,
5324 * but JMicron JMB363 doesn't follow it, so print
5325 * a debug message.
5326 */
5327 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5328 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5329 "ahci_port_reset: port %d SUD bit not set", port);
5330
5331 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5332 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5333 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5334
5335 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5336 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5337 port_scontrol);
5338
5339 /* Enable PxCMD.FRE to read device */
5340 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5341 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5342 port_cmd_status|AHCI_CMD_STATUS_FRE);
5343
5344 /*
5345 * The port enters P:StartComm state, and the HBA tells the link layer
5346 * to start communication, which involves sending COMRESET to the
5347 * device. And the HBA resets PxTFD.STS to 7Fh.
5348 *
5349 * Give time for COMRESET to percolate, according to the AHCI
5350 * spec, software shall wait at least 1 millisecond before
5351 * clearing PxSCTL.DET
5352 */
5353 drv_usecwait(AHCI_1MS_USECS * 2);
5354
5355 /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5356 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5357 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5358 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5359 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5360 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5361 port_scontrol);
5362
5363 /*
5364 * When a COMINIT is received from the device, then the port enters
5365 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5366 * PxSSTS.DET to 1h to indicate a device is detected but communication
5367 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5368 * a COMINIT has been received.
5369 */
5370 /*
5371 * The DET field is valid only if IPM field indicates
5372 * that the interface is in active state.
5373 */
5374 loop_count = 0;
5375 for (;;) {
5376 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5377 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5378
5379 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5380 /*
5381 * If the interface is not active, the DET field
5382 * is considered not accurate. So we want to
5383 * continue looping.
5384 */
5385 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5386 }
5387
5388 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5389 break;
5390
5391 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5392 /*
5393 * We are effectively timing out after 0.1 sec.
5394 */
5395 break;
5396 }
5397
5398 /* Wait for 10 millisec */
5399 drv_usecwait(AHCI_10MS_USECS);
5400 }
5401
5402 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5403 "ahci_port_reset: 1st loop count: %d, "
5404 "port_sstatus = 0x%x port %d",
5405 loop_count, port_sstatus, port);
5406
5407 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5408 /*
5409 * Either the port is not active or there
5410 * is no device present.
5411 */
5412 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5413 return (AHCI_SUCCESS);
5414 }
5415
5416 /* Clear port serror register for the port */
5417 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5418 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5419 AHCI_SERROR_CLEAR_ALL);
5420
5421 /*
5422 * Devices should return a FIS contains its signature to HBA after
5423 * COMINIT signal. Check whether a D2H Register FIS is received by
5424 * polling PxTFD.STS.
5425 */
5426 loop_count = 0;
5427 for (;;) {
5428 port_task_file =
5429 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5430 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5431
5432 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5433 AHCI_TFD_STS_ERR)) == 0)
5434 break;
5435
5436 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5437 /*
5438 * We are effectively timing out after 11 sec.
5439 */
5440 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5441 "the device hardware has been initialized and "
5442 "the power-up diagnostics failed",
5443 instance, port);
5444
5445 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5446 "port %d: some or all of BSY, DRQ and ERR in "
5447 "PxTFD.STS are not clear. We need another "
5448 "software reset.", port);
5449
5450 /* Clear port serror register for the port */
5451 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5452 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5453 AHCI_SERROR_CLEAR_ALL);
5454
5455 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5456
5457 /* Try another software reset. */
5458 if (ahci_software_reset(ahci_ctlp, ahci_portp,
5459 &pmult_addr) != AHCI_SUCCESS) {
5460 AHCIPORT_SET_STATE(ahci_portp, addrp,
5461 SATA_PSTATE_FAILED);
5462 return (AHCI_FAILURE);
5463 }
5464 break;
5465 }
5466
5467 /* Wait for 10 millisec */
5468 drv_usecwait(AHCI_10MS_USECS);
5469 }
5470
5471 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5472 "ahci_port_reset: 2nd loop count: %d, "
5473 "port_task_file = 0x%x port %d",
5474 loop_count, port_task_file, port);
5475
5476 /* Clear port serror register for the port */
5477 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5478 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5479 AHCI_SERROR_CLEAR_ALL);
5480
5481 /* Set port as ready */
5482 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5483 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5484
5485 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5486 "ahci_port_reset: succeed at port %d.", port);
5487 return (AHCI_SUCCESS);
5488 }
5489
5490 /*
5491 * COMRESET on a port multiplier port.
5492 *
5493 * NOTE: Only called in ahci_port_reset()
5494 */
5495 static int
5496 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5497 ahci_addr_t *addrp)
5498 {
5499 uint32_t port_scontrol, port_sstatus, port_serror;
5500 uint32_t port_cmd_status, port_intr_status;
5501 uint32_t port_state;
5502 uint8_t port = addrp->aa_port;
5503 uint8_t pmport = addrp->aa_pmport;
5504 int loop_count;
5505 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5506
5507 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5508 "port %d:%d: pmport resetting", port, pmport);
5509
5510 /* Initialize pmport state */
5511 AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5512
5513 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5514 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5515 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5516
5517 /* PxCMD.FRE should be set before. */
5518 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5519 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5520 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5521 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5522 return (AHCI_FAILURE);
5523
5524 /*
5525 * Give time for COMRESET to percolate, according to the AHCI
5526 * spec, software shall wait at least 1 millisecond before
5527 * clearing PxSCTL.DET
5528 */
5529 drv_usecwait(AHCI_1MS_USECS*2);
5530
5531 /*
5532 * Fetch the SCONTROL again and rewrite the DET part with 0
5533 * This will generate an Asychronous Notification events.
5534 */
5535 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5536 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5537 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5538
5539 /*
5540 * The port enters P:StartComm state, and HBA tells link layer to
5541 * start communication, which involves sending COMRESET to device.
5542 * And the HBA resets PxTFD.STS to 7Fh.
5543 *
5544 * When a COMINIT is received from the device, then the port enters
5545 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5546 * PxSSTS.DET to 1h to indicate a device is detected but communication
5547 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5548 * a COMINIT has been received.
5549 */
5550 /*
5551 * The DET field is valid only if IPM field indicates
5552 * that the interface is in active state.
5553 */
5554 loop_count = 0;
5555 do {
5556 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5557
5558 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5559 /*
5560 * If the interface is not active, the DET field
5561 * is considered not accurate. So we want to
5562 * continue looping.
5563 */
5564 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5565 }
5566
5567 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5568 /*
5569 * We are effectively timing out after 0.1 sec.
5570 */
5571 break;
5572 }
5573
5574 /* Wait for 10 millisec */
5575 drv_usecwait(AHCI_10MS_USECS);
5576
5577 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5578
5579 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5580 "ahci_pmport_reset: 1st loop count: %d, "
5581 "port_sstatus = 0x%x port %d:%d",
5582 loop_count, port_sstatus, port, pmport);
5583
5584 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5585 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5586 /*
5587 * Either the port is not active or there
5588 * is no device present.
5589 */
5590 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5591 "ahci_pmport_reset: "
5592 "no device attached to port %d:%d",
5593 port, pmport);
5594 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5595 return (AHCI_SUCCESS);
5596 }
5597
5598 /* Now we can make sure there is a device connected to the port */
5599 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5600 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5601
5602 if (!(port_serror & (1 << 26))) {
5603 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5604 "COMINIT signal from the device not received port %d:%d",
5605 instance, port, pmport);
5606
5607 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5608 return (AHCI_FAILURE);
5609 }
5610
5611 /*
5612 * After clear PxSERR register, we will receive a D2H FIS.
5613 * Normally this FIS will cause a IPMS error according to AHCI spec
5614 * v1.2 because there is no command outstanding for it. So we need
5615 * to ignore this error.
5616 */
5617 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5618 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5619
5620 /* Now we need to check the D2H FIS by checking IPMS error. */
5621 loop_count = 0;
5622 do {
5623 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5624 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5625
5626 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5627 /*
5628 * No D2H FIS received. This is possible according
5629 * to SATA 2.6 spec.
5630 */
5631 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5632 "PxIS.IPMS is not set, we need another "
5633 "software reset.", port, pmport);
5634
5635 break;
5636 }
5637
5638 /* Wait for 10 millisec */
5639 mutex_exit(&ahci_portp->ahciport_mutex);
5640 delay(AHCI_10MS_TICKS);
5641 mutex_enter(&ahci_portp->ahciport_mutex);
5642
5643 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5644
5645 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5646 "ahci_pmport_reset: 2st loop count: %d, "
5647 "port_sstatus = 0x%x port %d:%d",
5648 loop_count, port_sstatus, port, pmport);
5649
5650 /* Clear IPMS */
5651 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5652 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5653 AHCI_INTR_STATUS_IPMS);
5654 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5655
5656 /* This pmport is now ready for ahci_tran_start() */
5657 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5658 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5659
5660 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5661 "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5662 return (AHCI_SUCCESS);
5663
5664 err: /* R/W PMULT error */
5665 /* IPMS flags might be set before. */
5666 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5667 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5668 "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5669
5670 return (AHCI_FAILURE);
5671 }
5672
5673 /*
5674 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5675 * This is the most intrusive.
5676 *
5677 * When an HBA reset occurs, Phy communication will be re-established with
5678 * the device through a COMRESET followed by the normal out-of-band
5679 * communication sequence defined in Serial ATA. At the end of reset, the
5680 * device, if working properly, will send a D2H Register FIS, which contains
5681 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5682 * and PxTFD.ERR register fields, and updates the PxSIG register with the
5683 * signature.
5684 *
5685 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5686 */
5687 static int
5688 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5689 {
5690 ahci_port_t *ahci_portp;
5691 uint32_t ghc_control;
5692 uint8_t port;
5693 int loop_count;
5694 int rval = AHCI_SUCCESS;
5695
5696 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5697 NULL);
5698
5699 mutex_enter(&ahci_ctlp->ahcictl_mutex);
5700
5701 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5702 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5703
5704 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5705 ghc_control |= AHCI_HBA_GHC_HR;
5706 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5707 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5708
5709 /*
5710 * Wait until HBA Reset complete or timeout
5711 */
5712 loop_count = 0;
5713 do {
5714 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5715 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5716
5717 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5718 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5719 "ahci hba reset is timing out, "
5720 "ghc_control = 0x%x", ghc_control);
5721 /* We are effectively timing out after 1 sec. */
5722 break;
5723 }
5724
5725 /* Wait for 10 millisec */
5726 drv_usecwait(AHCI_10MS_USECS);
5727 } while (ghc_control & AHCI_HBA_GHC_HR);
5728
5729 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5730 "ahci_hba_reset: 1st loop count: %d, "
5731 "ghc_control = 0x%x", loop_count, ghc_control);
5732
5733 if (ghc_control & AHCI_HBA_GHC_HR) {
5734 /* The hba is not reset for some reasons */
5735 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5736 "hba reset failed: HBA in a hung or locked state", NULL);
5737 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5738 return (AHCI_FAILURE);
5739 }
5740
5741 /*
5742 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5743 */
5744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5746 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5748 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5749
5750 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5751
5752 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5753 /* Only check implemented ports */
5754 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5755 continue;
5756 }
5757
5758 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5759 mutex_enter(&ahci_portp->ahciport_mutex);
5760
5761 /* Make sure the drive is spun-up */
5762 ahci_staggered_spin_up(ahci_ctlp, port);
5763
5764 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5765 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5766 AHCI_SUCCESS) {
5767 rval = AHCI_FAILURE;
5768 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5769 "ahci_hba_reset: port %d failed", port);
5770 /*
5771 * Set the port state to SATA_PSTATE_FAILED if
5772 * failed to initialize it.
5773 */
5774 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5775 }
5776
5777 mutex_exit(&ahci_portp->ahciport_mutex);
5778 }
5779
5780 return (rval);
5781 }
5782
5783 /*
5784 * This routine is only called from AHCI_ATTACH or phyrdy change
5785 * case. It first calls software reset, then stop the port and try to
5786 * read PxSIG register to find the type of device attached to the port.
5787 *
5788 * The caller should make sure a valid device exists on specified port and
5789 * physical communication has been established so that the signature could
5790 * be retrieved by software reset.
5791 *
5792 * NOTE: The port interrupts should be disabled before the function is called.
5793 */
5794 static void
5795 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5796 ahci_addr_t *addrp)
5797 {
5798 ahci_addr_t dev_addr;
5799 uint32_t signature;
5800 uint8_t port = addrp->aa_port;
5801 uint8_t pmport = addrp->aa_pmport;
5802 int rval;
5803
5804 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5805 ASSERT(AHCI_ADDR_IS_VALID(addrp));
5806
5807 /*
5808 * If the HBA doesn't support port multiplier, then the driver
5809 * doesn't need to bother to check port multiplier device.
5810 *
5811 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5812 * connected to Silicon Image 4723, to which the two sata drives
5813 * attached can be set with RAID1, RAID0 or Spanning mode.
5814 *
5815 * We found software reset will get failure if port multiplier address
5816 * 0xf is used by software reset, so just ignore the check since
5817 * ICH7 doesn't support port multiplier device at all.
5818 */
5819 if (AHCI_ADDR_IS_PORT(addrp) &&
5820 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5821 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5822 "ahci_find_dev_signature enter: port %d", port);
5823
5824 /*
5825 * NOTE: when the ahci address is a HBA port, we do not know
5826 * it is a device or a port multiplier that attached. we need
5827 * try a software reset at port multiplier address (0xf
5828 * pmport)
5829 */
5830 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5831 } else {
5832 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5833 "ahci_find_dev_signature enter: port %d:%d",
5834 port, pmport);
5835 dev_addr = *addrp;
5836 }
5837
5838 /* Assume it is unknown. */
5839 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5840
5841 /* Issue a software reset to get the signature */
5842 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5843 if (rval != AHCI_SUCCESS) {
5844
5845 /*
5846 * Try to do software reset again with pmport set with 0 if
5847 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5848 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5849 */
5850 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5851 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5852 dev_addr.aa_pmport = 0;
5853 rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5854 &dev_addr);
5855 }
5856
5857 if (rval != AHCI_SUCCESS) {
5858 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5859 "ahci_find_dev_signature: software reset failed "
5860 "at port %d:%d, cannot get signature.",
5861 port, pmport);
5862
5863 AHCIPORT_SET_STATE(ahci_portp, addrp,
5864 SATA_PSTATE_FAILED);
5865 return;
5866 }
5867 }
5868
5869 /*
5870 * ahci_software_reset has started the port, so we need manually stop
5871 * the port again.
5872 */
5873 if (AHCI_ADDR_IS_PORT(addrp)) {
5874 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5875 ahci_portp, port) != AHCI_SUCCESS) {
5876 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5877 "ahci_find_dev_signature: cannot stop port %d.",
5878 port);
5879 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5880 return;
5881 }
5882 }
5883
5884 /* Now we can make sure that a valid signature is received. */
5885 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5886 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5887
5888 if (AHCI_ADDR_IS_PMPORT(addrp)) {
5889 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5890 "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5891 signature, port, pmport);
5892 } else {
5893 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5894 "ahci_find_dev_signature: signature = 0x%x at port %d",
5895 signature, port);
5896 }
5897
5898 /* NOTE: Only support ATAPI device at controller port. */
5899 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5900 signature = SATA_DTYPE_UNKNOWN;
5901
5902 switch (signature) {
5903
5904 case AHCI_SIGNATURE_DISK:
5905 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5906 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5907 "Disk is found at port: %d", port);
5908 break;
5909
5910 case AHCI_SIGNATURE_ATAPI:
5911 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5912 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5913 "ATAPI device is found at port: %d", port);
5914 break;
5915
5916 case AHCI_SIGNATURE_PORT_MULTIPLIER:
5917 /* Port Multiplier cannot recursively attached. */
5918 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5919 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5920 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5921 "Port Multiplier is found at port: %d", port);
5922 break;
5923
5924 default:
5925 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5926 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5927 "Unknown device is found at port: %d", port);
5928 }
5929 }
5930
5931 /*
5932 * According to the spec, to reliably detect hot plug removals, software
5933 * must disable interface power management. Software should perform the
5934 * following initialization on a port after a device is attached:
5935 * Set PxSCTL.IPM to 3h to disable interface state transitions
5936 * Set PxCMD.ALPE to '0' to disable aggressive power management
5937 * Disable device initiated interface power management by SET FEATURE
5938 *
5939 * We can ignore the last item because by default the feature is disabled
5940 */
5941 static void
5942 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5943 {
5944 uint32_t port_scontrol, port_cmd_status;
5945
5946 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5947 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5948 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5949 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5950 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5951
5952 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5953 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5954 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5955 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5956 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5957 }
5958
5959 /*
5960 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5961 * to 1, then set it firstly.
5962 *
5963 * Each port contains two major DMA engines. One DMA engine walks through
5964 * the command list, and is controlled by PxCMD.ST. The second DMA engine
5965 * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5966 *
5967 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5968 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5969 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5970 *
5971 * Software shall not set PxCMD.ST to '1' unless a functional device is
5972 * present on the port(as determined by PxTFD.STS.BSY = '0',
5973 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5974 */
5975 static int
5976 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5977 {
5978 uint32_t port_cmd_status;
5979
5980 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5981
5982 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5983
5984 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5985 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5986 "the state for port %d is 0x%x",
5987 port, ahci_portp->ahciport_port_state);
5988 return (AHCI_FAILURE);
5989 }
5990
5991 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5992 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5993 "no device is attached at port %d", port);
5994 return (AHCI_FAILURE);
5995 }
5996
5997 /* First to set PxCMD.FRE before setting PxCMD.ST. */
5998 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5999 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6000
6001 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
6002 port_cmd_status |= AHCI_CMD_STATUS_FRE;
6003 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6004 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6005 port_cmd_status);
6006 }
6007
6008 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6009 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6010
6011 port_cmd_status |= AHCI_CMD_STATUS_ST;
6012
6013 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6014 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6015 port_cmd_status);
6016
6017 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
6018
6019 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
6020 "PxCMD.ST set to '1' at port %d", port);
6021
6022 return (AHCI_SUCCESS);
6023 }
6024
6025 /*
6026 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
6027 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
6028 * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
6029 */
6030 static int
6031 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6032 {
6033 uint8_t port = ahci_portp->ahciport_port_num;
6034 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6035 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6036
6037 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6038
6039 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
6040 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
6041 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
6042 port) != AHCI_SUCCESS)
6043 return (AHCI_FAILURE);
6044
6045 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6046 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6047 }
6048
6049 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
6050 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
6051 int loop_count = 0;
6052
6053 /* Clear PxCMD.FRE */
6054 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
6055 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6056 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6057 port_cmd_status);
6058
6059 /* Wait until PxCMD.FR is cleared */
6060 for (;;) {
6061 port_cmd_status =
6062 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6063 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6064
6065 if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
6066 break;
6067
6068 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
6069 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
6070 "ahci_setup_port_base_addresses: cannot "
6071 "clear PxCMD.FR for port %d.", port);
6072
6073 /*
6074 * We are effectively timing out after 0.5 sec.
6075 * This value is specified in AHCI spec.
6076 */
6077 return (AHCI_FAILURE);
6078 }
6079
6080 /* Wait for 1 millisec */
6081 drv_usecwait(AHCI_1MS_USECS);
6082 }
6083 }
6084
6085 /* Step 3: Config Port Command List Base Address */
6086 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6087 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
6088 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6089
6090 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6091 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
6092 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
6093
6094 /* Step 4: Config Port Received FIS Base Address */
6095 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6096 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
6097 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6098
6099 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6100 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6101 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6102
6103 return (AHCI_SUCCESS);
6104 }
6105
6106 /*
6107 * Allocate the ahci_port_t including Received FIS and Command List.
6108 * The argument - port is the physical port number, and not logical
6109 * port number seen by the SATA framework.
6110 */
6111 static int
6112 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6113 {
6114 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6115 ahci_port_t *ahci_portp;
6116 char taskq_name[64] = "event_handle_taskq";
6117
6118 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6119
6120 ahci_portp =
6121 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6122
6123 ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6124 ahci_portp->ahciport_port_num = port;
6125
6126 /* Initialize the port condition variable */
6127 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6128
6129 /* Initialize the port mutex */
6130 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6131 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6132
6133 mutex_enter(&ahci_portp->ahciport_mutex);
6134
6135 /*
6136 * Allocate memory for received FIS structure and
6137 * command list for this port
6138 */
6139 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6140 goto err_case1;
6141 }
6142
6143 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6144 goto err_case2;
6145 }
6146
6147 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6148 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6149 AHCI_SUCCESS) {
6150 goto err_case3;
6151 }
6152
6153 (void) snprintf(taskq_name + strlen(taskq_name),
6154 sizeof (taskq_name) - strlen(taskq_name),
6155 "_port%d", port);
6156
6157 /* Create the taskq for the port */
6158 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6159 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6160 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6161 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6162 goto err_case3;
6163 }
6164
6165 /* Allocate the argument for the taskq */
6166 ahci_portp->ahciport_event_args =
6167 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6168
6169 ahci_portp->ahciport_event_args->ahciea_addrp =
6170 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6171
6172 /* Initialize the done queue */
6173 ahci_portp->ahciport_doneq = NULL;
6174 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6175 ahci_portp->ahciport_doneq_len = 0;
6176
6177 mutex_exit(&ahci_portp->ahciport_mutex);
6178
6179 return (AHCI_SUCCESS);
6180
6181 err_case3:
6182 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6183
6184 err_case2:
6185 ahci_dealloc_rcvd_fis(ahci_portp);
6186
6187 err_case1:
6188 mutex_exit(&ahci_portp->ahciport_mutex);
6189 mutex_destroy(&ahci_portp->ahciport_mutex);
6190 cv_destroy(&ahci_portp->ahciport_cv);
6191
6192 kmem_free(ahci_portp, sizeof (ahci_port_t));
6193
6194 return (AHCI_FAILURE);
6195 }
6196
6197 /*
6198 * Reverse of ahci_alloc_port_state().
6199 */
6200 static void
6201 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6202 {
6203 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6204
6205 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6206 ASSERT(ahci_portp != NULL);
6207
6208 mutex_enter(&ahci_portp->ahciport_mutex);
6209 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6210 sizeof (ahci_addr_t));
6211 ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6212 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6213 ahci_portp->ahciport_event_args = NULL;
6214 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6215 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6216 ahci_dealloc_rcvd_fis(ahci_portp);
6217 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6218 mutex_exit(&ahci_portp->ahciport_mutex);
6219
6220 mutex_destroy(&ahci_portp->ahciport_mutex);
6221 cv_destroy(&ahci_portp->ahciport_cv);
6222
6223 kmem_free(ahci_portp, sizeof (ahci_port_t));
6224
6225 ahci_ctlp->ahcictl_ports[port] = NULL;
6226 }
6227
6228 /*
6229 * Allocates memory for the Received FIS Structure
6230 */
6231 static int
6232 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6233 {
6234 size_t rcvd_fis_size;
6235 size_t ret_len;
6236 uint_t cookie_count;
6237
6238 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6239
6240 rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6241
6242 /* allocate rcvd FIS dma handle. */
6243 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6244 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6245 DDI_DMA_SLEEP,
6246 NULL,
6247 &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6248 DDI_SUCCESS) {
6249 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6250 "rcvd FIS dma handle alloc failed", NULL);
6251
6252 return (AHCI_FAILURE);
6253 }
6254
6255 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6256 rcvd_fis_size,
6257 &accattr,
6258 DDI_DMA_CONSISTENT,
6259 DDI_DMA_SLEEP,
6260 NULL,
6261 (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6262 &ret_len,
6263 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6264
6265 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6266 "rcvd FIS dma mem alloc fail", NULL);
6267 /* error.. free the dma handle. */
6268 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6269 return (AHCI_FAILURE);
6270 }
6271
6272 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6273 NULL,
6274 (caddr_t)ahci_portp->ahciport_rcvd_fis,
6275 rcvd_fis_size,
6276 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6277 DDI_DMA_SLEEP,
6278 NULL,
6279 &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6280 &cookie_count) != DDI_DMA_MAPPED) {
6281
6282 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6283 "rcvd FIS dma handle bind fail", NULL);
6284 /* error.. free the dma handle & free the memory. */
6285 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6286 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6287 return (AHCI_FAILURE);
6288 }
6289
6290 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6291
6292 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6293 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6294 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6295 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6296
6297 return (AHCI_SUCCESS);
6298 }
6299
6300 /*
6301 * Deallocates the Received FIS Structure
6302 */
6303 static void
6304 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6305 {
6306 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6307
6308 /* Unbind the cmd list dma handle first. */
6309 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6310
6311 /* Then free the underlying memory. */
6312 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6313
6314 /* Now free the handle itself. */
6315 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6316 }
6317
6318 /*
6319 * Allocates memory for the Command List, which contains up to 32 entries.
6320 * Each entry contains a command header, which is a 32-byte structure that
6321 * includes the pointer to the command table.
6322 */
6323 static int
6324 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6325 {
6326 size_t cmd_list_size;
6327 size_t ret_len;
6328 uint_t cookie_count;
6329
6330 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6331
6332 cmd_list_size =
6333 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6334
6335 /* allocate cmd list dma handle. */
6336 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6337 &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6338 DDI_DMA_SLEEP,
6339 NULL,
6340 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6341
6342 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6343 "cmd list dma handle alloc failed", NULL);
6344 return (AHCI_FAILURE);
6345 }
6346
6347 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6348 cmd_list_size,
6349 &accattr,
6350 DDI_DMA_CONSISTENT,
6351 DDI_DMA_SLEEP,
6352 NULL,
6353 (caddr_t *)&ahci_portp->ahciport_cmd_list,
6354 &ret_len,
6355 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6356
6357 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6358 "cmd list dma mem alloc fail", NULL);
6359 /* error.. free the dma handle. */
6360 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6361 return (AHCI_FAILURE);
6362 }
6363
6364 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6365 NULL,
6366 (caddr_t)ahci_portp->ahciport_cmd_list,
6367 cmd_list_size,
6368 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6369 DDI_DMA_SLEEP,
6370 NULL,
6371 &ahci_portp->ahciport_cmd_list_dma_cookie,
6372 &cookie_count) != DDI_DMA_MAPPED) {
6373
6374 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6375 "cmd list dma handle bind fail", NULL);
6376 /* error.. free the dma handle & free the memory. */
6377 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6378 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6379 return (AHCI_FAILURE);
6380 }
6381
6382 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6383
6384 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6385 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6386
6387 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6388 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6389
6390 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6391 goto err_out;
6392 }
6393
6394 return (AHCI_SUCCESS);
6395
6396 err_out:
6397 /* Unbind the cmd list dma handle first. */
6398 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6399
6400 /* Then free the underlying memory. */
6401 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6402
6403 /* Now free the handle itself. */
6404 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6405
6406 return (AHCI_FAILURE);
6407 }
6408
6409 /*
6410 * Deallocates the Command List
6411 */
6412 static void
6413 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6414 {
6415 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6416
6417 /* First dealloc command table */
6418 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6419
6420 /* Unbind the cmd list dma handle first. */
6421 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6422
6423 /* Then free the underlying memory. */
6424 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6425
6426 /* Now free the handle itself. */
6427 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6428 }
6429
6430 /*
6431 * Allocates memory for all Command Tables, which contains Command FIS,
6432 * ATAPI Command and Physical Region Descriptor Table.
6433 */
6434 static int
6435 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6436 {
6437 size_t ret_len;
6438 ddi_dma_cookie_t cmd_table_dma_cookie;
6439 uint_t cookie_count;
6440 int slot;
6441
6442 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6443
6444 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6445 "ahci_alloc_cmd_tables: port %d enter",
6446 ahci_portp->ahciport_port_num);
6447
6448 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6449 /* Allocate cmd table dma handle. */
6450 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6451 &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6452 DDI_DMA_SLEEP,
6453 NULL,
6454 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6455 DDI_SUCCESS) {
6456
6457 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6458 "cmd table dma handle alloc failed", NULL);
6459
6460 goto err_out;
6461 }
6462
6463 if (ddi_dma_mem_alloc(
6464 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6465 ahci_cmd_table_size,
6466 &accattr,
6467 DDI_DMA_CONSISTENT,
6468 DDI_DMA_SLEEP,
6469 NULL,
6470 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6471 &ret_len,
6472 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6473 NULL) {
6474
6475 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6476 "cmd table dma mem alloc fail", NULL);
6477
6478 /* error.. free the dma handle. */
6479 ddi_dma_free_handle(
6480 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6481 goto err_out;
6482 }
6483
6484 if (ddi_dma_addr_bind_handle(
6485 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6486 NULL,
6487 (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6488 ahci_cmd_table_size,
6489 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6490 DDI_DMA_SLEEP,
6491 NULL,
6492 &cmd_table_dma_cookie,
6493 &cookie_count) != DDI_DMA_MAPPED) {
6494
6495 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6496 "cmd table dma handle bind fail", NULL);
6497 /* error.. free the dma handle & free the memory. */
6498 ddi_dma_mem_free(
6499 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6500 ddi_dma_free_handle(
6501 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6502 goto err_out;
6503 }
6504
6505 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6506 ahci_cmd_table_size);
6507
6508 /* Config Port Command Table Base Address */
6509 SET_COMMAND_TABLE_BASE_ADDR(
6510 (&ahci_portp->ahciport_cmd_list[slot]),
6511 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6512
6513 #ifndef __lock_lint
6514 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6515 (&ahci_portp->ahciport_cmd_list[slot]),
6516 cmd_table_dma_cookie.dmac_laddress >> 32);
6517 #endif
6518 }
6519
6520 return (AHCI_SUCCESS);
6521 err_out:
6522
6523 for (slot--; slot >= 0; slot--) {
6524 /* Unbind the cmd table dma handle first */
6525 (void) ddi_dma_unbind_handle(
6526 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6527
6528 /* Then free the underlying memory */
6529 ddi_dma_mem_free(
6530 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6531
6532 /* Now free the handle itself */
6533 ddi_dma_free_handle(
6534 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6535 }
6536
6537 return (AHCI_FAILURE);
6538 }
6539
6540 /*
6541 * Deallocates memory for all Command Tables.
6542 */
6543 static void
6544 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6545 {
6546 int slot;
6547
6548 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6549
6550 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6551 "ahci_dealloc_cmd_tables: %d enter",
6552 ahci_portp->ahciport_port_num);
6553
6554 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6555 /* Unbind the cmd table dma handle first. */
6556 (void) ddi_dma_unbind_handle(
6557 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6558
6559 /* Then free the underlying memory. */
6560 ddi_dma_mem_free(
6561 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6562
6563 /* Now free the handle itself. */
6564 ddi_dma_free_handle(
6565 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6566 }
6567 }
6568
6569 /*
6570 * Update SATA registers at controller ports
6571 */
6572 static void
6573 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6574 sata_device_t *sd)
6575 {
6576 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6577
6578 sd->satadev_scr.sstatus =
6579 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6580 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6581 sd->satadev_scr.serror =
6582 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6583 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6584 sd->satadev_scr.scontrol =
6585 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6586 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6587 sd->satadev_scr.sactive =
6588 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6589 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6590 }
6591
6592 /*
6593 * For poll mode, ahci_port_intr will be called to emulate the interrupt
6594 */
6595 static void
6596 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6597 {
6598 uint32_t port_intr_status;
6599 uint32_t port_intr_enable;
6600
6601 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6602 "ahci_port_intr enter: port %d", port);
6603
6604 mutex_enter(&ahci_portp->ahciport_mutex);
6605 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6606 /* For SATA_OPMODE_POLLING commands */
6607 port_intr_enable =
6608 (AHCI_INTR_STATUS_DHRS |
6609 AHCI_INTR_STATUS_PSS |
6610 AHCI_INTR_STATUS_SDBS |
6611 AHCI_INTR_STATUS_UFS |
6612 AHCI_INTR_STATUS_PCS |
6613 AHCI_INTR_STATUS_PRCS |
6614 AHCI_INTR_STATUS_OFS |
6615 AHCI_INTR_STATUS_INFS |
6616 AHCI_INTR_STATUS_IFS |
6617 AHCI_INTR_STATUS_HBDS |
6618 AHCI_INTR_STATUS_HBFS |
6619 AHCI_INTR_STATUS_TFES);
6620 } else {
6621 /*
6622 * port_intr_enable indicates that the corresponding interrrupt
6623 * reporting is enabled.
6624 */
6625 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6626 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6627 }
6628
6629 /* IPMS error in port reset should be ignored according AHCI spec. */
6630 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6631 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6632 mutex_exit(&ahci_portp->ahciport_mutex);
6633
6634 /*
6635 * port_intr_stats indicates that the corresponding interrupt
6636 * condition is active.
6637 */
6638 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6639 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6640
6641 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6642 "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6643 "port_intr_enable = 0x%x",
6644 port, port_intr_status, port_intr_enable);
6645
6646 port_intr_status &= port_intr_enable;
6647
6648 /*
6649 * Pending interrupt events are indicated by the PxIS register.
6650 * Make sure we don't miss any event.
6651 */
6652 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6653 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6654 DDI_SERVICE_UNAFFECTED);
6655 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6656 DDI_FME_VERSION);
6657 return;
6658 }
6659
6660 /* First clear the port interrupts status */
6661 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6662 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6663 port_intr_status);
6664
6665 /* Check the completed non-queued commands */
6666 if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6667 AHCI_INTR_STATUS_PSS)) {
6668 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6669 ahci_portp, port);
6670 }
6671
6672 /* Check the completed queued commands */
6673 if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6674 (void) ahci_intr_set_device_bits(ahci_ctlp,
6675 ahci_portp, port);
6676 }
6677
6678 /* Check the port connect change status interrupt bit */
6679 if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6680 (void) ahci_intr_port_connect_change(ahci_ctlp,
6681 ahci_portp, port);
6682 }
6683
6684 /* Check the device mechanical presence status interrupt bit */
6685 if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6686 (void) ahci_intr_device_mechanical_presence_status(
6687 ahci_ctlp, ahci_portp, port);
6688 }
6689
6690 /* Check the PhyRdy change status interrupt bit */
6691 if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6692 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6693 port);
6694 }
6695
6696 /*
6697 * Check the non-fatal error interrupt bits, there are four
6698 * kinds of non-fatal errors at the time being:
6699 *
6700 * PxIS.UFS - Unknown FIS Error
6701 * PxIS.OFS - Overflow Error
6702 * PxIS.INFS - Interface Non-Fatal Error
6703 * PxIS.IPMS - Incorrect Port Multiplier Status Error
6704 *
6705 * For these non-fatal errors, the HBA can continue to operate,
6706 * so the driver just log the error messages.
6707 */
6708 if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6709 AHCI_INTR_STATUS_OFS |
6710 AHCI_INTR_STATUS_IPMS |
6711 AHCI_INTR_STATUS_INFS)) {
6712 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6713 port, port_intr_status);
6714 }
6715
6716 /*
6717 * Check the fatal error interrupt bits, there are four kinds
6718 * of fatal errors for AHCI controllers:
6719 *
6720 * PxIS.HBFS - Host Bus Fatal Error
6721 * PxIS.HBDS - Host Bus Data Error
6722 * PxIS.IFS - Interface Fatal Error
6723 * PxIS.TFES - Task File Error
6724 *
6725 * The fatal error means the HBA can not recover from it by
6726 * itself, and it will try to abort the transfer, and the software
6727 * must intervene to restart the port.
6728 */
6729 if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6730 AHCI_INTR_STATUS_HBDS |
6731 AHCI_INTR_STATUS_HBFS |
6732 AHCI_INTR_STATUS_TFES))
6733 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6734 port, port_intr_status);
6735
6736 /* Check the cold port detect interrupt bit */
6737 if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6738 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6739 }
6740
6741 /* Second clear the corresponding bit in IS.IPS */
6742 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6743 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6744
6745 /* Try to recover at the end of the interrupt handler. */
6746 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6747 DDI_FM_OK) {
6748 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6749 DDI_SERVICE_UNAFFECTED);
6750 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6751 DDI_FME_VERSION);
6752 }
6753 }
6754
6755 /*
6756 * Interrupt service handler
6757 */
6758 static uint_t
6759 ahci_intr(caddr_t arg1, caddr_t arg2)
6760 {
6761 #ifndef __lock_lint
6762 _NOTE(ARGUNUSED(arg2))
6763 #endif
6764 /* LINTED */
6765 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6766 ahci_port_t *ahci_portp;
6767 int32_t global_intr_status;
6768 uint8_t port;
6769
6770 /*
6771 * global_intr_status indicates that the corresponding port has
6772 * an interrupt pending.
6773 */
6774 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6775 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6776
6777 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6778 /* The interrupt is not ours */
6779 return (DDI_INTR_UNCLAIMED);
6780 }
6781
6782 /*
6783 * Check the handle after reading global_intr_status - we don't want
6784 * to miss any port with pending interrupts.
6785 */
6786 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6787 DDI_FM_OK) {
6788 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6789 DDI_SERVICE_UNAFFECTED);
6790 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6791 DDI_FME_VERSION);
6792 return (DDI_INTR_UNCLAIMED);
6793 }
6794
6795 /* Loop for all the ports */
6796 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6797 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6798 continue;
6799 }
6800 if (!((0x1 << port) & global_intr_status)) {
6801 continue;
6802 }
6803
6804 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6805
6806 /* Call ahci_port_intr */
6807 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6808 }
6809
6810 return (DDI_INTR_CLAIMED);
6811 }
6812
6813 /*
6814 * For non-queued commands, when the corresponding bit in the PxCI register
6815 * is cleared, it means the command is completed successfully. And according
6816 * to the HBA state machine, there are three conditions which possibly will
6817 * try to clear the PxCI register bit.
6818 * 1. Receive one D2H Register FIS which is with 'I' bit set
6819 * 2. Update PIO Setup FIS
6820 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6821 *
6822 * Process completed non-queued commands when the interrupt status bit -
6823 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6824 *
6825 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6826 * with the 'I' bit set. And the following commands will send thus
6827 * FIS with 'I' bit set upon the successful completion:
6828 * 1. Non-data commands
6829 * 2. DMA data-in command
6830 * 3. DMA data-out command
6831 * 4. PIO data-out command
6832 * 5. PACKET non-data commands
6833 * 6. PACKET PIO data-in command
6834 * 7. PACKET PIO data-out command
6835 * 8. PACKET DMA data-in command
6836 * 9. PACKET DMA data-out command
6837 *
6838 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6839 * with the 'I' bit set. And the following commands will send this
6840 * FIS upon the successful completion:
6841 * 1. PIO data-in command
6842 */
6843 static int
6844 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6845 ahci_port_t *ahci_portp, uint8_t port)
6846 {
6847 uint32_t port_cmd_issue = 0;
6848 uint32_t finished_tags;
6849 int finished_slot;
6850 sata_pkt_t *satapkt;
6851 ahci_fis_d2h_register_t *rcvd_fisp;
6852 #if AHCI_DEBUG
6853 ahci_cmd_header_t *cmd_header;
6854 uint32_t cmd_dmacount;
6855 #endif
6856
6857 mutex_enter(&ahci_portp->ahciport_mutex);
6858
6859 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6860 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6861 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6862 /*
6863 * Spurious interrupt. Nothing to be done.
6864 */
6865 mutex_exit(&ahci_portp->ahciport_mutex);
6866 return (AHCI_SUCCESS);
6867 }
6868
6869 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6870 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6871
6872 /* If the PxCI corrupts, don't complete the commmands. */
6873 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6874 != DDI_FM_OK) {
6875 mutex_exit(&ahci_portp->ahciport_mutex);
6876 return (AHCI_FAILURE);
6877 }
6878
6879 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6880 /* Slot 0 is always used during error recovery */
6881 finished_tags = 0x1 & ~port_cmd_issue;
6882 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6883 "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6884 "retrieval is finished, and finished_tags = 0x%x",
6885 port, finished_tags);
6886 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6887 finished_tags = 0x1 & ~port_cmd_issue;
6888 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6889 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6890 "port multiplier is finished, and finished_tags = 0x%x",
6891 port, finished_tags);
6892
6893 } else {
6894
6895 finished_tags = ahci_portp->ahciport_pending_tags &
6896 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6897 }
6898
6899 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6900 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6901 "port_cmd_issue = 0x%x finished_tags = 0x%x",
6902 ahci_portp->ahciport_pending_tags, port_cmd_issue,
6903 finished_tags);
6904
6905 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6906 (finished_tags == 0x1)) {
6907 satapkt = ahci_portp->ahciport_err_retri_pkt;
6908 ASSERT(satapkt != NULL);
6909
6910 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6911 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6912 "with SATA_PKT_COMPLETED", (void *)satapkt);
6913
6914 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6915 goto out;
6916 }
6917
6918 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6919 (finished_tags == 0x1)) {
6920 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6921 ASSERT(satapkt != NULL);
6922
6923 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6924 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6925 "with SATA_PKT_COMPLETED", (void *)satapkt);
6926
6927 /* READ PORTMULT need copy out FIS content. */
6928 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6929 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6930 ahcirf_d2h_register_fis);
6931 satapkt->satapkt_cmd.satacmd_status_reg =
6932 GET_RFIS_STATUS(rcvd_fisp);
6933 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6934 }
6935
6936 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6937 goto out;
6938 }
6939
6940 while (finished_tags) {
6941 finished_slot = ddi_ffs(finished_tags) - 1;
6942 if (finished_slot == -1) {
6943 goto out;
6944 }
6945
6946 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6947 ASSERT(satapkt != NULL);
6948 #if AHCI_DEBUG
6949 /*
6950 * For non-native queued commands, the PRD byte count field
6951 * shall contain an accurate count of the number of bytes
6952 * transferred for the command before the PxCI bit is cleared
6953 * to '0' for the command.
6954 *
6955 * The purpose of this field is to let software know how many
6956 * bytes transferred for a given operation in order to
6957 * determine if underflow occurred. When issuing native command
6958 * queuing commands, this field should not be used and is not
6959 * required to be valid since in this case underflow is always
6960 * illegal.
6961 *
6962 * For data reads, the HBA will update its PRD byte count with
6963 * the total number of bytes received from the last FIS, and
6964 * may be able to continue normally. For data writes, the
6965 * device will detect an error, and HBA most likely will get
6966 * a fatal error.
6967 *
6968 * Therefore, here just put code to debug part. And please
6969 * refer to the comment above ahci_intr_fatal_error for the
6970 * definition of underflow error.
6971 */
6972 cmd_dmacount =
6973 ahci_portp->ahciport_prd_bytecounts[finished_slot];
6974 if (cmd_dmacount) {
6975 cmd_header =
6976 &ahci_portp->ahciport_cmd_list[finished_slot];
6977 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6978 "ahci_intr_cmd_cmplt: port %d, "
6979 "PRD Byte Count = 0x%x, "
6980 "ahciport_prd_bytecounts = 0x%x", port,
6981 cmd_header->ahcich_prd_byte_count,
6982 cmd_dmacount);
6983
6984 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6985 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6986 "ahci_intr_cmd_cmplt: port %d, "
6987 "an underflow occurred", port);
6988 }
6989 }
6990 #endif
6991
6992 /*
6993 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6994 * feature, sata_special_regs flag will be set, and the
6995 * driver should copy the status and the other corresponding
6996 * register values in the D2H Register FIS received (It's
6997 * working on Non-data protocol) from the device back to
6998 * the sata_cmd.
6999 *
7000 * For every AHCI port, there is only one Received FIS
7001 * structure, which contains the FISes received from the
7002 * device, So we're trying to copy the content of D2H
7003 * Register FIS in the Received FIS structure back to
7004 * the sata_cmd.
7005 */
7006 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
7007 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
7008 ahcirf_d2h_register_fis);
7009 satapkt->satapkt_cmd.satacmd_status_reg =
7010 GET_RFIS_STATUS(rcvd_fisp);
7011 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
7012 }
7013
7014 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7015 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
7016 "with SATA_PKT_COMPLETED", (void *)satapkt);
7017
7018 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
7019 CLEAR_BIT(finished_tags, finished_slot);
7020 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7021
7022 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7023 }
7024 out:
7025 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
7026 "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
7027 ahci_portp->ahciport_pending_tags);
7028
7029 ahci_flush_doneq(ahci_portp);
7030
7031 mutex_exit(&ahci_portp->ahciport_mutex);
7032
7033 return (AHCI_SUCCESS);
7034 }
7035
7036 /*
7037 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
7038 * with the 'I' bit set and has been copied into system memory. It will
7039 * be sent under the following situations:
7040 *
7041 * 1. NCQ command is completed
7042 *
7043 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
7044 * via the Set Device Bits FIS. When such event is generated, the software
7045 * needs to read PxSACT register and compares the current value to the
7046 * list of commands previously issue by software. ahciport_pending_ncq_tags
7047 * keeps the tags of previously issued commands.
7048 *
7049 * 2. Asynchronous Notification
7050 *
7051 * Asynchronous Notification is a feature in SATA spec 2.6.
7052 *
7053 * 1) ATAPI device will send a signal to the host when media is inserted or
7054 * removed and avoids polling the device for media changes. The signal
7055 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
7056 * set to '1'. At the moment, it's not supported yet.
7057 *
7058 * 2) Port multiplier will send a signal to the host when a hot plug event
7059 * has occured on a port multiplier port. It is used when command based
7060 * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
7061 */
7062 static int
7063 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
7064 ahci_port_t *ahci_portp, uint8_t port)
7065 {
7066 ahci_addr_t addr;
7067
7068 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7069 "ahci_intr_set_device_bits enter: port %d", port);
7070
7071 /* Initialize HBA port address */
7072 AHCI_ADDR_SET_PORT(&addr, port);
7073
7074 /* NCQ plug handler */
7075 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
7076
7077 /* Check port multiplier's asynchronous notification events */
7078 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
7079 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
7080 ahci_portp, port);
7081 }
7082
7083 /* ATAPI events is not supported yet */
7084
7085 return (AHCI_SUCCESS);
7086 }
7087 /*
7088 * NCQ interrupt handler. Called upon a NCQ command is completed.
7089 * Only be called from ahci_intr_set_device_bits().
7090 */
7091 static int
7092 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7093 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7094 {
7095 uint32_t port_sactive;
7096 uint32_t port_cmd_issue;
7097 uint32_t issued_tags;
7098 int issued_slot;
7099 uint32_t finished_tags;
7100 int finished_slot;
7101 uint8_t port = addrp->aa_port;
7102 sata_pkt_t *satapkt;
7103
7104 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7105 "ahci_intr_set_device_bits enter: port %d", port);
7106
7107 mutex_enter(&ahci_portp->ahciport_mutex);
7108 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7109 mutex_exit(&ahci_portp->ahciport_mutex);
7110 return (AHCI_SUCCESS);
7111 }
7112
7113 /*
7114 * First the handler got which commands are finished by checking
7115 * PxSACT register
7116 */
7117 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7118 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7119
7120 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7121 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7122
7123 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7124 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7125 "port_sactive = 0x%x", port,
7126 ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7127
7128 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7129 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7130
7131 /*
7132 * For NCQ commands, the software can determine which command has
7133 * already been transmitted to the device by checking PxCI register.
7134 */
7135 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7136 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7137
7138 issued_tags = ahci_portp->ahciport_pending_tags &
7139 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7140
7141 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7142 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7143 != DDI_FM_OK) {
7144 mutex_exit(&ahci_portp->ahciport_mutex);
7145 return (AHCI_FAILURE);
7146 }
7147
7148 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7149 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7150 "port_cmd_issue = 0x%x", port,
7151 ahci_portp->ahciport_pending_tags, port_cmd_issue);
7152
7153 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7154 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7155
7156 /*
7157 * Clear ahciport_pending_tags bit when the corresponding command
7158 * is already sent down to the device.
7159 */
7160 while (issued_tags) {
7161 issued_slot = ddi_ffs(issued_tags) - 1;
7162 if (issued_slot == -1) {
7163 goto next;
7164 }
7165 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7166 CLEAR_BIT(issued_tags, issued_slot);
7167 }
7168
7169 next:
7170 while (finished_tags) {
7171 finished_slot = ddi_ffs(finished_tags) - 1;
7172 if (finished_slot == -1) {
7173 goto out;
7174 }
7175
7176 /* The command is certainly transmitted to the device */
7177 ASSERT(!(ahci_portp->ahciport_pending_tags &
7178 (0x1 << finished_slot)));
7179
7180 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7181 ASSERT(satapkt != NULL);
7182
7183 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7184 "ahci_intr_set_device_bits: sending up pkt 0x%p "
7185 "with SATA_PKT_COMPLETED", (void *)satapkt);
7186
7187 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7188 CLEAR_BIT(finished_tags, finished_slot);
7189 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7190
7191 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7192 }
7193 out:
7194 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7195 "ahci_intr_set_device_bits: port %d "
7196 "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7197 port, ahci_portp->ahciport_pending_ncq_tags,
7198 ahci_portp->ahciport_pending_tags);
7199
7200 ahci_flush_doneq(ahci_portp);
7201
7202 mutex_exit(&ahci_portp->ahciport_mutex);
7203
7204 return (AHCI_SUCCESS);
7205 }
7206
7207 /*
7208 * Port multiplier asynchronous notification event handler. Called upon a
7209 * device is hot plugged/pulled.
7210 *
7211 * The async-notification event will only be recorded by ahcipmi_snotif_tags
7212 * here and will be handled by ahci_probe_pmult().
7213 *
7214 * NOTE: called only from ahci_port_intr().
7215 */
7216 static int
7217 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7218 uint8_t port)
7219 {
7220 sata_device_t sdevice;
7221
7222 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7223 "ahci_intr_pmult_sntf_events enter: port %d ", port);
7224
7225 /* no hot-plug while attaching process */
7226 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7227 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7228 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7229 return (AHCI_SUCCESS);
7230 }
7231 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7232
7233 mutex_enter(&ahci_portp->ahciport_mutex);
7234 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7235 mutex_exit(&ahci_portp->ahciport_mutex);
7236 return (AHCI_SUCCESS);
7237 }
7238
7239 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7240
7241 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7242 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7243 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7244 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7245 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7246 AHCI_SNOTIF_CLEAR_ALL);
7247
7248 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7249 mutex_exit(&ahci_portp->ahciport_mutex);
7250 return (AHCI_SUCCESS);
7251 }
7252
7253 /* Port Multiplier sub-device hot-plug handler */
7254 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7255 mutex_exit(&ahci_portp->ahciport_mutex);
7256 return (AHCI_SUCCESS);
7257 }
7258
7259 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7260 /* Not allowed to re-enter. */
7261 mutex_exit(&ahci_portp->ahciport_mutex);
7262 return (AHCI_SUCCESS);
7263 }
7264
7265 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7266
7267 /*
7268 * NOTE:
7269 * Even if Asynchronous Notification is supported (and enabled) by
7270 * both controller and the port multiplier, the content of PxSNTF
7271 * register is always set to 0x8000 by async notification event. We
7272 * need to check GSCR[32] on the port multiplier to find out the
7273 * owner of this event.
7274 * This is not accord with SATA spec 2.6 and needs further
7275 * clarification.
7276 */
7277 /* hot-plug will not reported while reseting. */
7278 if (ahci_portp->ahciport_reset_in_progress == 1) {
7279 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7280 "port %d snotif event ignored", port);
7281 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7282 mutex_exit(&ahci_portp->ahciport_mutex);
7283 return (AHCI_SUCCESS);
7284 }
7285
7286 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7287 "PxSNTF is set to 0x%x by port multiplier",
7288 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7289
7290 /*
7291 * Now we need do some necessary operation and inform SATA framework
7292 * that link/device events has happened.
7293 */
7294 bzero((void *)&sdevice, sizeof (sata_device_t));
7295 sdevice.satadev_addr.cport = ahci_ctlp->
7296 ahcictl_port_to_cport[port];
7297 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7298 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7299 sdevice.satadev_state = SATA_PSTATE_PWRON;
7300
7301 /* Just reject packets, do not stop that port. */
7302 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7303
7304 mutex_exit(&ahci_portp->ahciport_mutex);
7305 sata_hba_event_notify(
7306 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7307 &sdevice,
7308 SATA_EVNT_PMULT_LINK_CHANGED);
7309 mutex_enter(&ahci_portp->ahciport_mutex);
7310
7311 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7312 mutex_exit(&ahci_portp->ahciport_mutex);
7313
7314 return (AHCI_SUCCESS);
7315 }
7316
7317 /*
7318 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7319 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7320 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7321 * indicates a COMINIT signal was received.
7322 *
7323 * Hot plug insertion is detected by reception of a COMINIT signal from the
7324 * device. On reception of unsolicited COMINIT, the HBA shall generate a
7325 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7326 * begin the normal communication negotiation sequence as outlined in the
7327 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7328 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7329 * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7330 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7331 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7332 * to handle hot plug insertion. In this interrupt handler, just do nothing
7333 * but print some log message and clear the bit.
7334 */
7335 static int
7336 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7337 ahci_port_t *ahci_portp, uint8_t port)
7338 {
7339 #if AHCI_DEBUG
7340 uint32_t port_serror;
7341 #endif
7342
7343 mutex_enter(&ahci_portp->ahciport_mutex);
7344
7345 #if AHCI_DEBUG
7346 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7347 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7348
7349 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7350 "ahci_intr_port_connect_change: port %d, "
7351 "port_serror = 0x%x", port, port_serror);
7352 #endif
7353
7354 /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7355 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7356 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7357 SERROR_EXCHANGED_ERR);
7358
7359 mutex_exit(&ahci_portp->ahciport_mutex);
7360
7361 return (AHCI_SUCCESS);
7362 }
7363
7364 /*
7365 * Hot Plug Operation for platforms that support Mechanical Presence
7366 * Switches.
7367 *
7368 * When set, it indicates that a mechanical presence switch attached to this
7369 * port has been opened or closed, which may lead to a change in the connection
7370 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7371 * are set to '1'.
7372 *
7373 * At the moment, this interrupt is not needed and disabled and we just log
7374 * the debug message.
7375 */
7376 static int
7377 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7378 ahci_port_t *ahci_portp, uint8_t port)
7379 {
7380 uint32_t cap_status, port_cmd_status;
7381
7382 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7383 "ahci_intr_device_mechanical_presence_status enter, "
7384 "port %d", port);
7385
7386 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7387 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7388
7389 mutex_enter(&ahci_portp->ahciport_mutex);
7390 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7391 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7392
7393 if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7394 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7395 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7396 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7397 "the interrupt: cap_status = 0x%x, "
7398 "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7399 mutex_exit(&ahci_portp->ahciport_mutex);
7400
7401 return (AHCI_SUCCESS);
7402 }
7403
7404 #if AHCI_DEBUG
7405 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7406 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7407 "The mechanical presence switch is open: "
7408 "port %d, port_cmd_status = 0x%x",
7409 port, port_cmd_status);
7410 } else {
7411 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7412 "The mechanical presence switch is close: "
7413 "port %d, port_cmd_status = 0x%x",
7414 port, port_cmd_status);
7415 }
7416 #endif
7417
7418 mutex_exit(&ahci_portp->ahciport_mutex);
7419
7420 return (AHCI_SUCCESS);
7421 }
7422
7423 /*
7424 * Native Hot Plug Support.
7425 *
7426 * When set, it indicates that the internal PHYRDY signal changed state.
7427 * This bit reflects the state of PxSERR.DIAG.N.
7428 *
7429 * There are three kinds of conditions to generate this interrupt event:
7430 * 1. a device is inserted
7431 * 2. a device is disconnected
7432 * 3. when the link enters/exits a Partial or Slumber interface power
7433 * management state
7434 *
7435 * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7436 * bit may be set due to the link entering the Partial or Slumber power
7437 * management state, rather than due to a hot plug insertion or removal
7438 * event. So far, the interface power management is disabled, so the
7439 * driver can reliably get removal detection notification via the
7440 * PxSERR.DIAG.N bit.
7441 */
7442 static int
7443 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7444 ahci_port_t *ahci_portp, uint8_t port)
7445 {
7446 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7447 sata_device_t sdevice;
7448 int dev_exists_now = 0;
7449 int dev_existed_previously = 0;
7450 ahci_addr_t port_addr;
7451
7452 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7453 "ahci_intr_phyrdy_change enter, port %d", port);
7454
7455 /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7456 mutex_enter(&ahci_portp->ahciport_mutex);
7457 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7458 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7459 SERROR_PHY_RDY_CHG);
7460 mutex_exit(&ahci_portp->ahciport_mutex);
7461
7462 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7463 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7464 (ahci_portp == NULL)) {
7465 /* The whole controller setup is not yet done. */
7466 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7467 return (AHCI_SUCCESS);
7468 }
7469 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7470
7471 mutex_enter(&ahci_portp->ahciport_mutex);
7472
7473 /* SStatus tells the presence of device. */
7474 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7475 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7476
7477 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7478 dev_exists_now = 1;
7479 }
7480
7481 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7482 dev_existed_previously = 1;
7483 }
7484
7485 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7486 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7487 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7488 "ahci_intr_phyrdy_change: port %d "
7489 "AHCI_PORT_FLAG_NODEV is cleared", port);
7490 if (dev_exists_now == 0)
7491 dev_existed_previously = 1;
7492 }
7493
7494 bzero((void *)&sdevice, sizeof (sata_device_t));
7495 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7496 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7497 sdevice.satadev_addr.pmport = 0;
7498 sdevice.satadev_state = SATA_PSTATE_PWRON;
7499 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7500
7501 AHCI_ADDR_SET_PORT(&port_addr, port);
7502
7503 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7504 if (dev_exists_now) {
7505 if (dev_existed_previously) { /* 1 -> 1 */
7506 /* Things are fine now. The loss was temporary. */
7507 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7508 "ahci_intr_phyrdy_change port %d "
7509 "device link lost/established", port);
7510
7511 mutex_exit(&ahci_portp->ahciport_mutex);
7512 sata_hba_event_notify(
7513 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7514 &sdevice,
7515 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7516 mutex_enter(&ahci_portp->ahciport_mutex);
7517
7518 } else { /* 0 -> 1 */
7519 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7520 "ahci_intr_phyrdy_change: port %d "
7521 "device link established", port);
7522
7523 /*
7524 * A new device has been detected. The new device
7525 * might be a port multiplier instead of a drive, so
7526 * we cannot update the signature directly.
7527 */
7528 (void) ahci_initialize_port(ahci_ctlp,
7529 ahci_portp, &port_addr);
7530
7531 /* Try to start the port */
7532 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7533 != AHCI_SUCCESS) {
7534 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7535 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7536 "ahci_intr_phyrdy_change: port %d failed "
7537 "at start port", port);
7538 }
7539
7540 /* Clear the max queue depth for inserted device */
7541 ahci_portp->ahciport_max_ncq_tags = 0;
7542
7543 mutex_exit(&ahci_portp->ahciport_mutex);
7544 sata_hba_event_notify(
7545 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7546 &sdevice,
7547 SATA_EVNT_LINK_ESTABLISHED);
7548 mutex_enter(&ahci_portp->ahciport_mutex);
7549
7550 }
7551 } else { /* No device exists now */
7552
7553 if (dev_existed_previously) { /* 1 -> 0 */
7554 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7555 "ahci_intr_phyrdy_change: port %d "
7556 "device link lost", port);
7557
7558 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7559 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7560 ahci_portp, port);
7561
7562 if (ahci_portp->ahciport_device_type ==
7563 SATA_DTYPE_PMULT) {
7564 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7565 }
7566
7567 /* An existing device is lost. */
7568 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7569 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7570
7571 mutex_exit(&ahci_portp->ahciport_mutex);
7572 sata_hba_event_notify(
7573 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7574 &sdevice,
7575 SATA_EVNT_LINK_LOST);
7576 mutex_enter(&ahci_portp->ahciport_mutex);
7577 }
7578 }
7579 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7580
7581 mutex_exit(&ahci_portp->ahciport_mutex);
7582
7583 return (AHCI_SUCCESS);
7584 }
7585
7586 /*
7587 * PxIS.UFS - Unknown FIS Error
7588 *
7589 * This interrupt event means an unknown FIS was received and has been
7590 * copied into system memory. An unknown FIS is not considered an illegal
7591 * FIS, unless the length received is more than 64 bytes. If an unknown
7592 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7593 * normal operation. If the unknown FIS is more than 64 bytes, then it
7594 * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7595 * a fatal error.
7596 *
7597 * PxIS.IPMS - Incorrect Port Multiplier Status
7598 *
7599 * IPMS Indicates that the HBA received a FIS from a device that did not
7600 * have a command outstanding. The IPMS bit may be set during enumeration
7601 * of devices on a Port Multiplier due to the normal Port Multiplier
7602 * enumeration process. It is recommended that IPMS only be used after
7603 * enumeration is complete on the Port Multiplier (copied from spec).
7604 *
7605 * PxIS.OFS - Overflow Error
7606 *
7607 * Command list overflow is defined as software building a command table
7608 * that has fewer total bytes than the transaction given to the device.
7609 * On device writes, the HBA will run out of data, and on reads, there
7610 * will be no room to put the data.
7611 *
7612 * For an overflow on data read, either PIO or DMA, the HBA will set
7613 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7614 * non-fatal error when the HBA can continues. Sometimes, it will cause
7615 * a fatal error and need the software to do something.
7616 *
7617 * For an overflow on data write, setting PxIS.OFS is optional for both
7618 * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7619 * software to clean up from this serious error.
7620 *
7621 * PxIS.INFS - Interface Non-Fatal Error
7622 *
7623 * This interrupt event indicates that the HBA encountered an error on
7624 * the Serial ATA interface but was able to continue operation. The kind
7625 * of error usually occurred during a non-Data FIS, and under this condition
7626 * the FIS will be re-transmitted by HBA automatically.
7627 *
7628 * When the FMA is implemented, there should be a stat structure to
7629 * record how many every kind of error happens.
7630 */
7631 static int
7632 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7633 uint8_t port, uint32_t intr_status)
7634 {
7635 uint32_t port_serror;
7636 #if AHCI_DEBUG
7637 uint32_t port_cmd_status;
7638 uint32_t port_cmd_issue;
7639 uint32_t port_sactive;
7640 int current_slot;
7641 uint32_t current_tags;
7642 sata_pkt_t *satapkt;
7643 ahci_cmd_header_t *cmd_header;
7644 uint32_t cmd_dmacount;
7645 #endif
7646
7647 mutex_enter(&ahci_portp->ahciport_mutex);
7648
7649 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7650 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7651
7652 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7653 "ahci_intr_non_fatal_error: port %d, "
7654 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7655
7656 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7657
7658 if (intr_status & AHCI_INTR_STATUS_UFS) {
7659 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7660 "ahci port %d has unknown FIS error", port);
7661
7662 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7663 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7664 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7665 SERROR_FIS_TYPE);
7666 }
7667
7668 #if AHCI_DEBUG
7669 if (intr_status & AHCI_INTR_STATUS_IPMS) {
7670 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7671 "has Incorrect Port Multiplier Status error", port);
7672 }
7673
7674 if (intr_status & AHCI_INTR_STATUS_OFS) {
7675 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7676 "ahci port %d has overflow error", port);
7677 }
7678
7679 if (intr_status & AHCI_INTR_STATUS_INFS) {
7680 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7681 "ahci port %d has interface non fatal error", port);
7682 }
7683
7684 /*
7685 * Record the error occurred command's slot.
7686 */
7687 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7688 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7689 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7690 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7691
7692 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7693 AHCI_CMD_STATUS_CCS_SHIFT;
7694
7695 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7696 satapkt = ahci_portp->ahciport_err_retri_pkt;
7697 ASSERT(satapkt != NULL);
7698 ASSERT(current_slot == 0);
7699 } else {
7700 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7701 }
7702
7703 if (satapkt != NULL) {
7704 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7705 "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7706 "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7707 satapkt->satapkt_cmd.satacmd_cmd_reg);
7708
7709 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7710 "ahci_intr_non_fatal_error: port %d, "
7711 "satapkt 0x%p is being processed when error occurs",
7712 port, (void *)satapkt);
7713
7714 /*
7715 * PRD Byte Count field of command header is not
7716 * required to reflect the total number of bytes
7717 * transferred when an overflow occurs, so here
7718 * just log the value.
7719 */
7720 cmd_dmacount =
7721 ahci_portp->ahciport_prd_bytecounts[current_slot];
7722 if (cmd_dmacount) {
7723 cmd_header = &ahci_portp->
7724 ahciport_cmd_list[current_slot];
7725 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7726 "ahci_intr_non_fatal_error: port %d, "
7727 "PRD Byte Count = 0x%x, "
7728 "ahciport_prd_bytecounts = 0x%x", port,
7729 cmd_header->ahcich_prd_byte_count,
7730 cmd_dmacount);
7731 }
7732 }
7733 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7734 /*
7735 * For queued command, list those command which have already
7736 * been transmitted to the device and still not completed.
7737 */
7738 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7739 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7740
7741 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7742 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7743
7744 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7745 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7746 "port_sactive = 0x%x port_cmd_issue = 0x%x",
7747 ahci_portp->ahciport_pending_ncq_tags,
7748 port_sactive, port_cmd_issue);
7749
7750 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7751 port_sactive & ~port_cmd_issue &
7752 AHCI_NCQ_SLOT_MASK(ahci_portp);
7753
7754 while (current_tags) {
7755 current_slot = ddi_ffs(current_tags) - 1;
7756 if (current_slot == -1) {
7757 goto out;
7758 }
7759
7760 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7761 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7762 ahci_ctlp, "ahci_intr_non_fatal_error: "
7763 "port %d, satapkt 0x%p is outstanding when "
7764 "error occurs", port, (void *)satapkt);
7765
7766 CLEAR_BIT(current_tags, current_slot);
7767 }
7768 }
7769 out:
7770 #endif
7771 mutex_exit(&ahci_portp->ahciport_mutex);
7772
7773 return (AHCI_SUCCESS);
7774 }
7775
7776 /*
7777 * According to the AHCI spec, the error types include system memory
7778 * errors, interface errors, port multiplier errors, device errors,
7779 * command list overflow, command list underflow, native command
7780 * queuing tag errors and pio data transfer errors.
7781 *
7782 * System memory errors such as target abort, master abort, and parity
7783 * may cause the host to stop, and they are serious errors and needed
7784 * to be recovered with software intervention. When system software
7785 * has given a pointer to the HBA that doesn't exist in physical memory,
7786 * a master/target abort error occurs, and PxIS.HBFS will be set. A
7787 * data error such as CRC or parity occurs, the HBA aborts the transfer
7788 * (if necessary) and PxIS.HBDS will be set.
7789 *
7790 * Interface errors are errors that occur due to electrical issues on
7791 * the interface, or protocol miscommunication between the device and
7792 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7793 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7794 * causes PxIS.IFS/PxIS.INFS to be set are
7795 * 1. in PxSERR.ERR, P bit is set to '1'
7796 * 2. in PxSERR.DIAG, C or H bit is set to '1'
7797 * 3. PhyRdy drop unexpectly, N bit is set to '1'
7798 * If the error occurred during a non-data FIS, the FIS must be
7799 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7800 * the error occurred during a data FIS, the transfer will stop, so
7801 * the error is fatal and PxIS.IFS is set.
7802 *
7803 * When a FIS arrives that updates the taskfile, the HBA checks to see
7804 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7805 * stops processing any more commands.
7806 *
7807 * Command list overflow is defined as software building a command table
7808 * that has fewer total bytes than the transaction given to the device.
7809 * On device writes, the HBA will run out of data, and on reads, there
7810 * will be no room to put the data. For an overflow on data read, either
7811 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7812 * For an overflow on data write, setting PxIS.OFS is optional for both
7813 * DMA and PIO, and a COMRESET is required by software to clean up from
7814 * this serious error.
7815 *
7816 * Command list underflow is defined as software building a command
7817 * table that has more total bytes than the transaction given to the
7818 * device. For data writes, both PIO and DMA, the device will detect
7819 * an error and end the transfer. And these errors are most likely going
7820 * to be fatal errors that will cause the port to be restarted. For
7821 * data reads, the HBA updates its PRD byte count, and may be
7822 * able to continue normally, but is not required to. And The HBA is
7823 * not required to detect underflow conditions for native command
7824 * queuing command.
7825 *
7826 * The HBA does not actively check incoming DMA Setup FISes to ensure
7827 * that the PxSACT register bit for that slot is set. Existing error
7828 * mechanisms, such as host bus failure, or bad protocol, are used to
7829 * recover from this case.
7830 *
7831 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7832 * DATA FIS must be an integral number of Dwords. If the HBA receives
7833 * a request which is not an integral number of Dwords, the HBA
7834 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7835 * software restarts the port. And the HBA ensures that the size
7836 * of the DATA FIS received during a PIO command matches the size in
7837 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7838 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7839 * stop running until software restarts the port.
7840 */
7841 /*
7842 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7843 *
7844 * PxIS.IFS indicates that the hba encountered an error on the serial ata
7845 * interface which caused the transfer to stop.
7846 *
7847 * PxIS.HBDS indicates that the hba encountered a data error
7848 * (uncorrectable ecc/parity) when reading from or writing to system memory.
7849 *
7850 * PxIS.HBFS indicates that the hba encountered a host bus error that it
7851 * cannot recover from, such as a bad software pointer.
7852 *
7853 * PxIS.TFES is set whenever the status register is updated by the device
7854 * and the error bit (bit 0) is set.
7855 */
7856 static int
7857 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7858 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7859 {
7860 uint32_t port_cmd_status;
7861 uint32_t port_serror;
7862 uint32_t task_file_status;
7863 int failed_slot;
7864 sata_pkt_t *spkt = NULL;
7865 uint8_t err_byte;
7866 ahci_event_arg_t *args;
7867 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7868 uint32_t failed_tags = 0;
7869 int task_fail_flag = 0, task_abort_flag = 0;
7870 uint32_t slot_status;
7871
7872 mutex_enter(&ahci_portp->ahciport_mutex);
7873
7874 /*
7875 * ahci_intr_phyrdy_change() may have rendered it to
7876 * SATA_DTYPE_NONE.
7877 */
7878 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7879 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7880 "ahci_intr_fatal_error: port %d no device attached, "
7881 "and just return without doing anything", port);
7882 goto out0;
7883 }
7884
7885 if (intr_status & AHCI_INTR_STATUS_TFES) {
7886 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7887 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7888 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7889 "ahci_intr_fatal_error: port %d "
7890 "task_file_status = 0x%x", port, task_file_status);
7891 task_fail_flag = 1;
7892
7893 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7894 >> AHCI_TFD_ERR_SHIFT;
7895 if (err_byte == SATA_ERROR_ABORT)
7896 task_abort_flag = 1;
7897 }
7898
7899 /*
7900 * Here we just log the fatal error info in interrupt context.
7901 * Misc recovery processing will be handled in task queue.
7902 */
7903 if (task_fail_flag == 1) {
7904 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7905 /*
7906 * Read PxCMD.CCS to determine the slot that the HBA
7907 * was processing when the error occurred.
7908 */
7909 port_cmd_status = ddi_get32(
7910 ahci_ctlp->ahcictl_ahci_acc_handle,
7911 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7912 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7913 AHCI_CMD_STATUS_CCS_SHIFT;
7914 failed_tags = 0x1 << failed_slot;
7915
7916 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7917 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7918 "ahci_intr_fatal_error: spkt 0x%p is being "
7919 "processed when fatal error occurred for port %d",
7920 spkt, port);
7921
7922 /*
7923 * Won't emit the error message if it is an IDENTIFY
7924 * DEVICE command sent to an ATAPI device.
7925 */
7926 if ((spkt != NULL) &&
7927 (spkt->satapkt_cmd.satacmd_cmd_reg ==
7928 SATAC_ID_DEVICE) &&
7929 (task_abort_flag == 1))
7930 goto out1;
7931
7932 /*
7933 * Won't emit the error message if it is an ATAPI PACKET
7934 * command
7935 */
7936 if ((spkt != NULL) &&
7937 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7938 goto out1;
7939
7940 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7941 slot_status = ddi_get32(
7942 ahci_ctlp->ahcictl_ahci_acc_handle,
7943 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7944 failed_tags = slot_status &
7945 AHCI_NCQ_SLOT_MASK(ahci_portp);
7946 }
7947 }
7948
7949 /* print the fatal error type */
7950 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7951 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7952
7953 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7954 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7955
7956 /* print PxSERR related error message */
7957 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7958
7959 /* print task file register value */
7960 if (task_fail_flag == 1) {
7961 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7962 "= 0x%x", instance, port, task_file_status);
7963 if (task_abort_flag == 1) {
7964 cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7965 "port %d are aborted", instance, port);
7966 ahci_dump_commands(ahci_ctlp, port, failed_tags);
7967 }
7968 }
7969
7970 out1:
7971 /* Prepare the argument for the taskq */
7972 args = ahci_portp->ahciport_event_args;
7973 args->ahciea_ctlp = (void *)ahci_ctlp;
7974 args->ahciea_portp = (void *)ahci_portp;
7975 args->ahciea_event = intr_status;
7976 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7977
7978 /* Start the taskq to handle error recovery */
7979 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7980 ahci_events_handler,
7981 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7982 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7983 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7984 "port %d failed", instance, port);
7985 }
7986 out0:
7987 mutex_exit(&ahci_portp->ahciport_mutex);
7988
7989 return (AHCI_SUCCESS);
7990 }
7991
7992 /*
7993 * Hot Plug Operation for platforms that support Cold Presence Detect.
7994 *
7995 * When set, a device status has changed as detected by the cold presence
7996 * detect logic. This bit can either be set due to a non-connected port
7997 * receiving a device, or a connected port having its device removed.
7998 * This bit is only valid if the port supports cold presence detect as
7999 * indicated by PxCMD.CPD set to '1'.
8000 *
8001 * At the moment, this interrupt is not needed and disabled and we just
8002 * log the debug message.
8003 */
8004 static int
8005 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
8006 ahci_port_t *ahci_portp, uint8_t port)
8007 {
8008 uint32_t port_cmd_status;
8009 sata_device_t sdevice;
8010
8011 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8012 "ahci_intr_cold_port_detect enter, port %d", port);
8013
8014 mutex_enter(&ahci_portp->ahciport_mutex);
8015
8016 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8017 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8018 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
8019 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8020 "port %d does not support cold presence detect, so "
8021 "we just ignore this interrupt", port);
8022 mutex_exit(&ahci_portp->ahciport_mutex);
8023 return (AHCI_SUCCESS);
8024 }
8025
8026 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8027 "port %d device status has changed", port);
8028
8029 bzero((void *)&sdevice, sizeof (sata_device_t));
8030 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
8031 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
8032 sdevice.satadev_addr.pmport = 0;
8033 sdevice.satadev_state = SATA_PSTATE_PWRON;
8034
8035 if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
8036 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8037 "port %d: a device is hot plugged", port);
8038 mutex_exit(&ahci_portp->ahciport_mutex);
8039 sata_hba_event_notify(
8040 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8041 &sdevice,
8042 SATA_EVNT_DEVICE_ATTACHED);
8043 mutex_enter(&ahci_portp->ahciport_mutex);
8044
8045 } else {
8046 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8047 "port %d: a device is hot unplugged", port);
8048 mutex_exit(&ahci_portp->ahciport_mutex);
8049 sata_hba_event_notify(
8050 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8051 &sdevice,
8052 SATA_EVNT_DEVICE_DETACHED);
8053 mutex_enter(&ahci_portp->ahciport_mutex);
8054 }
8055
8056 mutex_exit(&ahci_portp->ahciport_mutex);
8057
8058 return (AHCI_SUCCESS);
8059 }
8060
8061 /*
8062 * Enable the interrupts for a particular port.
8063 */
8064 static void
8065 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8066 {
8067 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8068
8069 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8070 "ahci_enable_port_intrs enter, port %d", port);
8071
8072 /*
8073 * Clear port interrupt status before enabling interrupt
8074 */
8075 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8076 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
8077 AHCI_PORT_INTR_MASK);
8078
8079 /*
8080 * Clear the pending bit from IS.IPS
8081 */
8082 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8083 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
8084
8085 /*
8086 * Enable the following interrupts:
8087 * Device to Host Register FIS Interrupt (DHRS)
8088 * PIO Setup FIS Interrupt (PSS)
8089 * Set Device Bits Interrupt (SDBS)
8090 * Unknown FIS Interrupt (UFS)
8091 * Port Connect Change Status (PCS)
8092 * PhyRdy Change Status (PRCS)
8093 * Overflow Status (OFS)
8094 * Interface Non-fatal Error Status (INFS)
8095 * Interface Fatal Error Status (IFS)
8096 * Host Bus Data Error Status (HBDS)
8097 * Host Bus Fatal Error Status (HBFS)
8098 * Task File Error Status (TFES)
8099 */
8100 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8101 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8102 (AHCI_INTR_STATUS_DHRS |
8103 AHCI_INTR_STATUS_PSS |
8104 AHCI_INTR_STATUS_SDBS |
8105 AHCI_INTR_STATUS_UFS |
8106 AHCI_INTR_STATUS_DPS |
8107 AHCI_INTR_STATUS_PCS |
8108 AHCI_INTR_STATUS_PRCS |
8109 AHCI_INTR_STATUS_OFS |
8110 AHCI_INTR_STATUS_INFS |
8111 AHCI_INTR_STATUS_IFS |
8112 AHCI_INTR_STATUS_HBDS |
8113 AHCI_INTR_STATUS_HBFS |
8114 AHCI_INTR_STATUS_TFES));
8115 }
8116
8117 /*
8118 * Enable interrupts for all the ports.
8119 */
8120 static void
8121 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8122 {
8123 uint32_t ghc_control;
8124
8125 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8126
8127 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8128
8129 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8130 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8131
8132 ghc_control |= AHCI_HBA_GHC_IE;
8133
8134 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8135 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8136 }
8137
8138 /*
8139 * Disable interrupts for a particular port.
8140 */
8141 static void
8142 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8143 {
8144 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8145 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8146
8147 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8148 "ahci_disable_port_intrs enter, port %d", port);
8149
8150 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8151 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8152 }
8153
8154 /*
8155 * Disable interrupts for the whole HBA.
8156 *
8157 * The global bit is cleared, then all interrupt sources from all
8158 * ports are disabled.
8159 */
8160 static void
8161 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8162 {
8163 uint32_t ghc_control;
8164
8165 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8166 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8167
8168 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8169 NULL);
8170
8171 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8172 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8173
8174 ghc_control &= ~AHCI_HBA_GHC_IE;
8175
8176 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8177 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8178 }
8179
8180 /*
8181 * Handle FIXED or MSI interrupts.
8182 */
8183 /*
8184 * According to AHCI spec, the HBA may support several interrupt modes:
8185 * * pin based interrupts (FIXED)
8186 * * single MSI message interrupts
8187 * * multiple MSI based message interrupts
8188 *
8189 * For pin based interrupts, the software interrupt handler need to check IS
8190 * register to find out which port has pending interrupts. And then check
8191 * PxIS register to find out which interrupt events happened on that port.
8192 *
8193 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8194 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8195 * in that software interrupt handler need to check IS register to determine
8196 * which port triggered the interrupts since it uses a single message for all
8197 * port interrupts.
8198 *
8199 * HBA may optionally support multiple MSI message for better performance. In
8200 * this mode, each port may have its own interrupt message, and thus generation
8201 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8202 * represents a power-of-2 wrapper on the number of implemented ports, and
8203 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8204 * maximum number of assigned interrupts. When the number of MSI messages
8205 * allocated is less than the number requested, then hardware may have two
8206 * implementation behaviors:
8207 * * assign each ports its own interrupt and then force all additional
8208 * ports to share the last interrupt message, and this condition is
8209 * indicated by clearing GHC.MRSM to '0'
8210 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8211 * When multiple-message MSI is enabled, hardware will still set IS register
8212 * as single message case. And this IS register may be used by software when
8213 * fewer than the requested number of messages is granted in order to determine
8214 * which port had the interrupt.
8215 *
8216 * Note: The current ahci driver only supports the first two interrupt modes:
8217 * pin based interrupts and single MSI message interrupts, and the reason
8218 * is indicated in below code.
8219 */
8220 static int
8221 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8222 {
8223 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8224 int count, avail, actual;
8225 int i, rc;
8226
8227 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8228 "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8229
8230 /* get number of interrupts. */
8231 rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8232 if ((rc != DDI_SUCCESS) || (count == 0)) {
8233 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8234 "ddi_intr_get_nintrs() failed, "
8235 "rc %d count %d\n", rc, count);
8236 return (DDI_FAILURE);
8237 }
8238
8239 /* get number of available interrupts. */
8240 rc = ddi_intr_get_navail(dip, intr_type, &avail);
8241 if ((rc != DDI_SUCCESS) || (avail == 0)) {
8242 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8243 "ddi_intr_get_navail() failed, "
8244 "rc %d avail %d\n", rc, avail);
8245 return (DDI_FAILURE);
8246 }
8247
8248 #if AHCI_DEBUG
8249 if (avail < count) {
8250 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8251 "ddi_intr_get_nintrs returned %d, navail() returned %d",
8252 count, avail);
8253 }
8254 #endif
8255
8256 /*
8257 * Note: So far Solaris restricts the maximum number of messages for
8258 * x86 to 2, that is avail is 2, so here we set the count with 1 to
8259 * force the driver to use single MSI message interrupt. In future if
8260 * Solaris remove the restriction, then we need to delete the below
8261 * code and try to use multiple interrupt routine to gain better
8262 * performance.
8263 */
8264 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8265 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8266 "force to use one interrupt routine though the "
8267 "HBA supports %d interrupt", count);
8268 count = 1;
8269 }
8270
8271 /* Allocate an array of interrupt handles. */
8272 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8273 ahci_ctlp->ahcictl_intr_htable =
8274 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8275
8276 /* call ddi_intr_alloc(). */
8277 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8278 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8279
8280 if ((rc != DDI_SUCCESS) || (actual == 0)) {
8281 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8282 "ddi_intr_alloc() failed, rc %d count %d actual %d "
8283 "avail %d\n", rc, count, actual, avail);
8284 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8285 ahci_ctlp->ahcictl_intr_size);
8286 return (DDI_FAILURE);
8287 }
8288
8289 /* use interrupt count returned */
8290 #if AHCI_DEBUG
8291 if (actual < count) {
8292 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8293 "Requested: %d, Received: %d", count, actual);
8294 }
8295 #endif
8296
8297 ahci_ctlp->ahcictl_intr_cnt = actual;
8298
8299 /*
8300 * Get priority for first, assume remaining are all the same.
8301 */
8302 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8303 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8304 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8305 "ddi_intr_get_pri() failed", NULL);
8306
8307 /* Free already allocated intr. */
8308 for (i = 0; i < actual; i++) {
8309 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8310 }
8311
8312 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8313 ahci_ctlp->ahcictl_intr_size);
8314 return (DDI_FAILURE);
8315 }
8316
8317 /* Test for high level interrupt. */
8318 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8319 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8320 "ahci_add_intrs: Hi level intr not supported", NULL);
8321
8322 /* Free already allocated intr. */
8323 for (i = 0; i < actual; i++) {
8324 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8325 }
8326
8327 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8328 sizeof (ddi_intr_handle_t));
8329
8330 return (DDI_FAILURE);
8331 }
8332
8333 /* Call ddi_intr_add_handler(). */
8334 for (i = 0; i < actual; i++) {
8335 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8336 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8337 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8338 "ddi_intr_add_handler() failed", NULL);
8339
8340 /* Free already allocated intr. */
8341 for (i = 0; i < actual; i++) {
8342 (void) ddi_intr_free(
8343 ahci_ctlp->ahcictl_intr_htable[i]);
8344 }
8345
8346 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8347 ahci_ctlp->ahcictl_intr_size);
8348 return (DDI_FAILURE);
8349 }
8350 }
8351
8352 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8353 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8354 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8355 "ddi_intr_get_cap() failed", NULL);
8356
8357 /* Free already allocated intr. */
8358 for (i = 0; i < actual; i++) {
8359 (void) ddi_intr_free(
8360 ahci_ctlp->ahcictl_intr_htable[i]);
8361 }
8362
8363 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8364 ahci_ctlp->ahcictl_intr_size);
8365 return (DDI_FAILURE);
8366 }
8367
8368 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8369 /* Call ddi_intr_block_enable() for MSI. */
8370 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8371 ahci_ctlp->ahcictl_intr_cnt);
8372 } else {
8373 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8374 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8375 (void) ddi_intr_enable(
8376 ahci_ctlp->ahcictl_intr_htable[i]);
8377 }
8378 }
8379
8380 return (DDI_SUCCESS);
8381 }
8382
8383 /*
8384 * Removes the registered interrupts irrespective of whether they
8385 * were legacy or MSI.
8386 *
8387 * NOTE: The controller interrupts must be disabled before calling
8388 * this routine.
8389 */
8390 static void
8391 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8392 {
8393 int x;
8394
8395 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8396
8397 /* Disable all interrupts. */
8398 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8399 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8400 /* Call ddi_intr_block_disable(). */
8401 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8402 ahci_ctlp->ahcictl_intr_cnt);
8403 } else {
8404 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8405 (void) ddi_intr_disable(
8406 ahci_ctlp->ahcictl_intr_htable[x]);
8407 }
8408 }
8409
8410 /* Call ddi_intr_remove_handler(). */
8411 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8412 (void) ddi_intr_remove_handler(
8413 ahci_ctlp->ahcictl_intr_htable[x]);
8414 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8415 }
8416
8417 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8418 }
8419
8420 /*
8421 * This routine tries to put port into P:NotRunning state by clearing
8422 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8423 * and PxCMD.CR to '0'.
8424 */
8425 static int
8426 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8427 ahci_port_t *ahci_portp, uint8_t port)
8428 {
8429 uint32_t port_cmd_status;
8430 int loop_count;
8431
8432 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8433 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8434
8435 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8436 "ahci_put_port_into_notrunning_state enter: port %d", port);
8437
8438 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8439 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8440
8441 port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8442 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8443 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8444
8445 /* Wait until PxCMD.CR is cleared */
8446 loop_count = 0;
8447 do {
8448 port_cmd_status =
8449 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8450 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8451
8452 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8453 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8454 "clearing port %d CMD.CR timeout, "
8455 "port_cmd_status = 0x%x", port,
8456 port_cmd_status);
8457 /*
8458 * We are effectively timing out after 0.5 sec.
8459 * This value is specified in AHCI spec.
8460 */
8461 break;
8462 }
8463
8464 /* Wait for 10 millisec */
8465 drv_usecwait(AHCI_10MS_USECS);
8466 } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8467
8468 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8469
8470 if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8471 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8472 "ahci_put_port_into_notrunning_state: failed to clear "
8473 "PxCMD.CR to '0' after loop count: %d, and "
8474 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8475 return (AHCI_FAILURE);
8476 } else {
8477 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8478 "ahci_put_port_into_notrunning_state: succeeded to clear "
8479 "PxCMD.CR to '0' after loop count: %d, and "
8480 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8481 return (AHCI_SUCCESS);
8482 }
8483 }
8484
8485 /*
8486 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8487 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8488 * stable state, then set PxCMD.ST to '1' to start the port directly.
8489 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8490 * COMRESET to the device to put it in an idle state.
8491 *
8492 * The fifth argument returns whether the port reset is involved during
8493 * the process.
8494 *
8495 * The routine will be called under following scenarios:
8496 * + To reset the HBA
8497 * + To abort the packet(s)
8498 * + To reset the port
8499 * + To activate the port
8500 * + Fatal error recovery
8501 * + To abort the timeout packet(s)
8502 *
8503 * NOTES!!! During this procedure, PxSERR register will be cleared, and
8504 * according to the spec, the clearance of three bits will also clear
8505 * three interrupt status bits.
8506 * 1. PxSERR.DIAG.F will clear PxIS.UFS
8507 * 2. PxSERR.DIAG.X will clear PxIS.PCS
8508 * 3. PxSERR.DIAG.N will clear PxIS.PRCS
8509 *
8510 * Among these three interrupt events, the driver needs to take care of
8511 * PxIS.PRCS, which is the hot plug event. When the driver found out
8512 * a device was unplugged, it will call the interrupt handler.
8513 */
8514 static int
8515 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8516 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8517 {
8518 uint32_t port_sstatus;
8519 uint32_t task_file_status;
8520 sata_device_t sdevice;
8521 int rval;
8522 ahci_addr_t addr_port;
8523 ahci_pmult_info_t *pminfo = NULL;
8524 int dev_exists_begin = 0;
8525 int dev_exists_end = 0;
8526 uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8527 int npmport = 0;
8528 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8529
8530 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8531
8532 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8533 "ahci_restart_port_wait_till_ready: port %d enter", port);
8534
8535 AHCI_ADDR_SET_PORT(&addr_port, port);
8536
8537 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8538 dev_exists_begin = 1;
8539
8540 /* First clear PxCMD.ST */
8541 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8542 port);
8543 if (rval != AHCI_SUCCESS)
8544 /*
8545 * If PxCMD.CR does not clear within a reasonable time, it
8546 * may assume the interface is in a hung condition and may
8547 * continue with issuing the port reset.
8548 */
8549 goto reset;
8550
8551 /* Then clear PxSERR */
8552 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8553 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8554 AHCI_SERROR_CLEAR_ALL);
8555
8556 /* Then get PxTFD */
8557 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8558 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8559
8560 /*
8561 * Check whether the device is in a stable status, if yes,
8562 * then start the port directly. However for ahci_tran_reset_dport,
8563 * we may have to perform a port reset.
8564 */
8565 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8566 !(flag & AHCI_PORT_RESET))
8567 goto out;
8568
8569 reset:
8570 /*
8571 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8572 * a COMRESET to the device
8573 */
8574 ahci_disable_port_intrs(ahci_ctlp, port);
8575 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8576 ahci_enable_port_intrs(ahci_ctlp, port);
8577
8578 #ifdef AHCI_DEBUG
8579 if (rval != AHCI_SUCCESS)
8580 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8581 "ahci_restart_port_wait_till_ready: port %d failed",
8582 port);
8583 #endif
8584
8585 if (reset_flag != NULL)
8586 *reset_flag = 1;
8587
8588 /* Indicate to the framework that a reset has happened. */
8589 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8590 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8591 !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8592 /* Set the reset in progress flag */
8593 ahci_portp->ahciport_reset_in_progress = 1;
8594
8595 bzero((void *)&sdevice, sizeof (sata_device_t));
8596 sdevice.satadev_addr.cport =
8597 ahci_ctlp->ahcictl_port_to_cport[port];
8598 sdevice.satadev_addr.pmport = 0;
8599 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8600
8601 sdevice.satadev_state = SATA_DSTATE_RESET |
8602 SATA_DSTATE_PWR_ACTIVE;
8603 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8604 mutex_exit(&ahci_portp->ahciport_mutex);
8605 sata_hba_event_notify(
8606 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8607 &sdevice,
8608 SATA_EVNT_DEVICE_RESET);
8609 mutex_enter(&ahci_portp->ahciport_mutex);
8610 }
8611
8612 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8613 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8614 } else {
8615 ahci_portp->ahciport_reset_in_progress = 0;
8616 }
8617
8618 out:
8619 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8620
8621 /* SStatus tells the presence of device. */
8622 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8623 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8624
8625 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8626 dev_exists_end = 1;
8627 }
8628
8629 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8630 return (rval);
8631
8632 /* Check whether a hot plug event happened */
8633 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8634 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8635 "ahci_restart_port_wait_till_ready: port %d "
8636 "device is removed", port);
8637 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8638 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8639 "ahci_restart_port_wait_till_ready: port %d "
8640 "AHCI_PORT_FLAG_NODEV flag is set", port);
8641 mutex_exit(&ahci_portp->ahciport_mutex);
8642 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8643 mutex_enter(&ahci_portp->ahciport_mutex);
8644
8645 return (rval);
8646 }
8647
8648
8649 /* 0/1 -> 1 : device may change */
8650 /*
8651 * May be called by ahci_fatal_error_recovery_handler, so
8652 * don't issue software if the previous device is ATAPI.
8653 */
8654 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8655 return (rval);
8656
8657 /*
8658 * The COMRESET will make port multiplier enter legacy mode.
8659 * Issue a software reset to make it work again.
8660 */
8661 ahci_disable_port_intrs(ahci_ctlp, port);
8662 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8663 ahci_enable_port_intrs(ahci_ctlp, port);
8664
8665 /*
8666 * Following codes are specific for the port multiplier
8667 */
8668 if (previous_dev_type != SATA_DTYPE_PMULT &&
8669 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8670 /* in case previous_dev_type is corrupt */
8671 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8672 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8673 return (rval);
8674 }
8675
8676 /* Device change: PMult -> Non-PMult */
8677 if (previous_dev_type == SATA_DTYPE_PMULT &&
8678 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8679 /*
8680 * This might happen because
8681 * 1. Software reset failed. Port multiplier is not correctly
8682 * enumerated.
8683 * 2. Another non-port-multiplier device is attached. Perhaps
8684 * the port multiplier was replaced by another device by
8685 * whatever reason, but AHCI driver missed hot-plug event.
8686 *
8687 * Now that the port has been initialized, we just need to
8688 * update the port structure according new device, then report
8689 * and wait SATA framework to probe new device.
8690 */
8691
8692 /* Force to release pmult resource */
8693 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8694 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8695
8696 bzero((void *)&sdevice, sizeof (sata_device_t));
8697 sdevice.satadev_addr.cport =
8698 ahci_ctlp->ahcictl_port_to_cport[port];
8699 sdevice.satadev_addr.pmport = 0;
8700 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8701
8702 sdevice.satadev_state = SATA_DSTATE_RESET |
8703 SATA_DSTATE_PWR_ACTIVE;
8704
8705 mutex_exit(&ahci_portp->ahciport_mutex);
8706 sata_hba_event_notify(
8707 ahci_ctlp->ahcictl_dip,
8708 &sdevice,
8709 SATA_EVNT_DEVICE_RESET);
8710 mutex_enter(&ahci_portp->ahciport_mutex);
8711
8712 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8713 "Port multiplier is [Gone] at port %d ", port);
8714 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8715 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8716
8717 return (AHCI_SUCCESS);
8718 }
8719
8720 /* Device change: Non-PMult -> PMult */
8721 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8722
8723 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8724 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8725
8726 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8727 }
8728 pminfo = ahci_portp->ahciport_pmult_info;
8729 ASSERT(pminfo != NULL);
8730
8731 /* Device (may) change: PMult -> PMult */
8732 /*
8733 * First initialize port multiplier. Set state to READY and wait for
8734 * probe entry point to initialize it
8735 */
8736 ahci_portp->ahciport_port_state = SATA_STATE_READY;
8737
8738 /*
8739 * It's a little complicated while target is a port multiplier. we
8740 * need to COMRESET all pmports behind that PMult otherwise those
8741 * sub-links between the PMult and the sub-devices will be in an
8742 * inactive state (indicated by PSCR0/PxSSTS) and the following access
8743 * to those sub-devices will be rejected by Link-Fatal-Error.
8744 */
8745 /*
8746 * The PxSNTF will be set soon after the pmult is plugged. While the
8747 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8748 * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8749 * will initialized it.
8750 */
8751 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8752 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8753
8754 /* Report reset event. */
8755 ahci_portp->ahciport_reset_in_progress = 1;
8756
8757 bzero((void *)&sdevice, sizeof (sata_device_t));
8758 sdevice.satadev_addr.cport = cport;
8759 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8760 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8761 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8762 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8763 SATA_EVNT_DEVICE_RESET);
8764
8765 return (rval);
8766 }
8767
8768 /*
8769 * This routine may be called under four scenarios:
8770 * a) do the recovery from fatal error
8771 * b) or we need to timeout some commands
8772 * c) or we need to abort some commands
8773 * d) or we need reset device/port/controller
8774 *
8775 * In all these scenarios, we need to send any pending unfinished
8776 * commands up to sata framework.
8777 */
8778 static void
8779 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8780 ahci_port_t *ahci_portp,
8781 uint32_t slot_status,
8782 uint32_t failed_tags,
8783 uint32_t timeout_tags,
8784 uint32_t aborted_tags,
8785 uint32_t reset_tags)
8786 {
8787 uint32_t finished_tags = 0;
8788 uint32_t unfinished_tags = 0;
8789 int tmp_slot;
8790 sata_pkt_t *satapkt;
8791 int ncq_cmd_in_progress = 0;
8792 int err_retri_cmd_in_progress = 0;
8793 int rdwr_pmult_cmd_in_progress = 0;
8794
8795 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8796
8797 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8798 "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8799 ahci_portp->ahciport_port_num, slot_status);
8800
8801 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8802 "ahci_mop_commands: failed_tags: 0x%x, "
8803 "timeout_tags: 0x%x aborted_tags: 0x%x, "
8804 "reset_tags: 0x%x", failed_tags,
8805 timeout_tags, aborted_tags, reset_tags);
8806
8807 #ifdef AHCI_DEBUG
8808 if (ahci_debug_flags & AHCIDBG_ERRS) {
8809 int i;
8810 char msg_buf[200] = {0, };
8811 for (i = 0x1f; i >= 0; i--) {
8812 if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8813 msg_buf[i] = 'X';
8814 else
8815 msg_buf[i] = '.';
8816 }
8817 msg_buf[0x20] = '\0';
8818 cmn_err(CE_NOTE, "port[%d] slots: %s",
8819 ahci_portp->ahciport_port_num, msg_buf);
8820 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8821 (void *)ahci_portp->ahciport_err_retri_pkt,
8822 (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8823 }
8824 #endif
8825
8826 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8827 finished_tags = ahci_portp->ahciport_pending_tags &
8828 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8829
8830 unfinished_tags = slot_status &
8831 AHCI_SLOT_MASK(ahci_ctlp) &
8832 ~failed_tags &
8833 ~aborted_tags &
8834 ~reset_tags &
8835 ~timeout_tags;
8836 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8837 ncq_cmd_in_progress = 1;
8838 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8839 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8840
8841 unfinished_tags = slot_status &
8842 AHCI_NCQ_SLOT_MASK(ahci_portp) &
8843 ~failed_tags &
8844 ~aborted_tags &
8845 ~reset_tags &
8846 ~timeout_tags;
8847 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8848
8849 /*
8850 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8851 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8852 * complete successfully due to one of the following three
8853 * conditions:
8854 *
8855 * 1. Fatal error - failed_tags includes its slot
8856 * 2. Timed out - timeout_tags includes its slot
8857 * 3. Aborted when hot unplug - aborted_tags includes its
8858 * slot
8859 *
8860 * Please note that the command is always sent down in Slot 0
8861 */
8862 err_retri_cmd_in_progress = 1;
8863 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8864 "ahci_mop_commands is called for port %d while "
8865 "REQUEST SENSE or READ LOG EXT for error retrieval "
8866 "is being executed slot_status = 0x%x",
8867 ahci_portp->ahciport_port_num, slot_status);
8868 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8869 ASSERT(slot_status == 0x1);
8870 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8871 rdwr_pmult_cmd_in_progress = 1;
8872 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8873 "ahci_mop_commands is called for port %d while "
8874 "READ/WRITE PORTMULT command is being executed",
8875 ahci_portp->ahciport_port_num);
8876
8877 ASSERT(slot_status == 0x1);
8878 }
8879
8880 #ifdef AHCI_DEBUG
8881 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8882 "ahci_mop_commands: finished_tags: 0x%x, "
8883 "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8884 #endif
8885
8886 /* Send up finished packets with SATA_PKT_COMPLETED */
8887 while (finished_tags) {
8888 tmp_slot = ddi_ffs(finished_tags) - 1;
8889 if (tmp_slot == -1) {
8890 break;
8891 }
8892
8893 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8894 ASSERT(satapkt != NULL);
8895
8896 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8897 "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8898 (void *)satapkt);
8899
8900 /*
8901 * Cannot fetch the return register content since the port
8902 * was restarted, so the corresponding tag will be set to
8903 * aborted tags.
8904 */
8905 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8906 CLEAR_BIT(finished_tags, tmp_slot);
8907 aborted_tags |= tmp_slot;
8908 continue;
8909 }
8910
8911 if (ncq_cmd_in_progress)
8912 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8913 tmp_slot);
8914 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8915 CLEAR_BIT(finished_tags, tmp_slot);
8916 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8917
8918 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8919 }
8920
8921 /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8922 while (failed_tags) {
8923 if (err_retri_cmd_in_progress) {
8924 satapkt = ahci_portp->ahciport_err_retri_pkt;
8925 ASSERT(satapkt != NULL);
8926 ASSERT(failed_tags == 0x1);
8927
8928 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8929 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8930 (void *)satapkt);
8931 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8932 break;
8933 }
8934 if (rdwr_pmult_cmd_in_progress) {
8935 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8936 ASSERT(satapkt != NULL);
8937 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8938 "ahci_mop_commands: sending up "
8939 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8940 (void *)satapkt);
8941 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8942 break;
8943 }
8944
8945 tmp_slot = ddi_ffs(failed_tags) - 1;
8946 if (tmp_slot == -1) {
8947 break;
8948 }
8949
8950 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8951 ASSERT(satapkt != NULL);
8952
8953 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8954 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8955 (void *)satapkt);
8956
8957 if (ncq_cmd_in_progress)
8958 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8959 tmp_slot);
8960 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8961 CLEAR_BIT(failed_tags, tmp_slot);
8962 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8963
8964 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8965 }
8966
8967 /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8968 while (timeout_tags) {
8969 if (err_retri_cmd_in_progress) {
8970 satapkt = ahci_portp->ahciport_err_retri_pkt;
8971 ASSERT(satapkt != NULL);
8972 ASSERT(timeout_tags == 0x1);
8973
8974 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8975 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8976 (void *)satapkt);
8977 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8978 break;
8979 }
8980 if (rdwr_pmult_cmd_in_progress) {
8981 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8982 ASSERT(satapkt != NULL);
8983 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8984 "ahci_mop_commands: sending up "
8985 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8986 (void *)satapkt);
8987 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8988 break;
8989 }
8990
8991 tmp_slot = ddi_ffs(timeout_tags) - 1;
8992 if (tmp_slot == -1) {
8993 break;
8994 }
8995
8996 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8997 ASSERT(satapkt != NULL);
8998
8999 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9000 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
9001 (void *)satapkt);
9002
9003 if (ncq_cmd_in_progress)
9004 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9005 tmp_slot);
9006 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9007 CLEAR_BIT(timeout_tags, tmp_slot);
9008 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9009
9010 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
9011 }
9012
9013 /* Send up aborted packets with SATA_PKT_ABORTED */
9014 while (aborted_tags) {
9015 if (err_retri_cmd_in_progress) {
9016 satapkt = ahci_portp->ahciport_err_retri_pkt;
9017 ASSERT(satapkt != NULL);
9018 ASSERT(aborted_tags == 0x1);
9019
9020 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9021 "sending up pkt 0x%p with SATA_PKT_ABORTED",
9022 (void *)satapkt);
9023 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9024 break;
9025 }
9026 if (rdwr_pmult_cmd_in_progress) {
9027 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9028 ASSERT(satapkt != NULL);
9029 ASSERT(aborted_tags == 0x1);
9030 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9031 "ahci_mop_commands: sending up "
9032 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
9033 (void *)satapkt);
9034 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9035 break;
9036 }
9037
9038 tmp_slot = ddi_ffs(aborted_tags) - 1;
9039 if (tmp_slot == -1) {
9040 break;
9041 }
9042
9043 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9044 ASSERT(satapkt != NULL);
9045
9046 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9047 "sending up pkt 0x%p with SATA_PKT_ABORTED",
9048 (void *)satapkt);
9049
9050 if (ncq_cmd_in_progress)
9051 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9052 tmp_slot);
9053 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9054 CLEAR_BIT(aborted_tags, tmp_slot);
9055 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9056
9057 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9058 }
9059
9060 /* Send up reset packets with SATA_PKT_RESET. */
9061 while (reset_tags) {
9062 if (rdwr_pmult_cmd_in_progress) {
9063 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9064 ASSERT(satapkt != NULL);
9065 ASSERT(aborted_tags == 0x1);
9066 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9067 "ahci_mop_commands: sending up "
9068 "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
9069 (void *)satapkt);
9070 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9071 break;
9072 }
9073
9074 tmp_slot = ddi_ffs(reset_tags) - 1;
9075 if (tmp_slot == -1) {
9076 break;
9077 }
9078
9079 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9080 ASSERT(satapkt != NULL);
9081
9082 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9083 "sending up pkt 0x%p with SATA_PKT_RESET",
9084 (void *)satapkt);
9085
9086 if (ncq_cmd_in_progress)
9087 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9088 tmp_slot);
9089 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9090 CLEAR_BIT(reset_tags, tmp_slot);
9091 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9092
9093 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9094 }
9095
9096 /* Send up unfinished packets with SATA_PKT_RESET */
9097 while (unfinished_tags) {
9098 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9099 if (tmp_slot == -1) {
9100 break;
9101 }
9102
9103 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9104 ASSERT(satapkt != NULL);
9105
9106 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9107 "sending up pkt 0x%p with SATA_PKT_RESET",
9108 (void *)satapkt);
9109
9110 if (ncq_cmd_in_progress)
9111 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9112 tmp_slot);
9113 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9114 CLEAR_BIT(unfinished_tags, tmp_slot);
9115 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9116
9117 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9118 }
9119
9120 ahci_portp->ahciport_mop_in_progress--;
9121 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9122
9123 if (ahci_portp->ahciport_mop_in_progress == 0)
9124 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9125
9126 ahci_flush_doneq(ahci_portp);
9127 }
9128
9129 /*
9130 * This routine is going to first request a READ LOG EXT sata pkt from sata
9131 * module, and then deliver it to the HBA to get the ncq failure context.
9132 * The return value is the exactly failed tags.
9133 */
9134 static uint32_t
9135 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9136 uint8_t port)
9137 {
9138 sata_device_t sdevice;
9139 sata_pkt_t *rdlog_spkt, *spkt;
9140 ddi_dma_handle_t buf_dma_handle;
9141 ahci_addr_t addr;
9142 int loop_count;
9143 int rval;
9144 int failed_slot;
9145 uint32_t failed_tags = 0;
9146 struct sata_ncq_error_recovery_page *ncq_err_page;
9147
9148 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9149 "ahci_get_rdlogext_data enter: port %d", port);
9150
9151 /* Prepare the sdevice data */
9152 bzero((void *)&sdevice, sizeof (sata_device_t));
9153 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9154
9155 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9156 sdevice.satadev_addr.pmport = 0;
9157
9158 /* Translate sata_device.satadev_addr -> ahci_addr */
9159 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9160
9161 /*
9162 * Call the sata hba interface to get a rdlog spkt
9163 */
9164 loop_count = 0;
9165 loop:
9166 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9167 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9168 if (rdlog_spkt == NULL) {
9169 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9170 /* Sleep for a while */
9171 drv_usecwait(AHCI_10MS_USECS);
9172 goto loop;
9173 }
9174 /* Timed out after 1s */
9175 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9176 "failed to get rdlog spkt for port %d", port);
9177 return (failed_tags);
9178 }
9179
9180 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9181
9182 /*
9183 * This flag is used to handle the specific error recovery when the
9184 * READ LOG EXT command gets a failure (fatal error or time-out).
9185 */
9186 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9187
9188 /*
9189 * This start is not supposed to fail because after port is restarted,
9190 * the whole command list is empty.
9191 */
9192 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9193 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9194 ahci_portp->ahciport_err_retri_pkt = NULL;
9195
9196 /* Remove the flag after READ LOG EXT command is completed */
9197 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9198
9199 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9200 /* Update the request log data */
9201 buf_dma_handle = *(ddi_dma_handle_t *)
9202 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9203 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9204 DDI_DMA_SYNC_FORKERNEL);
9205 if (rval == DDI_SUCCESS) {
9206 ncq_err_page =
9207 (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9208 satapkt_cmd.satacmd_bp->b_un.b_addr;
9209
9210 /* Get the failed tag */
9211 failed_slot = ncq_err_page->ncq_tag;
9212 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9213 "ahci_get_rdlogext_data: port %d "
9214 "failed slot %d", port, failed_slot);
9215 if (failed_slot & NQ) {
9216 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9217 "the failed slot is not a valid tag", NULL);
9218 goto out;
9219 }
9220
9221 failed_slot &= NCQ_TAG_MASK;
9222 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9223 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9224 "ahci_get_rdlogext_data: failed spkt 0x%p",
9225 (void *)spkt);
9226 if (spkt == NULL) {
9227 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9228 "the failed slot spkt is NULL", NULL);
9229 goto out;
9230 }
9231
9232 failed_tags = 0x1 << failed_slot;
9233
9234 /* Fill out the error context */
9235 ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9236 ncq_err_page);
9237 ahci_update_sata_registers(ahci_ctlp, port,
9238 &spkt->satapkt_device);
9239 }
9240 }
9241 out:
9242 sata_free_error_retrieval_pkt(rdlog_spkt);
9243
9244 return (failed_tags);
9245 }
9246
9247 /*
9248 * This routine is going to first request a REQUEST SENSE sata pkt from sata
9249 * module, and then deliver it to the HBA to get the sense data and copy
9250 * the sense data back to the orignal failed sata pkt, and free the REQUEST
9251 * SENSE sata pkt later.
9252 */
9253 static void
9254 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9255 uint8_t port, sata_pkt_t *spkt)
9256 {
9257 sata_device_t sdevice;
9258 sata_pkt_t *rs_spkt;
9259 sata_cmd_t *sata_cmd;
9260 ddi_dma_handle_t buf_dma_handle;
9261 ahci_addr_t addr;
9262 int loop_count;
9263 #if AHCI_DEBUG
9264 struct scsi_extended_sense *rqsense;
9265 #endif
9266
9267 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9268 "ahci_get_rqsense_data enter: port %d", port);
9269
9270 /* Prepare the sdevice data */
9271 bzero((void *)&sdevice, sizeof (sata_device_t));
9272 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9273
9274 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9275 sdevice.satadev_addr.pmport = 0;
9276
9277 /* Translate sata_device.satadev_addr -> ahci_addr */
9278 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9279
9280 sata_cmd = &spkt->satapkt_cmd;
9281
9282 /*
9283 * Call the sata hba interface to get a rs spkt
9284 */
9285 loop_count = 0;
9286 loop:
9287 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9288 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9289 if (rs_spkt == NULL) {
9290 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9291 /* Sleep for a while */
9292 drv_usecwait(AHCI_10MS_USECS);
9293 goto loop;
9294
9295 }
9296 /* Timed out after 1s */
9297 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9298 "failed to get rs spkt for port %d", port);
9299 return;
9300 }
9301
9302 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9303
9304 /*
9305 * This flag is used to handle the specific error recovery when the
9306 * REQUEST SENSE command gets a faiure (fatal error or time-out).
9307 */
9308 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9309
9310 /*
9311 * This start is not supposed to fail because after port is restarted,
9312 * the whole command list is empty.
9313 */
9314 ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9315 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9316 ahci_portp->ahciport_err_retri_pkt = NULL;
9317
9318 /* Remove the flag after REQUEST SENSE command is completed */
9319 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9320
9321 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9322 /* Update the request sense data */
9323 buf_dma_handle = *(ddi_dma_handle_t *)
9324 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9325 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9326 DDI_DMA_SYNC_FORKERNEL);
9327 /* Copy the request sense data */
9328 bcopy(rs_spkt->
9329 satapkt_cmd.satacmd_bp->b_un.b_addr,
9330 &sata_cmd->satacmd_rqsense,
9331 SATA_ATAPI_MIN_RQSENSE_LEN);
9332 #if AHCI_DEBUG
9333 rqsense = (struct scsi_extended_sense *)
9334 sata_cmd->satacmd_rqsense;
9335
9336 /* Dump the sense data */
9337 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9338 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9339 "Sense data for satapkt %p ATAPI cmd 0x%x",
9340 spkt, sata_cmd->satacmd_acdb[0]);
9341 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9342 " es_code 0x%x es_class 0x%x "
9343 "es_key 0x%x es_add_code 0x%x "
9344 "es_qual_code 0x%x",
9345 rqsense->es_code, rqsense->es_class,
9346 rqsense->es_key, rqsense->es_add_code,
9347 rqsense->es_qual_code);
9348 #endif
9349 }
9350
9351 sata_free_error_retrieval_pkt(rs_spkt);
9352 }
9353
9354 /*
9355 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9356 * the port must be restarted. When the HBA detects thus error, it may try
9357 * to abort a transfer. And if the transfer was aborted, the device is
9358 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9359 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9360 * that the device is in a stable status and transfers may be restarted without
9361 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9362 * then the software will send the COMRESET to do the port reset.
9363 *
9364 * Software should perform the appropriate error recovery actions based on
9365 * whether non-queued commands were being issued or natived command queuing
9366 * commands were being issued.
9367 *
9368 * And software will complete the command that had the error with error mark
9369 * to higher level software.
9370 *
9371 * Fatal errors include the following:
9372 * PxIS.IFS - Interface Fatal Error Status
9373 * PxIS.HBDS - Host Bus Data Error Status
9374 * PxIS.HBFS - Host Bus Fatal Error Status
9375 * PxIS.TFES - Task File Error Status
9376 */
9377 static void
9378 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9379 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9380 {
9381 uint32_t port_cmd_status;
9382 uint32_t slot_status = 0;
9383 uint32_t failed_tags = 0;
9384 int failed_slot;
9385 int reset_flag = 0, flag = 0;
9386 ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9387 sata_cmd_t *sata_cmd = NULL;
9388 sata_pkt_t *spkt = NULL;
9389 #if AHCI_DEBUG
9390 ahci_cmd_header_t *cmd_header;
9391 #endif
9392 uint8_t port = addrp->aa_port;
9393 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9394 int rval;
9395
9396 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9397
9398 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9399 "ahci_fatal_error_recovery_handler enter: port %d", port);
9400
9401 /* Port multiplier error */
9402 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9403 /* FBS code is neither completed nor tested. */
9404 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9405 port, intr_status);
9406
9407 /* Force a port reset */
9408 flag = AHCI_PORT_RESET;
9409 }
9410
9411 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9412 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9413
9414 /* Read PxCI to see which commands are still outstanding */
9415 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9416 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9417
9418 /*
9419 * Read PxCMD.CCS to determine the slot that the HBA
9420 * was processing when the error occurred.
9421 */
9422 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9423 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9424 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9425 AHCI_CMD_STATUS_CCS_SHIFT;
9426
9427 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9428 spkt = ahci_portp->ahciport_err_retri_pkt;
9429 ASSERT(spkt != NULL);
9430 } else {
9431 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9432 if (spkt == NULL) {
9433 /* May happen when interface errors occur? */
9434 goto next;
9435 }
9436 }
9437
9438 #if AHCI_DEBUG
9439 /*
9440 * Debugging purpose...
9441 */
9442 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9443 cmd_header =
9444 &ahci_portp->ahciport_cmd_list[failed_slot];
9445 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9446 "ahci_fatal_error_recovery_handler: port %d, "
9447 "PRD Byte Count = 0x%x, "
9448 "ahciport_prd_bytecounts = 0x%x", port,
9449 cmd_header->ahcich_prd_byte_count,
9450 ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9451 }
9452 #endif
9453
9454 sata_cmd = &spkt->satapkt_cmd;
9455
9456 /* Fill out the status and error registers for PxIS.TFES */
9457 if (intr_status & AHCI_INTR_STATUS_TFES) {
9458 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9459 ahcirf_d2h_register_fis);
9460
9461 /* Copy the error context back to the sata_cmd */
9462 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9463 }
9464
9465 /* The failed command must be one of the outstanding commands */
9466 failed_tags = 0x1 << failed_slot;
9467 ASSERT(failed_tags & slot_status);
9468
9469 /* Update the sata registers, especially PxSERR register */
9470 ahci_update_sata_registers(ahci_ctlp, port,
9471 &spkt->satapkt_device);
9472
9473 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9474 /* Read PxSACT to see which commands are still outstanding */
9475 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9476 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9477 }
9478 next:
9479
9480 #if AHCI_DEBUG
9481 /*
9482 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9483 * set, it means a fatal error happened after REQUEST SENSE command
9484 * or READ LOG EXT command is delivered to the HBA during the error
9485 * recovery process. At this time, the only outstanding command is
9486 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9487 */
9488 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9489 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9490 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9491 "command or READ LOG EXT command for error data retrieval "
9492 "failed", port);
9493 ASSERT(slot_status == 0x1);
9494 ASSERT(failed_slot == 0);
9495 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9496 SCMD_REQUEST_SENSE ||
9497 spkt->satapkt_cmd.satacmd_cmd_reg ==
9498 SATAC_READ_LOG_EXT);
9499 }
9500 #endif
9501
9502 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9503 ahci_portp->ahciport_mop_in_progress++;
9504
9505 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9506 port, flag, &reset_flag);
9507
9508 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9509 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9510 if (rval == AHCI_SUCCESS)
9511 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9512 "succeed", instance, port);
9513 else
9514 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9515 "failed", instance, port);
9516 }
9517
9518 /*
9519 * Won't retrieve error information:
9520 * 1. Port reset was involved to recover
9521 * 2. Device is gone
9522 * 3. IDENTIFY DEVICE command sent to ATAPI device
9523 * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9524 */
9525 if (reset_flag ||
9526 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9527 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9528 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9529 goto out;
9530
9531 /*
9532 * Deliver READ LOG EXT to gather information about the error when
9533 * a COMRESET has not been performed as part of the error recovery
9534 * during NCQ command processing.
9535 */
9536 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9537 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9538 ahci_portp, port);
9539 goto out;
9540 }
9541
9542 /*
9543 * Deliver REQUEST SENSE for ATAPI command to gather information about
9544 * the error when a COMRESET has not been performed as part of the
9545 * error recovery.
9546 */
9547 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9548 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9549 out:
9550 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9551 "ahci_fatal_error_recovery_handler: port %d fatal error "
9552 "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9553 "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9554 port, slot_status, ahci_portp->ahciport_pending_tags,
9555 ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9556
9557 ahci_mop_commands(ahci_ctlp,
9558 ahci_portp,
9559 slot_status,
9560 failed_tags, /* failed tags */
9561 0, /* timeout tags */
9562 0, /* aborted tags */
9563 0); /* reset tags */
9564 }
9565
9566 /*
9567 * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9568 * 1. device specific.PxFBS.SDE=1
9569 * 2. Non Device specific.
9570 * Nothing will be done when Command-based switching is employed.
9571 *
9572 * Currently code is neither completed nor tested.
9573 */
9574 static void
9575 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9576 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9577 {
9578 #ifndef __lock_lint
9579 _NOTE(ARGUNUSED(intr_status))
9580 #endif
9581 uint32_t port_fbs_ctrl;
9582 int loop_count = 0;
9583 ahci_addr_t addr;
9584
9585 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9586
9587 /* Nothing will be done under Command-based switching. */
9588 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9589 return;
9590
9591 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9592 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9593
9594 if (!(port_fbs_ctrl & AHCI_FBS_EN))
9595 /* FBS is not enabled. */
9596 return;
9597
9598 /* Problem's getting complicated now. */
9599 /*
9600 * If FIS-based switching is used, we need to check
9601 * the PxFBS to see the error type.
9602 */
9603 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9604 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9605
9606 /* Refer to spec(v1.2) 9.3.6.1 */
9607 if (port_fbs_ctrl & AHCI_FBS_SDE) {
9608 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9609 "A Device Sepcific Error: port %d", port);
9610 /*
9611 * Controller has paused commands for all other
9612 * sub-devices until PxFBS.DEC is set.
9613 */
9614 ahci_reject_all_abort_pkts(ahci_ctlp,
9615 ahci_portp, 0);
9616
9617 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9618 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9619 port_fbs_ctrl | AHCI_FBS_DEC);
9620
9621 /*
9622 * Wait controller clear PxFBS.DEC,
9623 * then we can continue.
9624 */
9625 loop_count = 0;
9626 do {
9627 port_fbs_ctrl = ddi_get32(ahci_ctlp->
9628 ahcictl_ahci_acc_handle, (uint32_t *)
9629 AHCI_PORT_PxFBS(ahci_ctlp, port));
9630
9631 if (loop_count++ > 1000)
9632 /*
9633 * Esclate the error. Follow
9634 * non-device specific error
9635 * procedure.
9636 */
9637 return;
9638
9639 drv_usecwait(AHCI_100US_USECS);
9640 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9641
9642 /*
9643 * Issue a software reset to ensure drive is in
9644 * a known state.
9645 */
9646 (void) ahci_software_reset(ahci_ctlp,
9647 ahci_portp, &addr);
9648
9649 } else {
9650
9651 /* Process Non-Device Specific Error. */
9652 /* This will be handled later on. */
9653 cmn_err(CE_NOTE, "!FBS is not supported now.");
9654 }
9655 }
9656 /*
9657 * Handle events - fatal error recovery
9658 */
9659 static void
9660 ahci_events_handler(void *args)
9661 {
9662 ahci_event_arg_t *ahci_event_arg;
9663 ahci_ctl_t *ahci_ctlp;
9664 ahci_port_t *ahci_portp;
9665 ahci_addr_t *addrp;
9666 uint32_t event;
9667 int instance;
9668
9669 ahci_event_arg = (ahci_event_arg_t *)args;
9670
9671 ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9672 ahci_portp = ahci_event_arg->ahciea_portp;
9673 addrp = ahci_event_arg->ahciea_addrp;
9674 event = ahci_event_arg->ahciea_event;
9675 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9676
9677 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9678 "ahci_events_handler enter: port %d intr_status = 0x%x",
9679 ahci_portp->ahciport_port_num, event);
9680
9681 mutex_enter(&ahci_portp->ahciport_mutex);
9682
9683 /*
9684 * ahci_intr_phyrdy_change() may have rendered it to
9685 * SATA_DTYPE_NONE.
9686 */
9687 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9688 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9689 "ahci_events_handler: port %d no device attached, "
9690 "and just return without doing anything",
9691 ahci_portp->ahciport_port_num);
9692
9693 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9694 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9695 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9696 "succeed", instance, ahci_portp->ahciport_port_num);
9697 }
9698
9699 goto out;
9700 }
9701
9702 if (event & (AHCI_INTR_STATUS_IFS |
9703 AHCI_INTR_STATUS_HBDS |
9704 AHCI_INTR_STATUS_HBFS |
9705 AHCI_INTR_STATUS_TFES))
9706 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9707 addrp, event);
9708
9709 out:
9710 mutex_exit(&ahci_portp->ahciport_mutex);
9711 }
9712
9713 /*
9714 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9715 * detect there are some commands which are timed out.
9716 */
9717 static void
9718 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9719 uint8_t port, uint32_t tmp_timeout_tags)
9720 {
9721 uint32_t slot_status = 0;
9722 uint32_t finished_tags = 0;
9723 uint32_t timeout_tags = 0;
9724
9725 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9726 "ahci_timeout_pkts enter: port %d", port);
9727
9728 mutex_enter(&ahci_portp->ahciport_mutex);
9729
9730 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9731 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9732 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9733 /* Read PxCI to see which commands are still outstanding */
9734 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9735 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9736 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9737 /* Read PxSACT to see which commands are still outstanding */
9738 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9739 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9740 }
9741
9742 #if AHCI_DEBUG
9743 /*
9744 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9745 * set, it means a fatal error happened after REQUEST SENSE command
9746 * or READ LOG EXT command is delivered to the HBA during the error
9747 * recovery process. At this time, the only outstanding command is
9748 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9749 */
9750 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9751 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9752 "ahci_timeout_pkts called while REQUEST SENSE "
9753 "command or READ LOG EXT command for error recovery "
9754 "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9755 "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9756 tmp_timeout_tags, slot_status,
9757 ahci_portp->ahciport_pending_tags,
9758 ahci_portp->ahciport_pending_ncq_tags);
9759 ASSERT(slot_status == 0x1);
9760 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9761 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9762 "ahci_timeout_pkts called while executing R/W PMULT "
9763 "command timeout_tags = 0x%x, slot_status = 0x%x",
9764 tmp_timeout_tags, slot_status);
9765 ASSERT(slot_status == 0x1);
9766 }
9767 #endif
9768
9769 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9770 ahci_portp->ahciport_mop_in_progress++;
9771
9772 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9773 port, AHCI_PORT_RESET, NULL);
9774
9775 /*
9776 * Re-identify timeout tags because some previously checked commands
9777 * could already complete.
9778 */
9779 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9780 finished_tags = ahci_portp->ahciport_pending_tags &
9781 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9782 timeout_tags = tmp_timeout_tags & ~finished_tags;
9783
9784 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9785 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9786 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9787 "pending_tags = 0x%x ",
9788 port, finished_tags, timeout_tags,
9789 slot_status, ahci_portp->ahciport_pending_tags);
9790 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9791 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9792 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9793 timeout_tags = tmp_timeout_tags & ~finished_tags;
9794
9795 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9796 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9797 "timeout_tags = 0x%x, port_sactive = 0x%x, "
9798 "pending_ncq_tags = 0x%x ",
9799 port, finished_tags, timeout_tags,
9800 slot_status, ahci_portp->ahciport_pending_ncq_tags);
9801 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9802 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9803 timeout_tags = tmp_timeout_tags;
9804 }
9805
9806 ahci_mop_commands(ahci_ctlp,
9807 ahci_portp,
9808 slot_status,
9809 0, /* failed tags */
9810 timeout_tags, /* timeout tags */
9811 0, /* aborted tags */
9812 0); /* reset tags */
9813
9814 mutex_exit(&ahci_portp->ahciport_mutex);
9815 }
9816
9817 /*
9818 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9819 * for long time.
9820 */
9821 static void
9822 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9823 {
9824 ahci_port_t *ahci_portp;
9825 sata_pkt_t *spkt;
9826 uint32_t pending_tags;
9827 uint32_t timeout_tags;
9828 uint32_t port_cmd_status;
9829 uint32_t port_sactive;
9830 uint8_t port;
9831 int tmp_slot;
9832 int current_slot;
9833 uint32_t current_tags;
9834 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9835
9836 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9837
9838 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9839 "ahci_watchdog_handler entered", NULL);
9840
9841 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9842 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9843 continue;
9844 }
9845
9846 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9847
9848 mutex_enter(&ahci_portp->ahciport_mutex);
9849 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9850 mutex_exit(&ahci_portp->ahciport_mutex);
9851 continue;
9852 }
9853
9854 /* Skip the check for those ports in error recovery */
9855 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9856 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9857 mutex_exit(&ahci_portp->ahciport_mutex);
9858 continue;
9859 }
9860
9861 pending_tags = 0;
9862 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9863 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9864
9865 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9866 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9867 current_slot = 0;
9868 pending_tags = 0x1;
9869 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9870 current_slot =
9871 (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9872 AHCI_CMD_STATUS_CCS_SHIFT;
9873 pending_tags = ahci_portp->ahciport_pending_tags;
9874 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9875 port_sactive = ddi_get32(
9876 ahci_ctlp->ahcictl_ahci_acc_handle,
9877 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9878 current_tags = port_sactive &
9879 ~port_cmd_status &
9880 AHCI_NCQ_SLOT_MASK(ahci_portp);
9881 pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9882 }
9883
9884 timeout_tags = 0;
9885 while (pending_tags) {
9886 tmp_slot = ddi_ffs(pending_tags) - 1;
9887 if (tmp_slot == -1) {
9888 break;
9889 }
9890
9891 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9892 spkt = ahci_portp->ahciport_err_retri_pkt;
9893 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9894 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9895 else
9896 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9897
9898 if ((spkt != NULL) && spkt->satapkt_time &&
9899 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9900 /*
9901 * If a packet has survived for more than it's
9902 * max life cycles, it is a candidate for time
9903 * out.
9904 */
9905 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9906 ahci_watchdog_timeout;
9907
9908 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9909 > 0)
9910 goto next;
9911
9912 #if AHCI_DEBUG
9913 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9914 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9915 ahci_ctlp, "watchdog: the current "
9916 "tags is 0x%x", current_tags);
9917 } else {
9918 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9919 ahci_ctlp, "watchdog: the current "
9920 "slot is %d", current_slot);
9921 }
9922 #endif
9923
9924 /*
9925 * We need to check whether the HBA has
9926 * begun to execute the command, if not,
9927 * then re-set the timer of the command.
9928 */
9929 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9930 (tmp_slot != current_slot) ||
9931 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9932 ((0x1 << tmp_slot) & current_tags)) {
9933 ahci_portp->ahciport_slot_timeout \
9934 [tmp_slot] = spkt->satapkt_time;
9935 } else {
9936 timeout_tags |= (0x1 << tmp_slot);
9937 cmn_err(CE_WARN, "!ahci%d: watchdog "
9938 "port %d satapkt 0x%p timed out\n",
9939 instance, port, (void *)spkt);
9940 }
9941 }
9942 next:
9943 CLEAR_BIT(pending_tags, tmp_slot);
9944 }
9945
9946 if (timeout_tags) {
9947 mutex_exit(&ahci_portp->ahciport_mutex);
9948 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9949 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9950 port, timeout_tags);
9951 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9952 mutex_enter(&ahci_portp->ahciport_mutex);
9953 }
9954
9955 mutex_exit(&ahci_portp->ahciport_mutex);
9956 }
9957
9958 /* Re-install the watchdog timeout handler */
9959 if (ahci_ctlp->ahcictl_timeout_id != 0) {
9960 ahci_ctlp->ahcictl_timeout_id =
9961 timeout((void (*)(void *))ahci_watchdog_handler,
9962 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9963 }
9964
9965 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9966 }
9967
9968 /*
9969 * Fill the error context into sata_cmd for non-queued command error.
9970 */
9971 static void
9972 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9973 {
9974 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9975 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9976 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9977 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9978 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9979 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9980 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9981
9982 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9983 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9984 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9985 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9986 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9987 }
9988 }
9989
9990 /*
9991 * Fill the ncq error page into sata_cmd for queued command error.
9992 */
9993 static void
9994 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9995 struct sata_ncq_error_recovery_page *ncq_err_page)
9996 {
9997 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9998 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9999 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
10000 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
10001 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
10002 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
10003 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
10004 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
10005 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
10006 scmd->satacmd_status_reg = ncq_err_page->ncq_status;
10007 scmd->satacmd_error_reg = ncq_err_page->ncq_error;
10008 }
10009
10010 /*
10011 * Put the respective register value to sata_cmd_t for satacmd_flags.
10012 */
10013 static void
10014 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
10015 {
10016 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
10017 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
10018 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
10019 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
10020 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
10021 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
10022 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
10023 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
10024 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
10025 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
10026 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
10027 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
10028 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
10029 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
10030 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
10031 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
10032 if (scmd->satacmd_flags.sata_copy_out_device_reg)
10033 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
10034 if (scmd->satacmd_flags.sata_copy_out_error_reg)
10035 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
10036 }
10037
10038 static void
10039 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10040 uint32_t intr_status)
10041 {
10042 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
10043
10044 if (intr_status & AHCI_INTR_STATUS_IFS)
10045 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
10046 "error", instance, port);
10047
10048 if (intr_status & AHCI_INTR_STATUS_HBDS)
10049 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
10050 instance, port);
10051
10052 if (intr_status & AHCI_INTR_STATUS_HBFS)
10053 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
10054 instance, port);
10055
10056 if (intr_status & AHCI_INTR_STATUS_TFES)
10057 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
10058 instance, port);
10059
10060 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
10061 "recovery", instance, port);
10062 }
10063
10064 static void
10065 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
10066 uint32_t slot_tags)
10067 {
10068 ahci_port_t *ahci_portp;
10069 int tmp_slot;
10070 sata_pkt_t *spkt;
10071 sata_cmd_t cmd;
10072
10073 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10074 ASSERT(ahci_portp != NULL);
10075
10076 while (slot_tags) {
10077 tmp_slot = ddi_ffs(slot_tags) - 1;
10078 if (tmp_slot == -1) {
10079 break;
10080 }
10081
10082 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
10083 if (spkt != NULL) {
10084 cmd = spkt->satapkt_cmd;
10085
10086 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
10087 "features_reg = 0x%x sec_count_msb = 0x%x "
10088 "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
10089 "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
10090 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
10091 "lba_high_lsb = 0x%x device_reg = 0x%x "
10092 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10093 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10094 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10095 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10096 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10097 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10098 cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10099 *((uint32_t *)&(cmd.satacmd_flags)));
10100 }
10101
10102 CLEAR_BIT(slot_tags, tmp_slot);
10103 }
10104 }
10105
10106 /*
10107 * Dump the serror message to the log.
10108 */
10109 static void
10110 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10111 uint32_t port_serror, int debug_only)
10112 {
10113 static char err_buf[512];
10114 static char err_msg_header[16];
10115 char *err_msg = err_buf;
10116
10117 *err_buf = '\0';
10118 *err_msg_header = '\0';
10119
10120 if (port_serror & SERROR_DATA_ERR_FIXED) {
10121 err_msg = strcat(err_msg,
10122 "\tRecovered Data Integrity Error (I)\n");
10123 }
10124
10125 if (port_serror & SERROR_COMM_ERR_FIXED) {
10126 err_msg = strcat(err_msg,
10127 "\tRecovered Communication Error (M)\n");
10128 }
10129
10130 if (port_serror & SERROR_DATA_ERR) {
10131 err_msg = strcat(err_msg,
10132 "\tTransient Data Integrity Error (T)\n");
10133 }
10134
10135 if (port_serror & SERROR_PERSISTENT_ERR) {
10136 err_msg = strcat(err_msg,
10137 "\tPersistent Communication or Data Integrity Error (C)\n");
10138 }
10139
10140 if (port_serror & SERROR_PROTOCOL_ERR) {
10141 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10142 }
10143
10144 if (port_serror & SERROR_INT_ERR) {
10145 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10146 }
10147
10148 if (port_serror & SERROR_PHY_RDY_CHG) {
10149 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10150 }
10151
10152 if (port_serror & SERROR_PHY_INT_ERR) {
10153 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10154 }
10155
10156 if (port_serror & SERROR_COMM_WAKE) {
10157 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10158 }
10159
10160 if (port_serror & SERROR_10B_TO_8B_ERR) {
10161 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10162 }
10163
10164 if (port_serror & SERROR_DISPARITY_ERR) {
10165 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10166 }
10167
10168 if (port_serror & SERROR_CRC_ERR) {
10169 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10170 }
10171
10172 if (port_serror & SERROR_HANDSHAKE_ERR) {
10173 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10174 }
10175
10176 if (port_serror & SERROR_LINK_SEQ_ERR) {
10177 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10178 }
10179
10180 if (port_serror & SERROR_TRANS_ERR) {
10181 err_msg = strcat(err_msg,
10182 "\tTransport state transition error (T)\n");
10183 }
10184
10185 if (port_serror & SERROR_FIS_TYPE) {
10186 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10187 }
10188
10189 if (port_serror & SERROR_EXCHANGED_ERR) {
10190 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10191 }
10192
10193 if (*err_msg == '\0')
10194 return;
10195
10196 if (debug_only) {
10197 (void) sprintf(err_msg_header, "port %d", port);
10198 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10199 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10200 } else if (ahci_ctlp) {
10201 cmn_err(CE_WARN, "!ahci%d: %s %s",
10202 ddi_get_instance(ahci_ctlp->ahcictl_dip),
10203 err_msg_header, err_msg);
10204
10205 /* sata trace debug */
10206 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10207 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10208 err_msg_header, err_msg);
10209 } else {
10210 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10211
10212 /* sata trace debug */
10213 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10214 }
10215 }
10216
10217 /*
10218 * Translate the sata_address_t type into the ahci_addr_t type.
10219 * sata_device.satadev_addr structure is used as source.
10220 */
10221 static void
10222 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10223 ahci_addr_t *ahci_addrp)
10224 {
10225 sata_address_t *sata_addrp = &sd->satadev_addr;
10226 ahci_addrp->aa_port =
10227 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10228 ahci_addrp->aa_pmport = sata_addrp->pmport;
10229
10230 switch (sata_addrp->qual) {
10231 case SATA_ADDR_DCPORT:
10232 case SATA_ADDR_CPORT:
10233 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10234 break;
10235 case SATA_ADDR_PMULT:
10236 case SATA_ADDR_PMULT_SPEC:
10237 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10238 break;
10239 case SATA_ADDR_DPMPORT:
10240 case SATA_ADDR_PMPORT:
10241 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10242 break;
10243 case SATA_ADDR_NULL:
10244 default:
10245 /* something went wrong */
10246 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10247 break;
10248 }
10249 }
10250
10251 /*
10252 * This routine is to calculate the total number of ports implemented
10253 * by the HBA.
10254 */
10255 static int
10256 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10257 {
10258 uint8_t i;
10259 int num = 0;
10260
10261 for (i = 0; i < AHCI_MAX_PORTS; i++) {
10262 if (((uint32_t)0x1 << i) & ports_implemented)
10263 num++;
10264 }
10265
10266 return (num);
10267 }
10268
10269 #if AHCI_DEBUG
10270 static void
10271 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10272 {
10273 static char name[16];
10274 va_list ap;
10275
10276 mutex_enter(&ahci_log_mutex);
10277
10278 va_start(ap, fmt);
10279 if (ahci_ctlp) {
10280 (void) sprintf(name, "ahci%d: ",
10281 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10282 } else {
10283 (void) sprintf(name, "ahci: ");
10284 }
10285
10286 (void) vsprintf(ahci_log_buf, fmt, ap);
10287 va_end(ap);
10288
10289 cmn_err(level, "%s%s", name, ahci_log_buf);
10290
10291 mutex_exit(&ahci_log_mutex);
10292 }
10293 #endif
10294
10295 /*
10296 * quiesce(9E) entry point.
10297 *
10298 * This function is called when the system is single-threaded at high
10299 * PIL with preemption disabled. Therefore, this function must not be
10300 * blocked. Because no taskqs are running, there is no need for us to
10301 * take any action for enclosure services which are running in the
10302 * taskq context, especially as no interrupts are generated by it nor
10303 * are any messages expected to come in.
10304 *
10305 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10306 * DDI_FAILURE indicates an error condition and should almost never happen.
10307 */
10308 static int
10309 ahci_quiesce(dev_info_t *dip)
10310 {
10311 ahci_ctl_t *ahci_ctlp;
10312 ahci_port_t *ahci_portp;
10313 int instance, port;
10314
10315 instance = ddi_get_instance(dip);
10316 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10317
10318 if (ahci_ctlp == NULL)
10319 return (DDI_FAILURE);
10320
10321 #if AHCI_DEBUG
10322 ahci_debug_flags = 0;
10323 #endif
10324
10325 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10326
10327 /* disable all the interrupts. */
10328 ahci_disable_all_intrs(ahci_ctlp);
10329
10330 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10331 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10332 continue;
10333 }
10334
10335 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10336
10337 /*
10338 * Stop the port by clearing PxCMD.ST
10339 *
10340 * Here we must disable the port interrupt because
10341 * ahci_disable_all_intrs only clear GHC.IE, and IS
10342 * register will be still set if PxIE is enabled.
10343 * When ahci shares one IRQ with other drivers, the
10344 * intr handler may claim the intr mistakenly.
10345 */
10346 ahci_disable_port_intrs(ahci_ctlp, port);
10347 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10348 ahci_portp, port);
10349 }
10350
10351 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10352
10353 return (DDI_SUCCESS);
10354 }
10355
10356 /*
10357 * The function will add a sata packet to the done queue.
10358 */
10359 static void
10360 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10361 {
10362 ASSERT(satapkt != NULL);
10363 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10364
10365 /* set the reason for all packets */
10366 satapkt->satapkt_reason = reason;
10367 satapkt->satapkt_hba_driver_private = NULL;
10368
10369 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10370 satapkt->satapkt_comp) {
10371 /*
10372 * only add to queue when mode is not synch and there is
10373 * completion callback
10374 */
10375 *ahci_portp->ahciport_doneqtail = satapkt;
10376 ahci_portp->ahciport_doneqtail =
10377 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10378 ahci_portp->ahciport_doneq_len++;
10379
10380 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10381 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10382 /*
10383 * for sync/non-poll mode, just call cv_broadcast
10384 */
10385 cv_broadcast(&ahci_portp->ahciport_cv);
10386 }
10387
10388 /*
10389 * The function will call completion callback of sata packet on the
10390 * completed queue
10391 */
10392 static void
10393 ahci_flush_doneq(ahci_port_t *ahci_portp)
10394 {
10395 sata_pkt_t *satapkt, *next;
10396
10397 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10398
10399 if (ahci_portp->ahciport_doneq) {
10400 satapkt = ahci_portp->ahciport_doneq;
10401
10402 ahci_portp->ahciport_doneq = NULL;
10403 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10404 ahci_portp->ahciport_doneq_len = 0;
10405
10406 mutex_exit(&ahci_portp->ahciport_mutex);
10407
10408 while (satapkt != NULL) {
10409 next = satapkt->satapkt_hba_driver_private;
10410 satapkt->satapkt_hba_driver_private = NULL;
10411
10412 /* Call the callback */
10413 (*satapkt->satapkt_comp)(satapkt);
10414
10415 satapkt = next;
10416 }
10417
10418 mutex_enter(&ahci_portp->ahciport_mutex);
10419 }
10420 }
10421
10422 /*
10423 * Sets the state for the specified port on the controller to desired state.
10424 * This must be run in the context of the enclosure taskq which ensures that
10425 * only one event is outstanding at any time.
10426 */
10427 static boolean_t
10428 ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire)
10429 {
10430 ahci_em_led_msg_t msg;
10431 ahci_em_msg_hdr_t hdr;
10432 uint32_t msgval, hdrval;
10433 uint_t i, max_delay = ahci_em_tx_delay_count;
10434
10435 msg.alm_hba = port;
10436 msg.alm_pminfo = 0;
10437 msg.alm_value = 0;
10438
10439 if (desire & AHCI_EM_LED_IDENT_ENABLE) {
10440 msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF;
10441 }
10442
10443 if (desire & AHCI_EM_LED_FAULT_ENABLE) {
10444 msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF;
10445 }
10446
10447 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 &&
10448 (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) {
10449 msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF;
10450 }
10451
10452 hdr.aemh_rsvd = 0;
10453 hdr.aemh_mlen = sizeof (ahci_em_led_msg_t);
10454 hdr.aemh_dlen = 0;
10455 hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED;
10456
10457 bcopy(&msg, &msgval, sizeof (msgval));
10458 bcopy(&hdr, &hdrval, sizeof (hdrval));
10459
10460 /*
10461 * First, make sure we can transmit. We should not have been placed in a
10462 * situation where an outstanding transmission is going on.
10463 */
10464 for (i = 0; i < max_delay; i++) {
10465 uint32_t val;
10466
10467 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10468 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10469 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10470 break;
10471
10472 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10473 }
10474
10475 if (i == max_delay)
10476 return (B_FALSE);
10477
10478 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10479 (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval);
10480 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10481 (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval);
10482 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10483 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM);
10484
10485 for (i = 0; i < max_delay; i++) {
10486 uint32_t val;
10487
10488 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10489 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10490 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10491 break;
10492
10493 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10494 }
10495
10496 if (i == max_delay)
10497 return (B_FALSE);
10498
10499 return (B_TRUE);
10500 }
10501
10502 typedef struct ahci_em_led_task_arg {
10503 ahci_ctl_t *aelta_ctl;
10504 uint8_t aelta_port;
10505 uint_t aelta_op;
10506 ahci_em_led_state_t aelta_state;
10507 uint_t aelta_ret;
10508 kcondvar_t aelta_cv;
10509 uint_t aelta_ref;
10510 } ahci_em_led_task_arg_t;
10511
10512 static void
10513 ahci_em_led_task_free(ahci_em_led_task_arg_t *task)
10514 {
10515 ASSERT3U(task->aelta_ref, ==, 0);
10516 cv_destroy(&task->aelta_cv);
10517 kmem_free(task, sizeof (*task));
10518 }
10519
10520 static void
10521 ahci_em_led_task(void *arg)
10522 {
10523 boolean_t ret, cleanup = B_FALSE;
10524 ahci_em_led_task_arg_t *led = arg;
10525 ahci_em_led_state_t state;
10526
10527 mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10528 if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) {
10529 led->aelta_ret = EIO;
10530 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10531 return;
10532 }
10533
10534 state = led->aelta_ctl->ahcictl_em_state[led->aelta_port];
10535 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10536
10537 switch (led->aelta_op) {
10538 case AHCI_EM_IOC_SET_OP_ADD:
10539 state |= led->aelta_state;
10540 break;
10541 case AHCI_EM_IOC_SET_OP_REM:
10542 state &= ~led->aelta_state;
10543 break;
10544 case AHCI_EM_IOC_SET_OP_SET:
10545 state = led->aelta_state;
10546 break;
10547 default:
10548 led->aelta_ret = ENOTSUP;
10549 return;
10550 }
10551
10552 ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state);
10553
10554 mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10555 if (ret) {
10556 led->aelta_ctl->ahcictl_em_state[led->aelta_port] =
10557 led->aelta_state;
10558 led->aelta_ret = 0;
10559 } else {
10560 led->aelta_ret = EIO;
10561 led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10562 }
10563 led->aelta_ref--;
10564 if (led->aelta_ref > 0) {
10565 cv_signal(&led->aelta_cv);
10566 } else {
10567 cleanup = B_TRUE;
10568 }
10569 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10570
10571 if (cleanup) {
10572 ahci_em_led_task_free(led);
10573 }
10574 }
10575
10576 static void
10577 ahci_em_reset(void *arg)
10578 {
10579 uint_t i, max_delay = ahci_em_reset_delay_count;
10580 ahci_ctl_t *ahci_ctlp = arg;
10581
10582 /*
10583 * We've been asked to reset the device. The caller should have set the
10584 * resetting flag. Make sure that we don't have a request to quiesce.
10585 */
10586 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10587 ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING);
10588 if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) {
10589 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10590 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10591 return;
10592 }
10593 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10594
10595 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10596 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST);
10597 for (i = 0; i < max_delay; i++) {
10598 uint32_t val;
10599
10600 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10601 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10602 if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0)
10603 break;
10604
10605 delay(drv_usectohz(ahci_em_reset_delay_ms * 1000));
10606 }
10607
10608 if (i == max_delay) {
10609 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10610 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10611 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10612 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10613 cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting",
10614 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10615 return;
10616 }
10617
10618 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10619
10620 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i))
10621 continue;
10622
10623 /*
10624 * Try to flush all the LEDs as part of reset. If it fails,
10625 * drive on.
10626 */
10627 if (!ahci_em_set_led(ahci_ctlp, i,
10628 ahci_ctlp->ahcictl_em_state[i])) {
10629 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10630 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10631 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10632 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10633 cmn_err(CE_WARN, "!ahci%d: enclosure timed out "
10634 "setting port %u",
10635 ddi_get_instance(ahci_ctlp->ahcictl_dip), i);
10636 return;
10637 }
10638 }
10639
10640 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10641 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10642 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY;
10643 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10644 }
10645
10646 static boolean_t
10647 ahci_em_init(ahci_ctl_t *ahci_ctlp)
10648 {
10649 char name[128];
10650
10651 /*
10652 * First make sure we actually have enclosure services and if so, that
10653 * we have the hardware support that we care about for this.
10654 */
10655 if (ahci_ctlp->ahcictl_em_loc == 0 ||
10656 (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0)
10657 return (B_TRUE);
10658
10659 /*
10660 * Next, make sure that the buffer is large enough for us. We need two
10661 * dwords or 8 bytes. The location register is stored in dwords.
10662 */
10663 if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) <
10664 AHCI_EM_BUFFER_MIN) {
10665 return (B_TRUE);
10666 }
10667
10668 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT;
10669
10670 ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc &
10671 AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4;
10672 ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr;
10673
10674 bzero(ahci_ctlp->ahcictl_em_state,
10675 sizeof (ahci_ctlp->ahcictl_em_state));
10676
10677 (void) snprintf(name, sizeof (name), "ahcti_em_taskq%d",
10678 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10679 if ((ahci_ctlp->ahcictl_em_taskq =
10680 ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1,
10681 TASKQ_DEFAULTPRI, 0)) == NULL) {
10682 cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em "
10683 "services", ddi_get_instance(ahci_ctlp->ahcictl_dip));
10684 return (B_FALSE);
10685 }
10686
10687 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10688 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
10689 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10690 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10691 ahci_ctlp, DDI_SLEEP);
10692
10693 return (B_TRUE);
10694 }
10695
10696 static int
10697 ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10698 {
10699 int i;
10700 ahci_ioc_em_get_t get;
10701
10702 bzero(&get, sizeof (get));
10703 get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented;
10704 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) {
10705 get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY;
10706 }
10707
10708 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10709 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10710 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) {
10711 continue;
10712 }
10713 get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i];
10714 }
10715 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10716
10717 if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0)
10718 return (EFAULT);
10719
10720 return (0);
10721 }
10722
10723 static int
10724 ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10725 {
10726 int ret;
10727 ahci_ioc_em_set_t set;
10728 ahci_em_led_task_arg_t *task;
10729 boolean_t signal, cleanup;
10730
10731 if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0)
10732 return (EFAULT);
10733
10734 if (set.aiems_port > ahci_ctlp->ahcictl_num_ports)
10735 return (EINVAL);
10736
10737 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) {
10738 return (EINVAL);
10739 }
10740
10741 if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE |
10742 AHCI_EM_LED_FAULT_ENABLE |
10743 AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) {
10744 return (EINVAL);
10745 }
10746
10747 switch (set.aiems_op) {
10748 case AHCI_EM_IOC_SET_OP_ADD:
10749 case AHCI_EM_IOC_SET_OP_REM:
10750 case AHCI_EM_IOC_SET_OP_SET:
10751 break;
10752 default:
10753 return (EINVAL);
10754 }
10755
10756 if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 &&
10757 ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) {
10758 return (ENOTSUP);
10759 }
10760
10761 task = kmem_alloc(sizeof (*task), KM_NOSLEEP | KM_NORMALPRI);
10762 if (task == NULL) {
10763 return (ENOMEM);
10764 }
10765
10766 task->aelta_ctl = ahci_ctlp;
10767 task->aelta_port = (uint8_t)set.aiems_port;
10768 task->aelta_op = set.aiems_op;
10769 task->aelta_state = set.aiems_leds;
10770
10771 cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL);
10772
10773 /*
10774 * Initialize the reference count to two. One for us and one for the
10775 * taskq. This will be used in case we get canceled.
10776 */
10777 task->aelta_ref = 2;
10778
10779 /*
10780 * Once dispatched, the task state is protected by our global mutex.
10781 */
10782 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq,
10783 ahci_em_led_task, task, DDI_SLEEP);
10784
10785 signal = B_FALSE;
10786 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10787 while (task->aelta_ref > 1) {
10788 if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) ==
10789 0) {
10790 signal = B_TRUE;
10791 break;
10792 }
10793 }
10794
10795 /*
10796 * Remove our reference count. If we were woken up because of a signal
10797 * then the taskq may still be dispatched. In which case we shouldn't
10798 * free this memory until it is done. In that case, the taskq will take
10799 * care of it.
10800 */
10801 task->aelta_ref--;
10802 cleanup = (task->aelta_ref == 0);
10803 if (signal) {
10804 ret = EINTR;
10805 } else {
10806 ret = task->aelta_ret;
10807 }
10808 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10809
10810 if (cleanup) {
10811 ahci_em_led_task_free(task);
10812 }
10813
10814 return (ret);
10815 }
10816
10817 static int
10818 ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg)
10819 {
10820 int inst;
10821 ahci_ctl_t *ahci_ctlp;
10822
10823 inst = ddi_get_instance(dip);
10824 if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) {
10825 return (ENXIO);
10826 }
10827
10828 switch (cmd) {
10829 case AHCI_EM_IOC_GET:
10830 return (ahci_em_ioctl_get(ahci_ctlp, arg));
10831 case AHCI_EM_IOC_SET:
10832 return (ahci_em_ioctl_set(ahci_ctlp, arg));
10833 default:
10834 return (ENOTTY);
10835 }
10836
10837 }
10838
10839 static void
10840 ahci_em_quiesce(ahci_ctl_t *ahci_ctlp)
10841 {
10842 ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT);
10843
10844 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10845 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE;
10846 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10847
10848 ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq);
10849 }
10850
10851 static void
10852 ahci_em_suspend(ahci_ctl_t *ahci_ctlp)
10853 {
10854 ahci_em_quiesce(ahci_ctlp);
10855
10856 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10857 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY;
10858 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10859 }
10860
10861 static void
10862 ahci_em_resume(ahci_ctl_t *ahci_ctlp)
10863 {
10864 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10865 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
10866 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10867
10868 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10869 ahci_ctlp, DDI_SLEEP);
10870 }
10871
10872 static void
10873 ahci_em_fini(ahci_ctl_t *ahci_ctlp)
10874 {
10875 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10876 return;
10877 }
10878
10879 ahci_em_quiesce(ahci_ctlp);
10880 ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq);
10881 ahci_ctlp->ahcictl_em_taskq = NULL;
10882 }