1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2016 Nexenta Systems, Inc.
14 */
15
16 #include <sys/atomic.h>
17 #include <sys/cmn_err.h>
18 #include <sys/conf.h>
19 #include <sys/cpuvar.h>
20 #include <sys/ddi.h>
21 #include <sys/errno.h>
22 #include <sys/fs/dv_node.h>
23 #include <sys/kmem.h>
24 #include <sys/kmem_impl.h>
25 #include <sys/list.h>
26 #include <sys/modctl.h>
27 #include <sys/pci.h>
28 #include <sys/scsi/scsi.h>
29 #include <sys/sunddi.h>
30 #include <sys/sysmacros.h>
31 #include <sys/time.h>
32 #include <sys/types.h>
33
34 #include "pvscsi.h"
35 #include "pvscsi_var.h"
36
37 int pvscsi_enable_msi = 1;
38 int pvscsi_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
39 int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
40
41 static int pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt);
42
43 static void *pvscsi_sstate;
44
45 /* HBA DMA attributes */
46 static ddi_dma_attr_t pvscsi_hba_dma_attr = {
47 .dma_attr_version = DMA_ATTR_V0,
48 .dma_attr_addr_lo = 0x0000000000000000ull,
49 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
50 .dma_attr_count_max = 0x000000007FFFFFFFull,
51 .dma_attr_align = 0x0000000000000001ull,
52 .dma_attr_burstsizes = 0x7ff,
53 .dma_attr_minxfer = 0x00000001u,
54 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
55 .dma_attr_seg = 0x00000000FFFFFFFFull,
56 .dma_attr_sgllen = 1,
57 .dma_attr_granular = 0x00000200u,
58 .dma_attr_flags = 0
59 };
60
61 /* DMA attributes for req/comp rings */
62 static ddi_dma_attr_t pvscsi_ring_dma_attr = {
63 .dma_attr_version = DMA_ATTR_V0,
64 .dma_attr_addr_lo = 0x0000000000000000ull,
65 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
66 .dma_attr_count_max = 0x000000007FFFFFFFull,
67 .dma_attr_align = 0x0000000000000001ull,
68 .dma_attr_burstsizes = 0x7ff,
69 .dma_attr_minxfer = 0x00000001u,
70 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
71 .dma_attr_seg = 0x00000000FFFFFFFFull,
72 .dma_attr_sgllen = 1,
73 .dma_attr_granular = 0x00000001u,
74 .dma_attr_flags = 0
75 };
76
77 /* DMA attributes for buffer I/O */
78 static ddi_dma_attr_t pvscsi_io_dma_attr = {
79 .dma_attr_version = DMA_ATTR_V0,
80 .dma_attr_addr_lo = 0x0000000000000000ull,
81 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
82 .dma_attr_count_max = 0x000000007FFFFFFFull,
83 .dma_attr_align = 0x0000000000000001ull,
84 .dma_attr_burstsizes = 0x7ff,
85 .dma_attr_minxfer = 0x00000001u,
86 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
87 .dma_attr_seg = 0x00000000FFFFFFFFull,
88 .dma_attr_sgllen = PVSCSI_MAX_SG_SIZE,
89 .dma_attr_granular = 0x00000200u,
90 .dma_attr_flags = 0
91 };
92
93 static ddi_device_acc_attr_t pvscsi_mmio_attr = {
94 DDI_DEVICE_ATTR_V1,
95 DDI_STRUCTURE_LE_ACC,
96 DDI_STRICTORDER_ACC,
97 DDI_DEFAULT_ACC
98 };
99
100 static ddi_device_acc_attr_t pvscsi_dma_attrs = {
101 DDI_DEVICE_ATTR_V0,
102 DDI_STRUCTURE_LE_ACC,
103 DDI_STRICTORDER_ACC,
104 DDI_DEFAULT_ACC,
105 };
106
107 static void
108 pvscsi_add_to_queue(pvscsi_cmd_t *cmd)
109 {
110 pvscsi_softc_t *pvs = cmd->cmd_pvs;
111
112 ASSERT(pvs != NULL);
113 ASSERT(mutex_owned(&pvs->mutex));
114 ASSERT(!list_link_active(&(cmd)->cmd_queue_node));
115
116 list_insert_tail(&pvs->cmd_queue, cmd);
117 pvs->cmd_queue_len++;
118 }
119
120 static void
121 pvscsi_remove_from_queue(pvscsi_cmd_t *cmd)
122 {
123 pvscsi_softc_t *pvs = cmd->cmd_pvs;
124
125 ASSERT(pvs != NULL);
126 ASSERT(mutex_owned(&pvs->mutex));
127 ASSERT(list_link_active(&cmd->cmd_queue_node));
128 ASSERT(pvs->cmd_queue_len > 0);
129
130 if (list_link_active(&cmd->cmd_queue_node)) {
131 list_remove(&pvs->cmd_queue, cmd);
132 pvs->cmd_queue_len--;
133 }
134 }
135
136 static uint64_t
137 pvscsi_map_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_ctx_t *io_ctx)
138 {
139 return (io_ctx - pvs->cmd_ctx + 1);
140 }
141
142 static pvscsi_cmd_ctx_t *
143 pvscsi_lookup_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
144 {
145 pvscsi_cmd_ctx_t *ctx, *end;
146
147 end = &pvs->cmd_ctx[pvs->req_depth];
148 for (ctx = pvs->cmd_ctx; ctx < end; ctx++) {
149 if (ctx->cmd == cmd)
150 return (ctx);
151 }
152
153 return (NULL);
154 }
155
156 static pvscsi_cmd_ctx_t *
157 pvscsi_resolve_ctx(pvscsi_softc_t *pvs, uint64_t ctx)
158 {
159 if (ctx > 0 && ctx <= pvs->req_depth)
160 return (&pvs->cmd_ctx[ctx - 1]);
161 else
162 return (NULL);
163 }
164
165 static boolean_t
166 pvscsi_acquire_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
167 {
168 pvscsi_cmd_ctx_t *ctx;
169
170 if (list_is_empty(&pvs->cmd_ctx_pool))
171 return (B_FALSE);
172
173 ctx = (pvscsi_cmd_ctx_t *)list_remove_head(&pvs->cmd_ctx_pool);
174 ASSERT(ctx != NULL);
175
176 ctx->cmd = cmd;
177 cmd->ctx = ctx;
178
179 return (B_TRUE);
180 }
181
182 static void
183 pvscsi_release_ctx(pvscsi_cmd_t *cmd)
184 {
185 pvscsi_softc_t *pvs = cmd->cmd_pvs;
186
187 ASSERT(mutex_owned(&pvs->mutex));
188
189 cmd->ctx->cmd = NULL;
190 list_insert_tail(&pvs->cmd_ctx_pool, cmd->ctx);
191 cmd->ctx = NULL;
192 }
193
194 static uint32_t
195 pvscsi_reg_read(pvscsi_softc_t *pvs, uint32_t offset)
196 {
197 uint32_t ret;
198
199 ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
200
201 ret = ddi_get32(pvs->mmio_handle,
202 (uint32_t *)(pvs->mmio_base + offset));
203
204 return (ret);
205 }
206
207 static void
208 pvscsi_reg_write(pvscsi_softc_t *pvs, uint32_t offset, uint32_t value)
209 {
210 ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
211
212 ddi_put32(pvs->mmio_handle, (uint32_t *)(pvs->mmio_base + offset),
213 value);
214 }
215
216 static void
217 pvscsi_write_cmd_desc(pvscsi_softc_t *pvs, uint32_t cmd, void *desc, size_t len)
218 {
219 len /= sizeof (uint32_t);
220 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_COMMAND, cmd);
221 ddi_rep_put32(pvs->mmio_handle, (uint32_t *)desc,
222 (uint32_t *)(pvs->mmio_base + PVSCSI_REG_OFFSET_COMMAND_DATA),
223 len, DDI_DEV_NO_AUTOINCR);
224 }
225
226 static uint32_t
227 pvscsi_read_intr_status(pvscsi_softc_t *pvs)
228 {
229 return (pvscsi_reg_read(pvs, PVSCSI_REG_OFFSET_INTR_STATUS));
230 }
231
232 static void
233 pvscsi_write_intr_status(pvscsi_softc_t *pvs, uint32_t val)
234 {
235 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_STATUS, val);
236 }
237
238 static void
239 pvscsi_mask_intr(pvscsi_softc_t *pvs)
240 {
241 mutex_enter(&pvs->intr_mutex);
242
243 VERIFY(pvs->intr_lock_counter >= 0);
244
245 if (++pvs->intr_lock_counter == 1)
246 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
247
248 mutex_exit(&pvs->intr_mutex);
249 }
250
251 static void
252 pvscsi_unmask_intr(pvscsi_softc_t *pvs)
253 {
254 mutex_enter(&pvs->intr_mutex);
255
256 VERIFY(pvs->intr_lock_counter > 0);
257
258 if (--pvs->intr_lock_counter == 0) {
259 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
260 PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
261 }
262
263 mutex_exit(&pvs->intr_mutex);
264 }
265
266 static void
267 pvscsi_reset_hba(pvscsi_softc_t *pvs)
268 {
269 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
270 }
271
272 static void
273 pvscsi_reset_bus(pvscsi_softc_t *pvs)
274 {
275 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_BUS, NULL, 0);
276 }
277
278 static void
279 pvscsi_submit_nonrw_io(pvscsi_softc_t *pvs)
280 {
281 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
282 }
283
284 static void
285 pvscsi_submit_rw_io(pvscsi_softc_t *pvs)
286 {
287 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
288 }
289
290
291 static int
292 pvscsi_inquiry_target(pvscsi_softc_t *pvs, int target, struct scsi_inquiry *inq)
293 {
294 int len = sizeof (struct scsi_inquiry);
295 int ret = -1;
296 struct buf *b;
297 struct scsi_address ap;
298 struct scsi_pkt *pkt;
299 uint8_t cdb[CDB_GROUP0];
300
301 ap.a_hba_tran = pvs->tran;
302 ap.a_target = (ushort_t)target;
303 ap.a_lun = (uchar_t)0;
304
305 if ((b = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL, len,
306 B_READ, NULL_FUNC, NULL)) == NULL)
307 return (-1);
308
309 if ((pkt = scsi_init_pkt(&ap, (struct scsi_pkt *)NULL, b,
310 CDB_GROUP0, sizeof (struct scsi_arq_status), 0, 0,
311 NULL_FUNC, NULL)) == NULL)
312 goto free_buf;
313
314 cdb[0] = SCMD_INQUIRY;
315 cdb[1] = 0;
316 cdb[2] = 0;
317 cdb[3] = (len & 0xff00) >> 8;
318 cdb[4] = (len & 0x00ff);
319 cdb[5] = 0;
320
321 if (inq != NULL)
322 bzero(inq, sizeof (*inq));
323 bcopy(cdb, pkt->pkt_cdbp, CDB_GROUP0);
324 bzero((struct scsi_inquiry *)b->b_un.b_addr, sizeof (*inq));
325
326 if ((ret = scsi_poll(pkt)) == 0 && inq != NULL)
327 bcopy(b->b_un.b_addr, inq, sizeof (*inq));
328
329 scsi_destroy_pkt(pkt);
330
331 free_buf:
332 scsi_free_consistent_buf(b);
333
334 return (ret);
335 }
336
337 static int
338 pvscsi_config_one(dev_info_t *pdip, pvscsi_softc_t *pvs, int target,
339 dev_info_t **childp)
340 {
341 char **compatible = NULL;
342 char *nodename = NULL;
343 dev_info_t *dip;
344 int inqrc;
345 int ncompatible = 0;
346 pvscsi_device_t *devnode;
347 struct scsi_inquiry inq;
348
349 ASSERT(DEVI_BUSY_OWNED(pdip));
350
351 /* Inquiry target */
352 inqrc = pvscsi_inquiry_target(pvs, target, &inq);
353
354 /* Find devnode */
355 for (devnode = list_head(&pvs->devnodes); devnode != NULL;
356 devnode = list_next(&pvs->devnodes, devnode)) {
357 if (devnode->target == target)
358 break;
359 }
360
361 if (devnode != NULL) {
362 if (inqrc != 0) {
363 /* Target disappeared, drop devnode */
364 if (i_ddi_devi_attached(devnode->pdip)) {
365 char *devname;
366 /* Get full devname */
367 devname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
368 (void) ddi_deviname(devnode->pdip, devname);
369 /* Clean cache and name */
370 (void) devfs_clean(devnode->parent, devname + 1,
371 DV_CLEAN_FORCE);
372 kmem_free(devname, MAXPATHLEN);
373 }
374
375 (void) ndi_devi_offline(devnode->pdip, NDI_DEVI_REMOVE);
376
377 list_remove(&pvs->devnodes, devnode);
378 kmem_free(devnode, sizeof (*devnode));
379 } else if (childp != NULL) {
380 /* Target exists */
381 *childp = devnode->pdip;
382 }
383 return (NDI_SUCCESS);
384 } else if (inqrc != 0) {
385 /* Target doesn't exist */
386 return (NDI_FAILURE);
387 }
388
389 scsi_hba_nodename_compatible_get(&inq, NULL, inq.inq_dtype, NULL,
390 &nodename, &compatible, &ncompatible);
391 if (nodename == NULL)
392 goto free_nodename;
393
394 if (ndi_devi_alloc(pdip, nodename, DEVI_SID_NODEID,
395 &dip) != NDI_SUCCESS) {
396 dev_err(pvs->dip, CE_WARN, "!failed to alloc device instance");
397 goto free_nodename;
398 }
399
400 if (ndi_prop_update_string(DDI_DEV_T_NONE, dip,
401 "device-type", "scsi") != DDI_PROP_SUCCESS ||
402 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
403 "target", target) != DDI_PROP_SUCCESS ||
404 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
405 "lun", 0) != DDI_PROP_SUCCESS ||
406 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
407 "pm-capable", 1) != DDI_PROP_SUCCESS ||
408 ndi_prop_update_string_array(DDI_DEV_T_NONE, dip,
409 "compatible", compatible, ncompatible) != DDI_PROP_SUCCESS) {
410 dev_err(pvs->dip, CE_WARN,
411 "!failed to update props for target %d", target);
412 goto free_devi;
413 }
414
415 if ((devnode = kmem_zalloc(sizeof (*devnode), KM_NOSLEEP)) == NULL)
416 goto free_devi;
417
418 if (ndi_devi_online(dip, NDI_ONLINE_ATTACH) != NDI_SUCCESS) {
419 dev_err(pvs->dip, CE_WARN, "!failed to online target %d",
420 target);
421 kmem_free(devnode, sizeof (*devnode));
422 goto free_devi;
423 }
424
425 devnode->target = target;
426 devnode->pdip = dip;
427 devnode->parent = pdip;
428 list_insert_tail(&pvs->devnodes, devnode);
429
430 if (childp != NULL)
431 *childp = dip;
432
433 scsi_hba_nodename_compatible_free(nodename, compatible);
434
435 return (NDI_SUCCESS);
436
437 free_devi:
438 ndi_prop_remove_all(dip);
439 (void) ndi_devi_free(dip);
440 free_nodename:
441 scsi_hba_nodename_compatible_free(nodename, compatible);
442
443 return (NDI_FAILURE);
444 }
445
446 static int
447 pvscsi_config_all(dev_info_t *pdip, pvscsi_softc_t *pvs)
448 {
449 int target;
450
451 for (target = 0; target < PVSCSI_MAXTGTS; target++) {
452 /* ndi_devi_enter is done in pvscsi_bus_config */
453 (void) pvscsi_config_one(pdip, pvs, target, NULL);
454 }
455
456 return (NDI_SUCCESS);
457 }
458
459 static pvscsi_cmd_t *
460 pvscsi_process_comp_ring(pvscsi_softc_t *pvs)
461 {
462 pvscsi_cmd_t **pnext_cmd = NULL;
463 pvscsi_cmd_t *cmd;
464 pvscsi_cmd_t *head = NULL;
465 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
466 uint32_t cmp_ne = sdesc->cmpNumEntriesLog2;
467
468 ASSERT(mutex_owned(&pvs->rx_mutex));
469
470 while (sdesc->cmpConsIdx != sdesc->cmpProdIdx) {
471 pvscsi_cmd_ctx_t *ctx;
472 struct PVSCSIRingCmpDesc *cdesc;
473
474 cdesc = CMP_RING(pvs) + (sdesc->cmpConsIdx & MASK(cmp_ne));
475 membar_consumer();
476
477 ctx = pvscsi_resolve_ctx(pvs, cdesc->context);
478 ASSERT(ctx != NULL);
479
480 if ((cmd = ctx->cmd) != NULL) {
481 cmd->next_cmd = NULL;
482
483 /* Save command status for further processing */
484 cmd->cmp_stat.host_status = cdesc->hostStatus;
485 cmd->cmp_stat.scsi_status = cdesc->scsiStatus;
486 cmd->cmp_stat.data_len = cdesc->dataLen;
487
488 /* Mark this command as arrived from hardware */
489 cmd->flags |= PVSCSI_FLAG_HW_STATUS;
490
491 if (head == NULL) {
492 head = cmd;
493 head->tail_cmd = cmd;
494 } else {
495 head->tail_cmd = cmd;
496 }
497
498 if (pnext_cmd == NULL) {
499 pnext_cmd = &cmd->next_cmd;
500 } else {
501 *pnext_cmd = cmd;
502 pnext_cmd = &cmd->next_cmd;
503 }
504 }
505
506 membar_consumer();
507 sdesc->cmpConsIdx++;
508 }
509
510 return (head);
511 }
512
513 static pvscsi_msg_t *
514 pvscsi_process_msg_ring(pvscsi_softc_t *pvs)
515 {
516 pvscsi_msg_t *msg;
517 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
518 struct PVSCSIRingMsgDesc *mdesc;
519 struct PVSCSIMsgDescDevStatusChanged *desc;
520 uint32_t msg_ne = sdesc->msgNumEntriesLog2;
521
522 ASSERT(mutex_owned(&pvs->rx_mutex));
523
524 if (sdesc->msgProdIdx == sdesc->msgConsIdx)
525 return (NULL);
526
527 mdesc = MSG_RING(pvs) + (sdesc->msgConsIdx & MASK(msg_ne));
528 membar_consumer();
529
530 switch (mdesc->type) {
531 case PVSCSI_MSG_DEV_ADDED:
532 case PVSCSI_MSG_DEV_REMOVED:
533 desc = (struct PVSCSIMsgDescDevStatusChanged *)mdesc;
534 msg = kmem_alloc(sizeof (pvscsi_msg_t), KM_NOSLEEP);
535 if (msg == NULL)
536 return (NULL);
537 msg->msg_pvs = pvs;
538 msg->type = mdesc->type;
539 msg->target = desc->target;
540 break;
541 default:
542 dev_err(pvs->dip, CE_WARN, "!unknown msg type: %d",
543 mdesc->type);
544 return (NULL);
545 }
546
547 membar_consumer();
548 sdesc->msgConsIdx++;
549
550 return (msg);
551 }
552
553 static void
554 pvscsi_handle_msg(void *arg)
555 {
556 pvscsi_msg_t *msg = (pvscsi_msg_t *)arg;
557 dev_info_t *dip = msg->msg_pvs->dip;
558 int circ;
559
560 ndi_devi_enter(dip, &circ);
561 (void) pvscsi_config_one(dip, msg->msg_pvs, msg->target, NULL);
562 ndi_devi_exit(dip, circ);
563
564 kmem_free(msg, sizeof (pvscsi_msg_t));
565 }
566
567 static int
568 pvscsi_abort_cmd(pvscsi_cmd_t *cmd, pvscsi_cmd_t **pending)
569 {
570 pvscsi_softc_t *pvs = cmd->cmd_pvs;
571 pvscsi_cmd_t *c;
572 pvscsi_cmd_t *done;
573 struct PVSCSICmdDescAbortCmd acmd;
574
575 dev_err(pvs->dip, CE_WARN, "!aborting command %p", (void *)cmd);
576
577 ASSERT(mutex_owned(&pvs->rx_mutex));
578 ASSERT(mutex_owned(&pvs->tx_mutex));
579
580 /* Check if the cmd was already completed by the HBA */
581 *pending = done = pvscsi_process_comp_ring(pvs);
582 for (c = done; c != NULL; c = c->next_cmd) {
583 if (c == cmd)
584 return (CMD_CMPLT);
585 }
586
587 /* Check if cmd was really scheduled by the HBA */
588 if (pvscsi_lookup_ctx(pvs, cmd) == NULL)
589 return (CMD_CMPLT);
590
591 /* Abort cmd in the HBA */
592 bzero(&acmd, sizeof (acmd));
593 acmd.target = cmd->cmd_target;
594 acmd.context = pvscsi_map_ctx(pvs, cmd->ctx);
595 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ABORT_CMD, &acmd, sizeof (acmd));
596
597 /* Check if cmd was completed by the HBA before it could be aborted */
598 if ((done = pvscsi_process_comp_ring(pvs)) != NULL) {
599 done->tail_cmd->next_cmd = *pending;
600 *pending = done;
601 for (c = done; c != NULL; c = c->next_cmd) {
602 if (c == cmd)
603 return (CMD_CMPLT);
604 }
605 }
606
607 /* Release I/O ctx */
608 mutex_enter(&pvs->mutex);
609 if (cmd->ctx != NULL)
610 pvscsi_release_ctx(cmd);
611 /* Remove cmd from the queue */
612 pvscsi_remove_from_queue(cmd);
613 mutex_exit(&pvs->mutex);
614
615 /* Insert cmd at the beginning of the list */
616 cmd->next_cmd = *pending;
617 *pending = cmd;
618
619 dev_err(pvs->dip, CE_WARN, "!command %p aborted", (void *)cmd);
620
621 return (CMD_ABORTED);
622 }
623
624 static void
625 pvscsi_map_buffers(pvscsi_cmd_t *cmd, struct PVSCSIRingReqDesc *rdesc)
626 {
627 int i;
628
629 ASSERT(cmd->ctx);
630 ASSERT(cmd->cmd_dmaccount > 0 && cmd->cmd_dmaccount <=
631 PVSCSI_MAX_SG_SIZE);
632
633 rdesc->dataLen = cmd->cmd_dma_count;
634 rdesc->dataAddr = 0;
635
636 if (cmd->cmd_dma_count == 0)
637 return;
638
639 if (cmd->cmd_dmaccount > 1) {
640 struct PVSCSISGElement *sgl = CMD_CTX_SGLIST_VA(cmd->ctx);
641
642 for (i = 0; i < cmd->cmd_dmaccount; i++) {
643 sgl[i].addr = cmd->cached_cookies[i].dmac_laddress;
644 sgl[i].length = cmd->cached_cookies[i].dmac_size;
645 sgl[i].flags = 0;
646 }
647 rdesc->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
648 rdesc->dataAddr = (uint64_t)CMD_CTX_SGLIST_PA(cmd->ctx);
649 } else {
650 rdesc->dataAddr = cmd->cached_cookies[0].dmac_laddress;
651 }
652 }
653
654 static void
655 pvscsi_comp_cmd(pvscsi_cmd_t *cmd, uint8_t status)
656 {
657 struct scsi_pkt *pkt = CMD2PKT(cmd);
658
659 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
660 STATE_GOT_STATUS);
661 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0)
662 pkt->pkt_state |= STATE_XFERRED_DATA;
663 pkt->pkt_reason = CMD_CMPLT;
664 pkt->pkt_resid = 0;
665 *(pkt->pkt_scbp) = status;
666 }
667
668 static void
669 pvscsi_set_status(pvscsi_cmd_t *cmd)
670 {
671 pvscsi_softc_t *pvs = cmd->cmd_pvs;
672 struct scsi_pkt *pkt = CMD2PKT(cmd);
673 uchar_t scsi_status = cmd->cmp_stat.scsi_status;
674 uint32_t host_status = cmd->cmp_stat.host_status;
675
676 if (scsi_status != STATUS_GOOD &&
677 (host_status == BTSTAT_SUCCESS ||
678 (host_status == BTSTAT_LINKED_COMMAND_COMPLETED) ||
679 (host_status == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG))) {
680 if (scsi_status == STATUS_CHECK) {
681 struct scsi_arq_status *astat = (void*)(pkt->pkt_scbp);
682 uint8_t *sensedata;
683 int arq_size;
684
685 *pkt->pkt_scbp = scsi_status;
686 pkt->pkt_state |= STATE_ARQ_DONE;
687
688 if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
689 arq_size = (cmd->cmd_rqslen >=
690 SENSE_BUFFER_SIZE) ? SENSE_BUFFER_SIZE :
691 cmd->cmd_rqslen;
692
693 astat->sts_rqpkt_resid = SENSE_BUFFER_SIZE -
694 arq_size;
695 sensedata = (uint8_t *)&astat->sts_sensedata;
696 bcopy(cmd->arqbuf->b_un.b_addr, sensedata,
697 arq_size);
698
699 pkt->pkt_state |= STATE_XARQ_DONE;
700 } else {
701 astat->sts_rqpkt_resid = 0;
702 }
703
704 astat->sts_rqpkt_statistics = 0;
705 astat->sts_rqpkt_reason = CMD_CMPLT;
706 (*(uint8_t *)&astat->sts_rqpkt_status) = STATUS_GOOD;
707 astat->sts_rqpkt_state = STATE_GOT_BUS |
708 STATE_GOT_TARGET | STATE_SENT_CMD |
709 STATE_XFERRED_DATA | STATE_GOT_STATUS;
710 }
711 pvscsi_comp_cmd(cmd, scsi_status);
712
713 return;
714 }
715
716 switch (host_status) {
717 case BTSTAT_SUCCESS:
718 case BTSTAT_LINKED_COMMAND_COMPLETED:
719 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
720 pvscsi_comp_cmd(cmd, STATUS_GOOD);
721 break;
722 case BTSTAT_DATARUN:
723 pkt->pkt_reason = CMD_DATA_OVR;
724 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
725 STATE_SENT_CMD | STATE_GOT_STATUS |
726 STATE_XFERRED_DATA);
727 pkt->pkt_resid = 0;
728 break;
729 case BTSTAT_DATA_UNDERRUN:
730 pkt->pkt_reason = pkt->pkt_state |= (STATE_GOT_BUS |
731 STATE_GOT_TARGET | STATE_SENT_CMD | STATE_GOT_STATUS);
732 pkt->pkt_resid = cmd->dma_count - cmd->cmp_stat.data_len;
733 if (pkt->pkt_resid != cmd->dma_count)
734 pkt->pkt_state |= STATE_XFERRED_DATA;
735 break;
736 case BTSTAT_SELTIMEO:
737 pkt->pkt_reason = CMD_DEV_GONE;
738 pkt->pkt_state |= STATE_GOT_BUS;
739 break;
740 case BTSTAT_TAGREJECT:
741 pkt->pkt_reason = CMD_TAG_REJECT;
742 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
743 STATE_SENT_CMD | STATE_GOT_STATUS);
744 break;
745 case BTSTAT_BADMSG:
746 pkt->pkt_reason = CMD_BADMSG;
747 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
748 STATE_SENT_CMD | STATE_GOT_STATUS);
749 break;
750 case BTSTAT_SENTRST:
751 case BTSTAT_RECVRST:
752 case BTSTAT_BUSRESET:
753 pkt->pkt_reason = CMD_RESET;
754 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
755 STATE_SENT_CMD | STATE_GOT_STATUS);
756 break;
757 case BTSTAT_ABORTQUEUE:
758 pkt->pkt_reason = CMD_ABORTED;
759 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
760 STATE_SENT_CMD | STATE_GOT_STATUS);
761 break;
762 case BTSTAT_HAHARDWARE:
763 case BTSTAT_INVPHASE:
764 case BTSTAT_HATIMEOUT:
765 case BTSTAT_NORESPONSE:
766 case BTSTAT_DISCONNECT:
767 case BTSTAT_HASOFTWARE:
768 case BTSTAT_BUSFREE:
769 case BTSTAT_SENSFAILED:
770 pkt->pkt_reason = CMD_TRAN_ERR;
771 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
772 STATE_SENT_CMD | STATE_GOT_STATUS);
773 break;
774 default:
775 dev_err(pvs->dip, CE_WARN,
776 "!unknown host status code: %d", host_status);
777 pkt->pkt_reason = CMD_TRAN_ERR;
778 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
779 STATE_SENT_CMD | STATE_GOT_STATUS);
780 break;
781 }
782 }
783
784 static void
785 pvscsi_complete_chained(void *arg)
786 {
787 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)arg;
788 pvscsi_cmd_t *c;
789 struct scsi_pkt *pkt;
790
791 while (cmd != NULL) {
792 pvscsi_softc_t *pvs = cmd->cmd_pvs;
793
794 c = cmd->next_cmd;
795 cmd->next_cmd = NULL;
796
797 pkt = CMD2PKT(cmd);
798 if (pkt == NULL)
799 return;
800
801 if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0 &&
802 (cmd->flags & PVSCSI_FLAG_IO_READ) != 0) {
803 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
804 DDI_DMA_SYNC_FORCPU);
805 }
806
807 mutex_enter(&pvs->mutex);
808 /* Release I/O ctx */
809 if (cmd->ctx != NULL)
810 pvscsi_release_ctx(cmd);
811 /* Remove command from queue */
812 pvscsi_remove_from_queue(cmd);
813 mutex_exit(&pvs->mutex);
814
815 if ((cmd->flags & PVSCSI_FLAG_HW_STATUS) != 0) {
816 pvscsi_set_status(cmd);
817 } else {
818 ASSERT((cmd->flags & PVSCSI_FLAGS_NON_HW_COMPLETION) !=
819 0);
820
821 if ((cmd->flags & PVSCSI_FLAG_TIMED_OUT) != 0) {
822 cmd->pkt->pkt_reason = CMD_TIMEOUT;
823 cmd->pkt->pkt_statistics |=
824 (STAT_TIMEOUT | STAT_ABORTED);
825 } else if ((cmd->flags & PVSCSI_FLAG_ABORTED) != 0) {
826 cmd->pkt->pkt_reason = CMD_ABORTED;
827 cmd->pkt->pkt_statistics |=
828 (STAT_TIMEOUT | STAT_ABORTED);
829 } else if ((cmd->flags & PVSCSI_FLAGS_RESET) != 0) {
830 cmd->pkt->pkt_reason = CMD_RESET;
831 if ((cmd->flags & PVSCSI_FLAG_RESET_BUS) != 0) {
832 cmd->pkt->pkt_statistics |=
833 STAT_BUS_RESET;
834 } else {
835 cmd->pkt->pkt_statistics |=
836 STAT_DEV_RESET;
837 }
838 }
839 }
840
841 cmd->flags |= PVSCSI_FLAG_DONE;
842 cmd->flags &= ~PVSCSI_FLAG_TRANSPORT;
843
844 if ((pkt->pkt_flags & FLAG_NOINTR) == 0 &&
845 pkt->pkt_comp != NULL)
846 (*pkt->pkt_comp)(pkt);
847
848 cmd = c;
849 }
850 }
851
852 static void
853 pvscsi_dev_reset(pvscsi_softc_t *pvs, int target)
854 {
855 struct PVSCSICmdDescResetDevice cmd = { 0 };
856
857 cmd.target = target;
858 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof (cmd));
859 }
860
861 static int
862 pvscsi_poll_cmd(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
863 {
864 boolean_t seen_intr;
865 int cycles = (cmd->pkt->pkt_time * 1000000) / USECS_TO_WAIT;
866 int i;
867 pvscsi_cmd_t *dcmd;
868 struct scsi_pkt *pkt = CMD2PKT(cmd);
869
870 /*
871 * Make sure we're not missing any commands completed
872 * concurrently before we have actually disabled interrupts.
873 */
874 mutex_enter(&pvs->rx_mutex);
875 dcmd = pvscsi_process_comp_ring(pvs);
876 mutex_exit(&pvs->rx_mutex);
877
878 pvscsi_complete_chained(dcmd);
879
880 while ((cmd->flags & PVSCSI_FLAG_DONE) == 0) {
881 seen_intr = B_FALSE;
882
883 /* Disable interrupts from H/W */
884 pvscsi_mask_intr(pvs);
885
886 /* Wait for interrupt to arrive */
887 for (i = 0; i < cycles; i++) {
888 uint32_t status;
889
890 mutex_enter(&pvs->rx_mutex);
891 mutex_enter(&pvs->intr_mutex);
892 status = pvscsi_read_intr_status(pvs);
893 if ((status & PVSCSI_INTR_ALL_SUPPORTED) != 0) {
894 /* Check completion ring */
895 mutex_exit(&pvs->intr_mutex);
896 dcmd = pvscsi_process_comp_ring(pvs);
897 mutex_exit(&pvs->rx_mutex);
898 seen_intr = B_TRUE;
899 break;
900 } else {
901 mutex_exit(&pvs->intr_mutex);
902 mutex_exit(&pvs->rx_mutex);
903 drv_usecwait(USECS_TO_WAIT);
904 }
905 }
906
907 /* Enable interrupts from H/W */
908 pvscsi_unmask_intr(pvs);
909
910 if (!seen_intr) {
911 /* No interrupts seen from device during the timeout */
912 mutex_enter(&pvs->tx_mutex);
913 mutex_enter(&pvs->rx_mutex);
914 if ((cmd->flags & PVSCSI_FLAGS_COMPLETION) != 0) {
915 /* Command was cancelled asynchronously */
916 dcmd = NULL;
917 } else if ((pvscsi_abort_cmd(cmd,
918 &dcmd)) == CMD_ABORTED) {
919 /* Command was cancelled in hardware */
920 pkt->pkt_state |= (STAT_TIMEOUT | STAT_ABORTED);
921 pkt->pkt_statistics |= (STAT_TIMEOUT |
922 STAT_ABORTED);
923 pkt->pkt_reason = CMD_TIMEOUT;
924 }
925 mutex_exit(&pvs->rx_mutex);
926 mutex_exit(&pvs->tx_mutex);
927
928 /*
929 * Complete commands that might be on completion list.
930 * Target command can also be on the list in case it was
931 * completed before it could be actually cancelled.
932 */
933 break;
934 }
935
936 pvscsi_complete_chained(dcmd);
937
938 if (!seen_intr)
939 break;
940 }
941
942 return (TRAN_ACCEPT);
943 }
944
945 static void
946 pvscsi_abort_all(struct scsi_address *ap, pvscsi_softc_t *pvs,
947 pvscsi_cmd_t **pending, int marker_flag)
948 {
949 int qlen = pvs->cmd_queue_len;
950 pvscsi_cmd_t *cmd, *pcmd, *phead = NULL;
951
952 ASSERT(mutex_owned(&pvs->rx_mutex));
953 ASSERT(mutex_owned(&pvs->tx_mutex));
954
955 /*
956 * Try to abort all queued commands, merging commands waiting
957 * for completion into a single list to complete them at one
958 * time when mutex is released.
959 */
960 while (qlen > 0) {
961 mutex_enter(&pvs->mutex);
962 cmd = list_remove_head(&pvs->cmd_queue);
963 ASSERT(cmd != NULL);
964
965 qlen--;
966
967 if (ap == NULL || ap->a_target == cmd->cmd_target) {
968 int c = --pvs->cmd_queue_len;
969
970 mutex_exit(&pvs->mutex);
971
972 if (pvscsi_abort_cmd(cmd, &pcmd) == CMD_ABORTED) {
973 /*
974 * Assume command is completely cancelled now,
975 * so mark it as requested.
976 */
977 cmd->flags |= marker_flag;
978 }
979
980 qlen -= (c - pvs->cmd_queue_len);
981
982 /*
983 * Now merge current pending commands with
984 * previous ones.
985 */
986 if (phead == NULL) {
987 phead = pcmd;
988 } else if (pcmd != NULL) {
989 phead->tail_cmd->next_cmd = pcmd;
990 phead->tail_cmd = pcmd->tail_cmd;
991 }
992 } else {
993 list_insert_tail(&pvs->cmd_queue, cmd);
994 mutex_exit(&pvs->mutex);
995 }
996 }
997
998 *pending = phead;
999 }
1000
1001 static void
1002 pvscsi_quiesce_notify(pvscsi_softc_t *pvs)
1003 {
1004 mutex_enter(&pvs->mutex);
1005 if (pvs->cmd_queue_len == 0 &&
1006 (pvs->flags & PVSCSI_HBA_QUIESCE_PENDING) != 0) {
1007 pvs->flags &= ~PVSCSI_HBA_QUIESCE_PENDING;
1008 cv_broadcast(&pvs->quiescevar);
1009 }
1010 mutex_exit(&pvs->mutex);
1011 }
1012
1013 static int
1014 pvscsi_transport_command(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
1015 {
1016 struct PVSCSIRingReqDesc *rdesc;
1017 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
1018 struct scsi_pkt *pkt = CMD2PKT(cmd);
1019 uint32_t req_ne = sdesc->reqNumEntriesLog2;
1020
1021 mutex_enter(&pvs->tx_mutex);
1022 mutex_enter(&pvs->mutex);
1023 if (!pvscsi_acquire_ctx(pvs, cmd)) {
1024 mutex_exit(&pvs->mutex);
1025 mutex_exit(&pvs->tx_mutex);
1026 dev_err(pvs->dip, CE_WARN, "!no free ctx available");
1027 return (TRAN_BUSY);
1028 }
1029
1030 if ((sdesc->reqProdIdx - sdesc->cmpConsIdx) >= (1 << req_ne)) {
1031 pvscsi_release_ctx(cmd);
1032 mutex_exit(&pvs->mutex);
1033 mutex_exit(&pvs->tx_mutex);
1034 dev_err(pvs->dip, CE_WARN, "!no free I/O slots available");
1035 return (TRAN_BUSY);
1036 }
1037 mutex_exit(&pvs->mutex);
1038
1039 cmd->flags |= PVSCSI_FLAG_TRANSPORT;
1040
1041 rdesc = REQ_RING(pvs) + (sdesc->reqProdIdx & MASK(req_ne));
1042
1043 bzero(&rdesc->lun, sizeof (rdesc->lun));
1044
1045 rdesc->bus = 0;
1046 rdesc->target = cmd->cmd_target;
1047
1048 if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
1049 bzero((void*)cmd->arqbuf->b_un.b_addr, SENSE_BUFFER_SIZE);
1050 rdesc->senseLen = SENSE_BUFFER_SIZE;
1051 rdesc->senseAddr = cmd->arqc.dmac_laddress;
1052 } else {
1053 rdesc->senseLen = 0;
1054 rdesc->senseAddr = 0;
1055 }
1056
1057 rdesc->vcpuHint = CPU->cpu_id;
1058 rdesc->cdbLen = cmd->cmdlen;
1059 bcopy(cmd->cmd_cdb, rdesc->cdb, cmd->cmdlen);
1060
1061 /* Setup tag info */
1062 if ((cmd->flags & PVSCSI_FLAG_TAG) != 0)
1063 rdesc->tag = cmd->tag;
1064 else
1065 rdesc->tag = MSG_SIMPLE_QTAG;
1066
1067 /* Setup I/O direction and map data buffers */
1068 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1069 if ((cmd->flags & PVSCSI_FLAG_IO_READ) != 0)
1070 rdesc->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
1071 else
1072 rdesc->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
1073 pvscsi_map_buffers(cmd, rdesc);
1074 } else {
1075 rdesc->flags = 0;
1076 }
1077
1078 rdesc->context = pvscsi_map_ctx(pvs, cmd->ctx);
1079 membar_producer();
1080
1081 sdesc->reqProdIdx++;
1082 membar_producer();
1083
1084 mutex_enter(&pvs->mutex);
1085 cmd->timeout_lbolt = ddi_get_lbolt() + SEC_TO_TICK(pkt->pkt_time);
1086 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD);
1087 pvscsi_add_to_queue(cmd);
1088
1089 switch (cmd->pkt->pkt_cdbp[0]) {
1090 case SCMD_READ:
1091 case SCMD_WRITE:
1092 case SCMD_READ_G1:
1093 case SCMD_WRITE_G1:
1094 case SCMD_READ_G4:
1095 case SCMD_WRITE_G4:
1096 case SCMD_READ_G5:
1097 case SCMD_WRITE_G5:
1098 ASSERT((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0);
1099 pvscsi_submit_rw_io(pvs);
1100 break;
1101 default:
1102 pvscsi_submit_nonrw_io(pvs);
1103 break;
1104 }
1105 mutex_exit(&pvs->mutex);
1106 mutex_exit(&pvs->tx_mutex);
1107
1108 return (TRAN_ACCEPT);
1109 }
1110
1111 static int
1112 pvscsi_reset_generic(pvscsi_softc_t *pvs, struct scsi_address *ap)
1113 {
1114 boolean_t bus_reset = (ap == NULL);
1115 int flags;
1116 pvscsi_cmd_t *done, *aborted;
1117
1118 flags = bus_reset ? PVSCSI_FLAG_RESET_BUS : PVSCSI_FLAG_RESET_DEV;
1119
1120 mutex_enter(&pvs->tx_mutex);
1121 mutex_enter(&pvs->rx_mutex);
1122 /* Try to process pending requests */
1123 done = pvscsi_process_comp_ring(pvs);
1124
1125 /* Abort all pending requests */
1126 pvscsi_abort_all(ap, pvs, &aborted, flags);
1127
1128 /* Reset at hardware level */
1129 if (bus_reset) {
1130 pvscsi_reset_bus(pvs);
1131 /* Should never happen after bus reset */
1132 ASSERT(pvscsi_process_comp_ring(pvs) == NULL);
1133 } else {
1134 pvscsi_dev_reset(pvs, ap->a_target);
1135 }
1136 mutex_exit(&pvs->rx_mutex);
1137 mutex_exit(&pvs->tx_mutex);
1138
1139 pvscsi_complete_chained(done);
1140 pvscsi_complete_chained(aborted);
1141
1142 return (1);
1143 }
1144
1145 static void
1146 pvscsi_cmd_ext_free(pvscsi_cmd_t *cmd)
1147 {
1148 struct scsi_pkt *pkt = CMD2PKT(cmd);
1149
1150 if ((cmd->flags & PVSCSI_FLAG_CDB_EXT) != 0) {
1151 kmem_free(pkt->pkt_cdbp, cmd->cmdlen);
1152 cmd->flags &= ~PVSCSI_FLAG_CDB_EXT;
1153 }
1154 if ((cmd->flags & PVSCSI_FLAG_SCB_EXT) != 0) {
1155 kmem_free(pkt->pkt_scbp, cmd->statuslen);
1156 cmd->flags &= ~PVSCSI_FLAG_SCB_EXT;
1157 }
1158 if ((cmd->flags & PVSCSI_FLAG_PRIV_EXT) != 0) {
1159 kmem_free(pkt->pkt_private, cmd->tgtlen);
1160 cmd->flags &= ~PVSCSI_FLAG_PRIV_EXT;
1161 }
1162 }
1163
1164 /* ARGSUSED pvs */
1165 static int
1166 pvscsi_cmd_ext_alloc(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd, int kf)
1167 {
1168 struct scsi_pkt *pkt = CMD2PKT(cmd);
1169 void *buf;
1170
1171 if (cmd->cmdlen > sizeof (cmd->cmd_cdb)) {
1172 if ((buf = kmem_zalloc(cmd->cmdlen, kf)) == NULL)
1173 return (DDI_FAILURE);
1174 pkt->pkt_cdbp = buf;
1175 cmd->flags |= PVSCSI_FLAG_CDB_EXT;
1176 }
1177
1178 if (cmd->statuslen > sizeof (cmd->cmd_scb)) {
1179 if ((buf = kmem_zalloc(cmd->statuslen, kf)) == NULL)
1180 goto out;
1181 pkt->pkt_scbp = buf;
1182 cmd->flags |= PVSCSI_FLAG_SCB_EXT;
1183 cmd->cmd_rqslen = (cmd->statuslen - sizeof (cmd->cmd_scb));
1184 }
1185
1186 if (cmd->tgtlen > sizeof (cmd->tgt_priv)) {
1187 if ((buf = kmem_zalloc(cmd->tgtlen, kf)) == NULL)
1188 goto out;
1189 pkt->pkt_private = buf;
1190 cmd->flags |= PVSCSI_FLAG_PRIV_EXT;
1191 }
1192
1193 return (DDI_SUCCESS);
1194
1195 out:
1196 pvscsi_cmd_ext_free(cmd);
1197
1198 return (DDI_FAILURE);
1199 }
1200
1201 static int
1202 pvscsi_setup_dma_buffer(pvscsi_softc_t *pvs, size_t length,
1203 pvscsi_dma_buf_t *buf)
1204 {
1205 ddi_dma_cookie_t cookie;
1206 uint_t ccount;
1207
1208 if ((ddi_dma_alloc_handle(pvs->dip, &pvscsi_ring_dma_attr,
1209 DDI_DMA_SLEEP, NULL, &buf->dma_handle)) != DDI_SUCCESS) {
1210 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1211 return (DDI_FAILURE);
1212 }
1213
1214 if ((ddi_dma_mem_alloc(buf->dma_handle, length, &pvscsi_dma_attrs,
1215 DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &buf->addr,
1216 &buf->real_length, &buf->acc_handle)) != DDI_SUCCESS) {
1217 dev_err(pvs->dip, CE_WARN,
1218 "!failed to allocate %ld bytes for DMA buffer", length);
1219 ddi_dma_free_handle(&buf->dma_handle);
1220 return (DDI_FAILURE);
1221 }
1222
1223 if ((ddi_dma_addr_bind_handle(buf->dma_handle, NULL, buf->addr,
1224 buf->real_length, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, DDI_DMA_SLEEP,
1225 NULL, &cookie, &ccount)) != DDI_SUCCESS) {
1226 dev_err(pvs->dip, CE_WARN, "!failed to bind DMA buffer");
1227 ddi_dma_free_handle(&buf->dma_handle);
1228 ddi_dma_mem_free(&buf->acc_handle);
1229 return (DDI_FAILURE);
1230 }
1231
1232 /* TODO Support multipart SG regions */
1233 ASSERT(ccount == 1);
1234
1235 buf->pa = cookie.dmac_laddress;
1236
1237 return (DDI_SUCCESS);
1238 }
1239
1240 static void
1241 pvscsi_free_dma_buffer(pvscsi_dma_buf_t *buf)
1242 {
1243 ddi_dma_free_handle(&buf->dma_handle);
1244 ddi_dma_mem_free(&buf->acc_handle);
1245 }
1246
1247 static int
1248 pvscsi_setup_sg(pvscsi_softc_t *pvs)
1249 {
1250 int i;
1251 pvscsi_cmd_ctx_t *ctx;
1252 size_t size = pvs->req_depth * sizeof (pvscsi_cmd_ctx_t);
1253
1254 ctx = pvs->cmd_ctx = kmem_zalloc(size, KM_SLEEP);
1255
1256 for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1257 list_insert_tail(&pvs->cmd_ctx_pool, ctx);
1258 if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1259 &ctx->dma_buf) != DDI_SUCCESS)
1260 goto cleanup;
1261 }
1262
1263 return (DDI_SUCCESS);
1264
1265 cleanup:
1266 for (; i >= 0; --i, --ctx) {
1267 list_remove(&pvs->cmd_ctx_pool, ctx);
1268 pvscsi_free_dma_buffer(&ctx->dma_buf);
1269 }
1270 kmem_free(pvs->cmd_ctx, size);
1271
1272 return (DDI_FAILURE);
1273 }
1274
1275 static void
1276 pvscsi_free_sg(pvscsi_softc_t *pvs)
1277 {
1278 int i;
1279 pvscsi_cmd_ctx_t *ctx = pvs->cmd_ctx;
1280
1281 for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1282 list_remove(&pvs->cmd_ctx_pool, ctx);
1283 pvscsi_free_dma_buffer(&ctx->dma_buf);
1284 }
1285
1286 kmem_free(pvs->cmd_ctx, pvs->req_pages << PAGE_SHIFT);
1287 }
1288
1289 static int
1290 pvscsi_allocate_rings(pvscsi_softc_t *pvs)
1291 {
1292 /* Allocate DMA buffer for rings state */
1293 if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1294 &pvs->rings_state_buf) != DDI_SUCCESS)
1295 return (DDI_FAILURE);
1296
1297 /* Allocate DMA buffer for request ring */
1298 pvs->req_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_REQ_RING);
1299 pvs->req_depth = pvs->req_pages * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
1300 if (pvscsi_setup_dma_buffer(pvs, pvs->req_pages * PAGE_SIZE,
1301 &pvs->req_ring_buf) != DDI_SUCCESS)
1302 goto free_rings_state;
1303
1304 /* Allocate completion ring */
1305 pvs->cmp_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_CMP_RING);
1306 if (pvscsi_setup_dma_buffer(pvs, pvs->cmp_pages * PAGE_SIZE,
1307 &pvs->cmp_ring_buf) != DDI_SUCCESS)
1308 goto free_req_buf;
1309
1310 /* Allocate message ring */
1311 pvs->msg_pages = MIN(pvscsi_msg_ring_pages,
1312 PVSCSI_MAX_NUM_PAGES_MSG_RING);
1313 if (pvscsi_setup_dma_buffer(pvs, pvs->msg_pages * PAGE_SIZE,
1314 &pvs->msg_ring_buf) != DDI_SUCCESS)
1315 goto free_cmp_buf;
1316
1317 return (DDI_SUCCESS);
1318
1319 free_cmp_buf:
1320 pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1321 free_req_buf:
1322 pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1323 free_rings_state:
1324 pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1325
1326 return (DDI_FAILURE);
1327 }
1328
1329 static void
1330 pvscsi_free_rings(pvscsi_softc_t *pvs)
1331 {
1332 pvscsi_free_dma_buffer(&pvs->msg_ring_buf);
1333 pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1334 pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1335 pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1336 }
1337
1338 static void
1339 pvscsi_setup_rings(pvscsi_softc_t *pvs)
1340 {
1341 int i;
1342 struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
1343 struct PVSCSICmdDescSetupRings cmd = { 0 };
1344 uint64_t base;
1345
1346 cmd.ringsStatePPN = pvs->rings_state_buf.pa >> PAGE_SHIFT;
1347 cmd.reqRingNumPages = pvs->req_pages;
1348 cmd.cmpRingNumPages = pvs->cmp_pages;
1349
1350 /* Setup request ring */
1351 base = pvs->req_ring_buf.pa;
1352 for (i = 0; i < pvs->req_pages; i++) {
1353 cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
1354 base += PAGE_SIZE;
1355 }
1356
1357 /* Setup completion ring */
1358 base = pvs->cmp_ring_buf.pa;
1359 for (i = 0; i < pvs->cmp_pages; i++) {
1360 cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
1361 base += PAGE_SIZE;
1362 }
1363
1364 bzero(RINGS_STATE(pvs), PAGE_SIZE);
1365 bzero(REQ_RING(pvs), pvs->req_pages * PAGE_SIZE);
1366 bzero(CMP_RING(pvs), pvs->cmp_pages * PAGE_SIZE);
1367
1368 /* Issue SETUP command */
1369 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof (cmd));
1370
1371 /* Setup message ring */
1372 cmd_msg.numPages = pvs->msg_pages;
1373 base = pvs->msg_ring_buf.pa;
1374
1375 for (i = 0; i < pvs->msg_pages; i++) {
1376 cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
1377 base += PAGE_SIZE;
1378 }
1379 bzero(MSG_RING(pvs), pvs->msg_pages * PAGE_SIZE);
1380
1381 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_MSG_RING, &cmd_msg,
1382 sizeof (cmd_msg));
1383 }
1384
1385 static int
1386 pvscsi_setup_io(pvscsi_softc_t *pvs)
1387 {
1388 int offset, rcount, rn, type;
1389 int ret = DDI_FAILURE;
1390 off_t regsize;
1391 pci_regspec_t *regs;
1392 uint_t regs_length;
1393
1394 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, pvs->dip,
1395 DDI_PROP_DONTPASS, "reg", (int **)®s,
1396 ®s_length) != DDI_PROP_SUCCESS) {
1397 dev_err(pvs->dip, CE_WARN, "!failed to lookup 'reg' property");
1398 return (DDI_FAILURE);
1399 }
1400
1401 rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t);
1402
1403 for (offset = PCI_CONF_BASE0; offset <= PCI_CONF_BASE5; offset += 4) {
1404 for (rn = 0; rn < rcount; ++rn) {
1405 if (PCI_REG_REG_G(regs[rn].pci_phys_hi) == offset) {
1406 type = regs[rn].pci_phys_hi & PCI_ADDR_MASK;
1407 break;
1408 }
1409 }
1410
1411 if (rn >= rcount)
1412 continue;
1413
1414 if (type != PCI_ADDR_IO) {
1415 if (ddi_dev_regsize(pvs->dip, rn,
1416 ®size) != DDI_SUCCESS) {
1417 dev_err(pvs->dip, CE_WARN,
1418 "!failed to get size of reg %d", rn);
1419 goto out;
1420 }
1421 if (regsize == PVSCSI_MEM_SPACE_SIZE) {
1422 if (ddi_regs_map_setup(pvs->dip, rn,
1423 &pvs->mmio_base, 0, 0,
1424 &pvscsi_mmio_attr,
1425 &pvs->mmio_handle) != DDI_SUCCESS) {
1426 dev_err(pvs->dip, CE_WARN,
1427 "!failed to map MMIO BAR");
1428 goto out;
1429 }
1430 ret = DDI_SUCCESS;
1431 break;
1432 }
1433 }
1434 }
1435
1436 out:
1437 ddi_prop_free(regs);
1438
1439 return (ret);
1440 }
1441
1442 static void
1443 pvscsi_free_io(pvscsi_softc_t *pvs)
1444 {
1445 ddi_regs_map_free(&pvs->mmio_handle);
1446 }
1447
1448 static int
1449 pvscsi_enable_intrs(pvscsi_softc_t *pvs)
1450 {
1451 int i, rc, intr_caps;
1452
1453 if ((rc = ddi_intr_get_cap(pvs->intr_htable[0], &intr_caps)) !=
1454 DDI_SUCCESS) {
1455 dev_err(pvs->dip, CE_WARN, "!failed to get interrupt caps");
1456 return (DDI_FAILURE);
1457 }
1458
1459 if ((intr_caps & DDI_INTR_FLAG_BLOCK) != 0) {
1460 if ((rc = ddi_intr_block_enable(pvs->intr_htable,
1461 pvs->intr_cnt)) != DDI_SUCCESS) {
1462 dev_err(pvs->dip, CE_WARN,
1463 "!failed to enable interrupt block");
1464 }
1465 } else {
1466 for (i = 0; i < pvs->intr_cnt; i++) {
1467 if ((rc = ddi_intr_enable(pvs->intr_htable[i])) ==
1468 DDI_SUCCESS)
1469 continue;
1470 dev_err(pvs->dip, CE_WARN,
1471 "!failed to enable interrupt");
1472 while (--i >= 0)
1473 (void) ddi_intr_disable(pvs->intr_htable[i]);
1474 break;
1475 }
1476 }
1477
1478 /* Unmask interrupts */
1479 if (rc == DDI_SUCCESS) {
1480 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
1481 PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
1482 }
1483
1484 return (rc);
1485 }
1486
1487 /* ARGSUSED arg2 */
1488 static uint32_t
1489 pvscsi_intr_handler(caddr_t arg1, caddr_t arg2)
1490 {
1491 boolean_t handled;
1492 pvscsi_softc_t *pvs = (pvscsi_softc_t *)arg1;
1493 uint32_t status;
1494
1495 mutex_enter(&pvs->intr_mutex);
1496 if (pvs->num_pollers > 0) {
1497 mutex_exit(&pvs->intr_mutex);
1498 return (DDI_INTR_CLAIMED);
1499 }
1500
1501 if (pvscsi_enable_msi) {
1502 handled = B_TRUE;
1503 } else {
1504 status = pvscsi_read_intr_status(pvs);
1505 handled = (status & PVSCSI_INTR_ALL_SUPPORTED) != 0;
1506 if (handled)
1507 pvscsi_write_intr_status(pvs, status);
1508 }
1509 mutex_exit(&pvs->intr_mutex);
1510
1511 if (handled) {
1512 boolean_t qnotify;
1513 pvscsi_cmd_t *pending;
1514 pvscsi_msg_t *msg;
1515
1516 mutex_enter(&pvs->rx_mutex);
1517 pending = pvscsi_process_comp_ring(pvs);
1518 msg = pvscsi_process_msg_ring(pvs);
1519 mutex_exit(&pvs->rx_mutex);
1520
1521 mutex_enter(&pvs->mutex);
1522 qnotify = HBA_QUIESCE_PENDING(pvs);
1523 mutex_exit(&pvs->mutex);
1524
1525 if (pending != NULL && ddi_taskq_dispatch(pvs->comp_tq,
1526 pvscsi_complete_chained, pending,
1527 DDI_NOSLEEP) == DDI_FAILURE)
1528 pvscsi_complete_chained(pending);
1529
1530 if (msg != NULL && ddi_taskq_dispatch(pvs->msg_tq,
1531 pvscsi_handle_msg, msg, DDI_NOSLEEP) == DDI_FAILURE) {
1532 dev_err(pvs->dip, CE_WARN,
1533 "!failed to process msg type %d for target %d",
1534 msg->type, msg->target);
1535 kmem_free(msg, sizeof (pvscsi_msg_t));
1536 }
1537
1538 if (qnotify)
1539 pvscsi_quiesce_notify(pvs);
1540 }
1541
1542 return (handled ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
1543 }
1544
1545 static int
1546 pvscsi_register_isr(pvscsi_softc_t *pvs, int type)
1547 {
1548 int navail, nactual;
1549 int i;
1550
1551 if (ddi_intr_get_navail(pvs->dip, type, &navail) != DDI_SUCCESS ||
1552 navail == 0) {
1553 dev_err(pvs->dip, CE_WARN,
1554 "!failed to get number of available interrupts of type %d",
1555 type);
1556 return (DDI_FAILURE);
1557 }
1558 navail = MIN(navail, PVSCSI_MAX_INTRS);
1559
1560 pvs->intr_size = navail * sizeof (ddi_intr_handle_t);
1561 if ((pvs->intr_htable = kmem_alloc(pvs->intr_size, KM_SLEEP)) == NULL) {
1562 dev_err(pvs->dip, CE_WARN,
1563 "!failed to allocate %d bytes for interrupt hashtable",
1564 pvs->intr_size);
1565 return (DDI_FAILURE);
1566 }
1567
1568 if (ddi_intr_alloc(pvs->dip, pvs->intr_htable, type, 0, navail,
1569 &nactual, DDI_INTR_ALLOC_NORMAL) != DDI_SUCCESS || nactual == 0) {
1570 dev_err(pvs->dip, CE_WARN, "!failed to allocate %d interrupts",
1571 navail);
1572 goto free_htable;
1573 }
1574
1575 pvs->intr_cnt = nactual;
1576
1577 if (ddi_intr_get_pri(pvs->intr_htable[0],
1578 (uint_t *)&pvs->intr_pri) != DDI_SUCCESS) {
1579 dev_err(pvs->dip, CE_WARN, "!failed to get interrupt priority");
1580 goto free_intrs;
1581 }
1582
1583 for (i = 0; i < nactual; i++) {
1584 if (ddi_intr_add_handler(pvs->intr_htable[i],
1585 pvscsi_intr_handler, (caddr_t)pvs, NULL) != DDI_SUCCESS) {
1586 dev_err(pvs->dip, CE_WARN,
1587 "!failed to add interrupt handler");
1588 goto free_intrs;
1589 }
1590 }
1591
1592 return (DDI_SUCCESS);
1593
1594 free_intrs:
1595 for (i = 0; i < nactual; i++)
1596 (void) ddi_intr_free(pvs->intr_htable[i]);
1597 free_htable:
1598 kmem_free(pvs->intr_htable, pvs->intr_size);
1599
1600 return (DDI_FAILURE);
1601 }
1602
1603 static void
1604 pvscsi_free_intr_resources(pvscsi_softc_t *pvs)
1605 {
1606 int i;
1607
1608 for (i = 0; i < pvs->intr_cnt; i++) {
1609 (void) ddi_intr_disable(pvs->intr_htable[i]);
1610 (void) ddi_intr_remove_handler(pvs->intr_htable[i]);
1611 (void) ddi_intr_free(pvs->intr_htable[i]);
1612 }
1613 kmem_free(pvs->intr_htable, pvs->intr_size);
1614 }
1615
1616 static int
1617 pvscsi_setup_isr(pvscsi_softc_t *pvs)
1618 {
1619 int intr_types;
1620
1621 if (ddi_intr_get_supported_types(pvs->dip,
1622 &intr_types) != DDI_SUCCESS) {
1623 dev_err(pvs->dip, CE_WARN,
1624 "!failed to get supported interrupt types");
1625 return (DDI_FAILURE);
1626 }
1627
1628 if ((intr_types & DDI_INTR_TYPE_MSIX) != 0 && pvscsi_enable_msi) {
1629 if (pvscsi_register_isr(pvs,
1630 DDI_INTR_TYPE_MSIX) == DDI_SUCCESS) {
1631 pvs->intr_type = DDI_INTR_TYPE_MSIX;
1632 } else {
1633 dev_err(pvs->dip, CE_WARN,
1634 "!failed to install MSI-X interrupt handler");
1635 }
1636 } else if ((intr_types & DDI_INTR_TYPE_MSI) != 0 && pvscsi_enable_msi) {
1637 if (pvscsi_register_isr(pvs,
1638 DDI_INTR_TYPE_MSI) == DDI_SUCCESS) {
1639 pvs->intr_type = DDI_INTR_TYPE_MSI;
1640 } else {
1641 dev_err(pvs->dip, CE_WARN,
1642 "!failed to install MSI interrupt handler");
1643 }
1644 } else if ((intr_types & DDI_INTR_TYPE_FIXED) != 0) {
1645 if (pvscsi_register_isr(pvs,
1646 DDI_INTR_TYPE_FIXED) == DDI_SUCCESS) {
1647 pvs->intr_type = DDI_INTR_TYPE_FIXED;
1648 } else {
1649 dev_err(pvs->dip, CE_WARN,
1650 "!failed to install FIXED interrupt handler");
1651 }
1652 }
1653
1654 return (pvs->intr_type == 0 ? DDI_FAILURE : DDI_SUCCESS);
1655 }
1656
1657 static void
1658 pvscsi_wd_thread(pvscsi_softc_t *pvs)
1659 {
1660 clock_t now;
1661 pvscsi_cmd_t *expired, *c, *cn, **pnext;
1662
1663 mutex_enter(&pvs->mutex);
1664 for (;;) {
1665 expired = NULL;
1666 pnext = NULL;
1667 now = ddi_get_lbolt();
1668
1669 for (c = list_head(&pvs->cmd_queue); c != NULL; ) {
1670 cn = list_next(&pvs->cmd_queue, c);
1671
1672 /*
1673 * Commands with 'FLAG_NOINTR' are watched using their
1674 * own timeouts, so we should not touch them.
1675 */
1676 if ((c->pkt->pkt_flags & FLAG_NOINTR) == 0 &&
1677 now > c->timeout_lbolt) {
1678 dev_err(pvs->dip, CE_WARN,
1679 "!expired command: %p (%ld > %ld)",
1680 (void *)c, now, c->timeout_lbolt);
1681 pvscsi_remove_from_queue(c);
1682 if (expired == NULL)
1683 expired = c;
1684 if (pnext == NULL) {
1685 pnext = &c->next_cmd;
1686 } else {
1687 *pnext = c;
1688 pnext = &c->next_cmd;
1689 }
1690 }
1691 c = cn;
1692 }
1693 mutex_exit(&pvs->mutex);
1694
1695 /* Now cancel all expired commands */
1696 if (expired != NULL) {
1697 struct scsi_address sa = {0};
1698 /* Build a fake SCSI address */
1699 sa.a_hba_tran = pvs->tran;
1700 while (expired != NULL) {
1701 c = expired->next_cmd;
1702 sa.a_target = expired->cmd_target;
1703 sa.a_lun = 0;
1704 (void) pvscsi_abort(&sa, CMD2PKT(expired));
1705 expired = c;
1706 }
1707 }
1708
1709 mutex_enter(&pvs->mutex);
1710 if ((pvs->flags & PVSCSI_DRIVER_SHUTDOWN) != 0) {
1711 /* Finish job */
1712 break;
1713 }
1714 if (cv_reltimedwait(&pvs->wd_condvar, &pvs->mutex,
1715 SEC_TO_TICK(1), TR_CLOCK_TICK) > 0) {
1716 /* Explicitly woken up, finish job */
1717 break;
1718 }
1719 }
1720
1721 /* Confirm thread termination */
1722 cv_signal(&pvs->syncvar);
1723 mutex_exit(&pvs->mutex);
1724 }
1725
1726 static int
1727 pvscsi_ccache_constructor(void *buf, void *cdrarg, int kmflags)
1728 {
1729 int (*callback)(caddr_t);
1730 uint_t cookiec;
1731 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)buf;
1732 pvscsi_softc_t *pvs = cdrarg;
1733 struct scsi_address ap;
1734
1735 callback = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
1736 ap.a_hba_tran = pvs->tran;
1737 ap.a_target = 0;
1738 ap.a_lun = 0;
1739
1740 /* Allocate a DMA handle for data transfers */
1741 if ((ddi_dma_alloc_handle(pvs->dip, &pvs->io_dma_attr, callback,
1742 NULL, &cmd->cmd_dmahdl)) != DDI_SUCCESS) {
1743 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1744 return (-1);
1745 }
1746
1747 /* Setup ARQ buffer */
1748 if ((cmd->arqbuf = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL,
1749 SENSE_BUFFER_SIZE, B_READ, callback, NULL)) == NULL) {
1750 dev_err(pvs->dip, CE_WARN, "!failed to allocate ARQ buffer");
1751 goto free_handle;
1752 }
1753
1754 if (ddi_dma_alloc_handle(pvs->dip, &pvs->hba_dma_attr,
1755 callback, NULL, &cmd->arqhdl) != DDI_SUCCESS) {
1756 dev_err(pvs->dip, CE_WARN,
1757 "!failed to allocate DMA handle for ARQ buffer");
1758 goto free_arqbuf;
1759 }
1760
1761 if (ddi_dma_buf_bind_handle(cmd->arqhdl, cmd->arqbuf,
1762 (DDI_DMA_READ | DDI_DMA_CONSISTENT), callback, NULL,
1763 &cmd->arqc, &cookiec) != DDI_SUCCESS) {
1764 dev_err(pvs->dip, CE_WARN, "!failed to bind ARQ buffer");
1765 goto free_arqhdl;
1766 }
1767
1768 return (0);
1769
1770 free_arqhdl:
1771 ddi_dma_free_handle(&cmd->arqhdl);
1772 free_arqbuf:
1773 scsi_free_consistent_buf(cmd->arqbuf);
1774 free_handle:
1775 ddi_dma_free_handle(&cmd->cmd_dmahdl);
1776
1777 return (-1);
1778 }
1779
1780 /* ARGSUSED cdrarg */
1781 static void
1782 pvscsi_ccache_destructor(void *buf, void *cdrarg)
1783 {
1784 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)buf;
1785
1786 if (cmd->cmd_dmahdl != NULL) {
1787 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1788 ddi_dma_free_handle(&cmd->cmd_dmahdl);
1789 cmd->cmd_dmahdl = NULL;
1790 }
1791
1792 if (cmd->arqhdl != NULL) {
1793 (void) ddi_dma_unbind_handle(cmd->arqhdl);
1794 ddi_dma_free_handle(&cmd->arqhdl);
1795 cmd->arqhdl = NULL;
1796 }
1797
1798 if (cmd->arqbuf != NULL) {
1799 scsi_free_consistent_buf(cmd->arqbuf);
1800 cmd->arqbuf = NULL;
1801 }
1802 }
1803
1804 /* tran_* entry points and setup */
1805 /* ARGSUSED hba_dip tgt_dip hba_tran */
1806 static int
1807 pvscsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
1808 scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
1809 {
1810 pvscsi_softc_t *pvs = SDEV2PRIV(sd);
1811
1812 ASSERT(pvs != NULL);
1813
1814 if (sd->sd_address.a_target >= PVSCSI_MAXTGTS)
1815 return (DDI_FAILURE);
1816
1817 return (DDI_SUCCESS);
1818 }
1819
1820 static int
1821 pvscsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
1822 {
1823 boolean_t poll = ((pkt->pkt_flags & FLAG_NOINTR) != 0);
1824 int rc;
1825 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1826 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1827
1828 ASSERT(cmd->pkt == pkt);
1829 ASSERT(cmd->cmd_pvs == pvs);
1830
1831 /*
1832 * Reinitialize some fields because the packet may
1833 * have been resubmitted.
1834 */
1835 pkt->pkt_reason = CMD_CMPLT;
1836 pkt->pkt_state = 0;
1837 pkt->pkt_statistics = 0;
1838
1839 /* Zero status byte */
1840 *(pkt->pkt_scbp) = 0;
1841
1842 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1843 ASSERT(cmd->cmd_dma_count != 0);
1844 pkt->pkt_resid = cmd->cmd_dma_count;
1845
1846 /*
1847 * Consistent packets need to be synced first
1848 * (only for data going out).
1849 */
1850 if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0) {
1851 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
1852 DDI_DMA_SYNC_FORDEV);
1853 }
1854 }
1855
1856 cmd->cmd_target = ap->a_target;
1857
1858 mutex_enter(&pvs->mutex);
1859 if (HBA_IS_QUIESCED(pvs) && !poll) {
1860 mutex_exit(&pvs->mutex);
1861 return (TRAN_BUSY);
1862 }
1863 mutex_exit(&pvs->mutex);
1864
1865 rc = pvscsi_transport_command(pvs, cmd);
1866
1867 if (poll) {
1868 pvscsi_cmd_t *dcmd;
1869 boolean_t qnotify;
1870
1871 if (rc == TRAN_ACCEPT)
1872 rc = pvscsi_poll_cmd(pvs, cmd);
1873
1874 mutex_enter(&pvs->rx_mutex);
1875 dcmd = pvscsi_process_comp_ring(pvs);
1876 mutex_exit(&pvs->rx_mutex);
1877
1878 mutex_enter(&pvs->mutex);
1879 qnotify = HBA_QUIESCE_PENDING(pvs);
1880 mutex_exit(&pvs->mutex);
1881
1882 pvscsi_complete_chained(dcmd);
1883
1884 if (qnotify)
1885 pvscsi_quiesce_notify(pvs);
1886 }
1887
1888 return (rc);
1889 }
1890
1891 static int
1892 pvscsi_reset(struct scsi_address *ap, int level)
1893 {
1894 pvscsi_softc_t *pvs = AP2PRIV(ap);
1895
1896 switch (level) {
1897 case RESET_ALL:
1898 return (pvscsi_reset_generic(pvs, NULL));
1899 case RESET_TARGET:
1900 ASSERT(ap != NULL);
1901 return (pvscsi_reset_generic(pvs, ap));
1902 default:
1903 return (0);
1904 }
1905 }
1906
1907 static int
1908 pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
1909 {
1910 boolean_t qnotify = B_FALSE;
1911 pvscsi_cmd_t *pending;
1912 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1913
1914 mutex_enter(&pvs->tx_mutex);
1915 mutex_enter(&pvs->rx_mutex);
1916 if (pkt != NULL) {
1917 /* Abort single command */
1918 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1919
1920 if (pvscsi_abort_cmd(cmd, &pending) == CMD_ABORTED) {
1921 /* Assume command is completely cancelled now */
1922 cmd->flags |= PVSCSI_FLAG_ABORTED;
1923 }
1924 } else {
1925 /* Abort all commands on the bus */
1926 pvscsi_abort_all(ap, pvs, &pending, PVSCSI_FLAG_ABORTED);
1927 }
1928 qnotify = HBA_QUIESCE_PENDING(pvs);
1929 mutex_exit(&pvs->rx_mutex);
1930 mutex_exit(&pvs->tx_mutex);
1931
1932 pvscsi_complete_chained(pending);
1933
1934 if (qnotify)
1935 pvscsi_quiesce_notify(pvs);
1936
1937 return (1);
1938 }
1939
1940 /* ARGSUSED tgtonly */
1941 static int
1942 pvscsi_getcap(struct scsi_address *ap, char *cap, int tgtonly)
1943 {
1944 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1945
1946 if (cap == NULL)
1947 return (-1);
1948
1949 switch (scsi_hba_lookup_capstr(cap)) {
1950 case SCSI_CAP_ARQ:
1951 return ((pvs->flags & PVSCSI_HBA_AUTO_REQUEST_SENSE) != 0);
1952 case SCSI_CAP_UNTAGGED_QING:
1953 return (1);
1954 default:
1955 return (-1);
1956 }
1957 }
1958
1959 /* ARGSUSED tgtonly */
1960 static int
1961 pvscsi_setcap(struct scsi_address *ap, char *cap, int value, int tgtonly)
1962 {
1963 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1964
1965 if (cap == NULL)
1966 return (-1);
1967
1968 switch (scsi_hba_lookup_capstr(cap)) {
1969 case SCSI_CAP_ARQ:
1970 mutex_enter(&pvs->mutex);
1971 if (value == 0)
1972 pvs->flags &= ~PVSCSI_HBA_AUTO_REQUEST_SENSE;
1973 else
1974 pvs->flags |= PVSCSI_HBA_AUTO_REQUEST_SENSE;
1975 mutex_exit(&pvs->mutex);
1976 return (1);
1977 default:
1978 return (0);
1979 }
1980 }
1981
1982 static void
1983 pvscsi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
1984 {
1985 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1986 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1987
1988 ASSERT(cmd->cmd_pvs == pvs);
1989
1990 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1991 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
1992 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1993 }
1994
1995 if (cmd->ctx != NULL) {
1996 mutex_enter(&pvs->mutex);
1997 pvscsi_release_ctx(cmd);
1998 mutex_exit(&pvs->mutex);
1999 }
2000
2001 if ((cmd->flags & PVSCSI_FLAGS_EXT) != 0)
2002 pvscsi_cmd_ext_free(cmd);
2003
2004 kmem_cache_free(pvs->cmd_cache, cmd);
2005 }
2006
2007 static struct scsi_pkt *
2008 pvscsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt, struct buf *bp,
2009 int cmdlen, int statuslen, int tgtlen, int flags, int (*callback)(),
2010 caddr_t arg)
2011 {
2012 boolean_t is_new;
2013 int kf = (callback == SLEEP_FUNC) ? KM_SLEEP: KM_NOSLEEP;
2014 int rc, i;
2015 pvscsi_cmd_t *cmd;
2016 pvscsi_softc_t *pvs;
2017
2018 pvs = ap->a_hba_tran->tran_hba_private;
2019 ASSERT(pvs != NULL);
2020
2021 /* Allocate a new SCSI packet */
2022 if (pkt == NULL) {
2023 ddi_dma_handle_t saved_dmahdl, saved_arqhdl;
2024 struct buf *saved_arqbuf;
2025 ddi_dma_cookie_t saved_arqc;
2026
2027 is_new = B_TRUE;
2028
2029 if ((cmd = kmem_cache_alloc(pvs->cmd_cache, kf)) == NULL)
2030 return (NULL);
2031
2032 saved_dmahdl = cmd->cmd_dmahdl;
2033 saved_arqhdl = cmd->arqhdl;
2034 saved_arqbuf = cmd->arqbuf;
2035 saved_arqc = cmd->arqc;
2036
2037 bzero(cmd, sizeof (pvscsi_cmd_t) -
2038 sizeof (cmd->cached_cookies));
2039
2040 cmd->cmd_pvs = pvs;
2041 cmd->cmd_dmahdl = saved_dmahdl;
2042 cmd->arqhdl = saved_arqhdl;
2043 cmd->arqbuf = saved_arqbuf;
2044 cmd->arqc = saved_arqc;
2045
2046 pkt = &cmd->cached_pkt;
2047 pkt->pkt_ha_private = (opaque_t)cmd;
2048 pkt->pkt_address = *ap;
2049 pkt->pkt_scbp = (uint8_t *)&cmd->cmd_scb;
2050 pkt->pkt_cdbp = (uint8_t *)&cmd->cmd_cdb;
2051 pkt->pkt_private = (opaque_t)&cmd->tgt_priv;
2052
2053 cmd->tgtlen = tgtlen;
2054 cmd->statuslen = statuslen;
2055 cmd->cmdlen = cmdlen;
2056 cmd->pkt = pkt;
2057 cmd->ctx = NULL;
2058
2059 /* Allocate extended buffers */
2060 if ((cmdlen > sizeof (cmd->cmd_cdb)) ||
2061 (statuslen > sizeof (cmd->cmd_scb)) ||
2062 (tgtlen > sizeof (cmd->tgt_priv))) {
2063 if (pvscsi_cmd_ext_alloc(pvs, cmd, kf) != DDI_SUCCESS) {
2064 dev_err(pvs->dip, CE_WARN,
2065 "!extent allocation failed");
2066 goto out;
2067 }
2068 }
2069 } else {
2070 is_new = B_FALSE;
2071
2072 cmd = PKT2CMD(pkt);
2073 cmd->flags &= PVSCSI_FLAGS_PERSISTENT;
2074 }
2075
2076 ASSERT((cmd->flags & PVSCSI_FLAG_TRANSPORT) == 0);
2077
2078 if ((flags & PKT_XARQ) != 0)
2079 cmd->flags |= PVSCSI_FLAG_XARQ;
2080
2081 /* Handle partial DMA transfers */
2082 if (cmd->cmd_nwin > 0) {
2083 if (++cmd->cmd_winindex >= cmd->cmd_nwin)
2084 return (NULL);
2085 if (ddi_dma_getwin(cmd->cmd_dmahdl, cmd->cmd_winindex,
2086 &cmd->cmd_dma_offset, &cmd->cmd_dma_len,
2087 &cmd->cmd_dmac, &cmd->cmd_dmaccount) == DDI_FAILURE)
2088 return (NULL);
2089 goto handle_dma_cookies;
2090 }
2091
2092 /* Setup data buffer */
2093 if (bp != NULL && bp->b_bcount > 0 &&
2094 (cmd->flags & PVSCSI_FLAG_DMA_VALID) == 0) {
2095 int dma_flags;
2096
2097 ASSERT(cmd->cmd_dmahdl != NULL);
2098
2099 if ((bp->b_flags & B_READ) != 0) {
2100 cmd->flags |= PVSCSI_FLAG_IO_READ;
2101 dma_flags = DDI_DMA_READ;
2102 } else {
2103 cmd->flags &= ~PVSCSI_FLAG_IO_READ;
2104 dma_flags = DDI_DMA_WRITE;
2105 }
2106 if ((flags & PKT_CONSISTENT) != 0) {
2107 cmd->flags |= PVSCSI_FLAG_IO_IOPB;
2108 dma_flags |= DDI_DMA_CONSISTENT;
2109 }
2110 if ((flags & PKT_DMA_PARTIAL) != 0)
2111 dma_flags |= DDI_DMA_PARTIAL;
2112
2113 rc = ddi_dma_buf_bind_handle(cmd->cmd_dmahdl, bp,
2114 dma_flags, callback, arg, &cmd->cmd_dmac,
2115 &cmd->cmd_dmaccount);
2116 if (rc == DDI_DMA_PARTIAL_MAP) {
2117 (void) ddi_dma_numwin(cmd->cmd_dmahdl,
2118 &cmd->cmd_nwin);
2119 cmd->cmd_winindex = 0;
2120 (void) ddi_dma_getwin(cmd->cmd_dmahdl,
2121 cmd->cmd_winindex, &cmd->cmd_dma_offset,
2122 &cmd->cmd_dma_len, &cmd->cmd_dmac,
2123 &cmd->cmd_dmaccount);
2124 } else if (rc != 0 && rc != DDI_DMA_MAPPED) {
2125 switch (rc) {
2126 case DDI_DMA_NORESOURCES:
2127 bioerror(bp, 0);
2128 break;
2129 case DDI_DMA_BADATTR:
2130 case DDI_DMA_NOMAPPING:
2131 bioerror(bp, EFAULT);
2132 break;
2133 case DDI_DMA_TOOBIG:
2134 default:
2135 bioerror(bp, EINVAL);
2136 break;
2137 }
2138 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2139 goto out;
2140 }
2141
2142 handle_dma_cookies:
2143 ASSERT(cmd->cmd_dmaccount > 0);
2144 if (cmd->cmd_dmaccount > PVSCSI_MAX_SG_SIZE) {
2145 dev_err(pvs->dip, CE_WARN,
2146 "!invalid cookie count: %d (max %d)",
2147 cmd->cmd_dmaccount, PVSCSI_MAX_SG_SIZE);
2148 bioerror(bp, EINVAL);
2149 goto out;
2150 }
2151
2152 cmd->flags |= PVSCSI_FLAG_DMA_VALID;
2153 cmd->cmd_dma_count = cmd->cmd_dmac.dmac_size;
2154 cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2155
2156 cmd->cached_cookies[0] = cmd->cmd_dmac;
2157
2158 /*
2159 * Calculate total amount of bytes for this I/O and
2160 * store cookies for further processing.
2161 */
2162 for (i = 1; i < cmd->cmd_dmaccount; i++) {
2163 ddi_dma_nextcookie(cmd->cmd_dmahdl, &cmd->cmd_dmac);
2164 cmd->cached_cookies[i] = cmd->cmd_dmac;
2165 cmd->cmd_dma_count += cmd->cmd_dmac.dmac_size;
2166 cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2167 }
2168
2169 pkt->pkt_resid = (bp->b_bcount - cmd->cmd_total_dma_count);
2170 }
2171
2172 return (pkt);
2173
2174 out:
2175 if (is_new)
2176 pvscsi_destroy_pkt(ap, pkt);
2177
2178 return (NULL);
2179 }
2180
2181 /* ARGSUSED ap */
2182 static void
2183 pvscsi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt)
2184 {
2185 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
2186
2187 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
2188 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
2189 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2190 }
2191 }
2192
2193 /* ARGSUSED ap */
2194 static void
2195 pvscsi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
2196 {
2197 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
2198
2199 if (cmd->cmd_dmahdl != NULL) {
2200 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
2201 (cmd->flags & PVSCSI_FLAG_IO_READ) ?
2202 DDI_DMA_SYNC_FORCPU : DDI_DMA_SYNC_FORDEV);
2203 }
2204
2205 }
2206
2207 /* ARGSUSED ap flag callback arg */
2208 static int
2209 pvscsi_reset_notify(struct scsi_address *ap, int flag,
2210 void (*callback)(caddr_t), caddr_t arg)
2211 {
2212 return (DDI_FAILURE);
2213 }
2214
2215 static int
2216 pvscsi_quiesce_hba(dev_info_t *dip)
2217 {
2218 pvscsi_softc_t *pvs;
2219 scsi_hba_tran_t *tran;
2220
2221 if ((tran = ddi_get_driver_private(dip)) == NULL ||
2222 (pvs = TRAN2PRIV(tran)) == NULL)
2223 return (-1);
2224
2225 mutex_enter(&pvs->mutex);
2226 if (!HBA_IS_QUIESCED(pvs))
2227 pvs->flags |= PVSCSI_HBA_QUIESCED;
2228
2229 if (pvs->cmd_queue_len != 0) {
2230 /* Outstanding commands present, wait */
2231 pvs->flags |= PVSCSI_HBA_QUIESCE_PENDING;
2232 cv_wait(&pvs->quiescevar, &pvs->mutex);
2233 ASSERT(pvs->cmd_queue_len == 0);
2234 }
2235 mutex_exit(&pvs->mutex);
2236
2237 /* Suspend taskq delivery and complete all scheduled tasks */
2238 ddi_taskq_suspend(pvs->msg_tq);
2239 ddi_taskq_wait(pvs->msg_tq);
2240 ddi_taskq_suspend(pvs->comp_tq);
2241 ddi_taskq_wait(pvs->comp_tq);
2242
2243 return (0);
2244 }
2245
2246 static int
2247 pvscsi_unquiesce_hba(dev_info_t *dip)
2248 {
2249 pvscsi_softc_t *pvs;
2250 scsi_hba_tran_t *tran;
2251
2252 if ((tran = ddi_get_driver_private(dip)) == NULL ||
2253 (pvs = TRAN2PRIV(tran)) == NULL)
2254 return (-1);
2255
2256 mutex_enter(&pvs->mutex);
2257 if (!HBA_IS_QUIESCED(pvs)) {
2258 mutex_exit(&pvs->mutex);
2259 return (0);
2260 }
2261 ASSERT(pvs->cmd_queue_len == 0);
2262 pvs->flags &= ~PVSCSI_HBA_QUIESCED;
2263 mutex_exit(&pvs->mutex);
2264
2265 /* Resume taskq delivery */
2266 ddi_taskq_resume(pvs->msg_tq);
2267 ddi_taskq_resume(pvs->comp_tq);
2268
2269 return (0);
2270 }
2271
2272 static int
2273 pvscsi_bus_config(dev_info_t *pdip, uint_t flags, ddi_bus_config_op_t op,
2274 void *arg, dev_info_t **childp)
2275 {
2276 char *p;
2277 int circ;
2278 int ret = NDI_FAILURE;
2279 long target = 0;
2280 pvscsi_softc_t *pvs;
2281 scsi_hba_tran_t *tran;
2282
2283 tran = ddi_get_driver_private(pdip);
2284 pvs = tran->tran_hba_private;
2285
2286 ndi_devi_enter(pdip, &circ);
2287 switch (op) {
2288 case BUS_CONFIG_ONE:
2289 if ((p = strrchr((char *)arg, '@')) != NULL &&
2290 ddi_strtol(p + 1, NULL, 16, &target) == 0)
2291 ret = pvscsi_config_one(pdip, pvs, (int)target, childp);
2292 break;
2293 case BUS_CONFIG_DRIVER:
2294 case BUS_CONFIG_ALL:
2295 ret = pvscsi_config_all(pdip, pvs);
2296 break;
2297 default:
2298 break;
2299 }
2300
2301 if (ret == NDI_SUCCESS)
2302 ret = ndi_busop_bus_config(pdip, flags, op, arg, childp, 0);
2303 ndi_devi_exit(pdip, circ);
2304
2305 return (ret);
2306 }
2307
2308 static int
2309 pvscsi_hba_setup(pvscsi_softc_t *pvs)
2310 {
2311 scsi_hba_tran_t *hba_tran;
2312
2313 hba_tran = pvs->tran = scsi_hba_tran_alloc(pvs->dip,
2314 SCSI_HBA_CANSLEEP);
2315 ASSERT(pvs->tran != NULL);
2316
2317 hba_tran->tran_hba_private = pvs;
2318 hba_tran->tran_tgt_private = NULL;
2319
2320 hba_tran->tran_tgt_init = pvscsi_tgt_init;
2321 hba_tran->tran_tgt_free = NULL;
2322 hba_tran->tran_tgt_probe = scsi_hba_probe;
2323
2324 hba_tran->tran_start = pvscsi_start;
2325 hba_tran->tran_reset = pvscsi_reset;
2326 hba_tran->tran_abort = pvscsi_abort;
2327 hba_tran->tran_getcap = pvscsi_getcap;
2328 hba_tran->tran_setcap = pvscsi_setcap;
2329 hba_tran->tran_init_pkt = pvscsi_init_pkt;
2330 hba_tran->tran_destroy_pkt = pvscsi_destroy_pkt;
2331
2332 hba_tran->tran_dmafree = pvscsi_dmafree;
2333 hba_tran->tran_sync_pkt = pvscsi_sync_pkt;
2334 hba_tran->tran_reset_notify = pvscsi_reset_notify;
2335
2336 hba_tran->tran_quiesce = pvscsi_quiesce_hba;
2337 hba_tran->tran_unquiesce = pvscsi_unquiesce_hba;
2338 hba_tran->tran_bus_reset = NULL;
2339
2340 hba_tran->tran_add_eventcall = NULL;
2341 hba_tran->tran_get_eventcookie = NULL;
2342 hba_tran->tran_post_event = NULL;
2343 hba_tran->tran_remove_eventcall = NULL;
2344
2345 hba_tran->tran_bus_config = pvscsi_bus_config;
2346
2347 hba_tran->tran_interconnect_type = INTERCONNECT_SAS;
2348
2349 if (scsi_hba_attach_setup(pvs->dip, &pvs->hba_dma_attr, hba_tran,
2350 SCSI_HBA_TRAN_CDB | SCSI_HBA_TRAN_SCB | SCSI_HBA_TRAN_CLONE) !=
2351 DDI_SUCCESS) {
2352 dev_err(pvs->dip, CE_WARN, "!failed to attach HBA");
2353 scsi_hba_tran_free(hba_tran);
2354 pvs->tran = NULL;
2355 return (-1);
2356 }
2357
2358 return (0);
2359 }
2360
2361 static int
2362 pvscsi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2363 {
2364 int instance;
2365 pvscsi_softc_t *pvs;
2366 char buf[32];
2367
2368 ASSERT(scsi_hba_iport_unit_address(dip) == NULL);
2369
2370 switch (cmd) {
2371 case DDI_ATTACH:
2372 case DDI_RESUME:
2373 break;
2374 default:
2375 return (DDI_FAILURE);
2376 }
2377
2378 instance = ddi_get_instance(dip);
2379
2380 /* Allocate softstate information */
2381 if (ddi_soft_state_zalloc(pvscsi_sstate, instance) != DDI_SUCCESS) {
2382 cmn_err(CE_WARN,
2383 "!ddi_soft_state_zalloc() failed for instance %d",
2384 instance);
2385 return (DDI_FAILURE);
2386 }
2387
2388 if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2389 cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2390 instance);
2391 goto fail;
2392 }
2393
2394 /*
2395 * Indicate that we are 'sizeof (scsi_*(9S))' clean, we use
2396 * scsi_pkt_size() instead.
2397 */
2398 scsi_size_clean(dip);
2399
2400 /* Setup HBA instance */
2401 pvs->instance = instance;
2402 pvs->dip = dip;
2403 pvs->hba_dma_attr = pvscsi_hba_dma_attr;
2404 pvs->ring_dma_attr = pvscsi_ring_dma_attr;
2405 pvs->io_dma_attr = pvscsi_io_dma_attr;
2406 mutex_init(&pvs->mutex, "pvscsi instance mutex", MUTEX_DRIVER, NULL);
2407 mutex_init(&pvs->intr_mutex, "pvscsi instance interrupt mutex",
2408 MUTEX_DRIVER, NULL);
2409 mutex_init(&pvs->rx_mutex, "pvscsi rx ring mutex", MUTEX_DRIVER, NULL);
2410 mutex_init(&pvs->tx_mutex, "pvscsi tx ring mutex", MUTEX_DRIVER, NULL);
2411 list_create(&pvs->cmd_ctx_pool, sizeof (pvscsi_cmd_ctx_t),
2412 offsetof(pvscsi_cmd_ctx_t, list));
2413 list_create(&pvs->devnodes, sizeof (pvscsi_device_t),
2414 offsetof(pvscsi_device_t, list));
2415 list_create(&pvs->cmd_queue, sizeof (pvscsi_cmd_t),
2416 offsetof(pvscsi_cmd_t, cmd_queue_node));
2417 cv_init(&pvs->syncvar, "pvscsi synchronization cv", CV_DRIVER, NULL);
2418 cv_init(&pvs->wd_condvar, "pvscsi watchdog cv", CV_DRIVER, NULL);
2419 cv_init(&pvs->quiescevar, "pvscsi quiesce cv", CV_DRIVER, NULL);
2420
2421 (void) sprintf(buf, "pvscsi%d_cache", instance);
2422 pvs->cmd_cache = kmem_cache_create(buf, sizeof (pvscsi_cmd_t), 0,
2423 pvscsi_ccache_constructor, pvscsi_ccache_destructor, NULL,
2424 (void *)pvs, NULL, 0);
2425 if (pvs->cmd_cache == NULL) {
2426 dev_err(pvs->dip, CE_WARN,
2427 "!failed to create a cache for SCSI commands");
2428 goto fail;
2429 }
2430
2431 if ((pvscsi_setup_io(pvs)) != DDI_SUCCESS) {
2432 dev_err(pvs->dip, CE_WARN, "!failed to setup I/O region");
2433 goto free_cache;
2434 }
2435
2436 pvscsi_reset_hba(pvs);
2437
2438 if ((pvscsi_allocate_rings(pvs)) != DDI_SUCCESS) {
2439 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA rings");
2440 goto free_io;
2441 }
2442
2443 pvscsi_setup_rings(pvs);
2444
2445 if (pvscsi_setup_isr(pvs) != DDI_SUCCESS) {
2446 dev_err(pvs->dip, CE_WARN, "!failed to setup ISR");
2447 goto free_rings;
2448 }
2449
2450 if (pvscsi_setup_sg(pvs) != DDI_SUCCESS) {
2451 dev_err(pvs->dip, CE_WARN, "!failed to setup S/G");
2452 goto free_intr;
2453 }
2454
2455 if (pvscsi_hba_setup(pvs) != 0) {
2456 dev_err(pvs->dip, CE_WARN, "!failed to setup HBA");
2457 goto free_sg;
2458 }
2459
2460 if ((pvs->comp_tq = ddi_taskq_create(pvs->dip, "comp_tq",
2461 MIN(UINT16_MAX, ncpus), TASKQ_DEFAULTPRI, 0)) == NULL) {
2462 dev_err(pvs->dip, CE_WARN,
2463 "!failed to create completion taskq");
2464 goto free_sg;
2465 }
2466
2467 if ((pvs->msg_tq = ddi_taskq_create(pvs->dip, "msg_tq",
2468 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
2469 dev_err(pvs->dip, CE_WARN,
2470 "!failed to create message taskq");
2471 goto free_comp_tq;
2472 }
2473
2474 if (pvscsi_enable_intrs(pvs) != DDI_SUCCESS) {
2475 dev_err(pvs->dip, CE_WARN, "!failed to enable interrupts");
2476 goto free_msg_tq;
2477 }
2478
2479 /* Launch watchdog thread */
2480 pvs->wd_thread = thread_create(NULL, 0, pvscsi_wd_thread, pvs, 0, &p0,
2481 TS_RUN, minclsyspri);
2482
2483 return (DDI_SUCCESS);
2484
2485 free_msg_tq:
2486 ddi_taskq_destroy(pvs->msg_tq);
2487 free_comp_tq:
2488 ddi_taskq_destroy(pvs->comp_tq);
2489 free_sg:
2490 pvscsi_free_sg(pvs);
2491 free_intr:
2492 pvscsi_free_intr_resources(pvs);
2493 free_rings:
2494 pvscsi_reset_hba(pvs);
2495 pvscsi_free_rings(pvs);
2496 free_io:
2497 pvscsi_free_io(pvs);
2498 free_cache:
2499 kmem_cache_destroy(pvs->cmd_cache);
2500 fail:
2501 ddi_soft_state_free(pvscsi_sstate, instance);
2502
2503 return (DDI_FAILURE);
2504 }
2505
2506 static int
2507 pvscsi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2508 {
2509 int instance;
2510 pvscsi_softc_t *pvs;
2511
2512 switch (cmd) {
2513 case DDI_DETACH:
2514 break;
2515 default:
2516 return (DDI_FAILURE);
2517 }
2518
2519 instance = ddi_get_instance(dip);
2520 if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2521 cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2522 instance);
2523 return (DDI_FAILURE);
2524 }
2525
2526 pvscsi_reset_hba(pvs);
2527 pvscsi_free_intr_resources(pvs);
2528
2529 /* Shutdown message taskq */
2530 ddi_taskq_wait(pvs->msg_tq);
2531 ddi_taskq_destroy(pvs->msg_tq);
2532
2533 /* Shutdown completion taskq */
2534 ddi_taskq_wait(pvs->comp_tq);
2535 ddi_taskq_destroy(pvs->comp_tq);
2536
2537 /* Shutdown watchdog thread */
2538 mutex_enter(&pvs->mutex);
2539 pvs->flags |= PVSCSI_DRIVER_SHUTDOWN;
2540 cv_signal(&pvs->wd_condvar);
2541 cv_wait(&pvs->syncvar, &pvs->mutex);
2542 mutex_exit(&pvs->mutex);
2543
2544 pvscsi_free_sg(pvs);
2545 pvscsi_free_rings(pvs);
2546 pvscsi_free_io(pvs);
2547
2548 kmem_cache_destroy(pvs->cmd_cache);
2549
2550 mutex_destroy(&pvs->mutex);
2551 mutex_destroy(&pvs->intr_mutex);
2552 mutex_destroy(&pvs->rx_mutex);
2553
2554 cv_destroy(&pvs->syncvar);
2555 cv_destroy(&pvs->wd_condvar);
2556 cv_destroy(&pvs->quiescevar);
2557
2558 ddi_soft_state_free(pvscsi_sstate, instance);
2559 ddi_prop_remove_all(dip);
2560
2561 return (DDI_SUCCESS);
2562 }
2563
2564 static int
2565 pvscsi_ioctl(dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp,
2566 int *rval)
2567 {
2568 int ret;
2569
2570 if (ddi_get_soft_state(pvscsi_sstate, getminor(dev)) == NULL) {
2571 cmn_err(CE_WARN, "!invalid device instance: %d", getminor(dev));
2572 return (ENXIO);
2573 }
2574
2575 /* Try to handle command in a common way */
2576 if ((ret = scsi_hba_ioctl(dev, cmd, data, mode, credp, rval)) != ENOTTY)
2577 return (ret);
2578
2579 cmn_err(CE_WARN, "!unsupported IOCTL command: 0x%X", cmd);
2580
2581 return (ENXIO);
2582 }
2583
2584 static int
2585 pvscsi_quiesce(dev_info_t *devi)
2586 {
2587 scsi_hba_tran_t *tran;
2588 pvscsi_softc_t *pvs;
2589
2590 if ((tran = ddi_get_driver_private(devi)) == NULL)
2591 return (DDI_SUCCESS);
2592
2593 if ((pvs = tran->tran_hba_private) == NULL)
2594 return (DDI_SUCCESS);
2595
2596 /* Mask all interrupts from device */
2597 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
2598
2599 /* Reset the HBA */
2600 pvscsi_reset_hba(pvs);
2601
2602 return (DDI_SUCCESS);
2603 }
2604
2605 /* module */
2606
2607 static struct cb_ops pvscsi_cb_ops = {
2608 .cb_open = scsi_hba_open,
2609 .cb_close = scsi_hba_close,
2610 .cb_strategy = nodev,
2611 .cb_print = nodev,
2612 .cb_dump = nodev,
2613 .cb_read = nodev,
2614 .cb_write = nodev,
2615 .cb_ioctl = pvscsi_ioctl,
2616 .cb_devmap = nodev,
2617 .cb_mmap = nodev,
2618 .cb_segmap = nodev,
2619 .cb_chpoll = nochpoll,
2620 .cb_prop_op = ddi_prop_op,
2621 .cb_str = NULL,
2622 .cb_flag = D_MP,
2623 .cb_rev = CB_REV,
2624 .cb_aread = nodev,
2625 .cb_awrite = nodev
2626 };
2627
2628 static struct dev_ops pvscsi_ops = {
2629 .devo_rev = DEVO_REV,
2630 .devo_refcnt = 0,
2631 .devo_getinfo = ddi_no_info,
2632 .devo_identify = nulldev,
2633 .devo_probe = nulldev,
2634 .devo_attach = pvscsi_attach,
2635 .devo_detach = pvscsi_detach,
2636 .devo_reset = nodev,
2637 .devo_cb_ops = &pvscsi_cb_ops,
2638 .devo_bus_ops = NULL,
2639 .devo_power = NULL,
2640 .devo_quiesce = pvscsi_quiesce
2641 };
2642
2643 #define PVSCSI_IDENT "VMware PVSCSI"
2644
2645 static struct modldrv modldrv = {
2646 &mod_driverops,
2647 PVSCSI_IDENT,
2648 &pvscsi_ops,
2649 };
2650
2651 static struct modlinkage modlinkage = {
2652 MODREV_1,
2653 &modldrv,
2654 NULL
2655 };
2656
2657 int
2658 _init(void)
2659 {
2660 int ret;
2661
2662 if ((ret = ddi_soft_state_init(&pvscsi_sstate,
2663 sizeof (struct pvscsi_softc), PVSCSI_INITIAL_SSTATE_ITEMS)) != 0) {
2664 cmn_err(CE_WARN, "!ddi_soft_state_init() failed");
2665 return (ret);
2666 }
2667
2668 if ((ret = scsi_hba_init(&modlinkage)) != 0) {
2669 cmn_err(CE_WARN, "!scsi_hba_init() failed");
2670 ddi_soft_state_fini(&pvscsi_sstate);
2671 return (ret);
2672 }
2673
2674 if ((ret = mod_install(&modlinkage)) != 0) {
2675 cmn_err(CE_WARN, "!mod_install() failed");
2676 ddi_soft_state_fini(&pvscsi_sstate);
2677 scsi_hba_fini(&modlinkage);
2678 }
2679
2680 return (ret);
2681 }
2682
2683 int
2684 _info(struct modinfo *modinfop)
2685 {
2686 return (mod_info(&modlinkage, modinfop));
2687 }
2688
2689 int
2690 _fini(void)
2691 {
2692 int ret;
2693
2694 if ((ret = mod_remove(&modlinkage)) == 0) {
2695 ddi_soft_state_fini(&pvscsi_sstate);
2696 scsi_hba_fini(&modlinkage);
2697 }
2698
2699 return (ret);
2700 }