1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2018 Nexenta Systems, Inc.
14 */
15
16 #include <sys/atomic.h>
17 #include <sys/cmn_err.h>
18 #include <sys/conf.h>
19 #include <sys/cpuvar.h>
20 #include <sys/ddi.h>
21 #include <sys/errno.h>
22 #include <sys/kmem.h>
23 #include <sys/kmem_impl.h>
24 #include <sys/list.h>
25 #include <sys/modctl.h>
26 #include <sys/pci.h>
27 #include <sys/scsi/scsi.h>
28 #include <sys/sunddi.h>
29 #include <sys/sysmacros.h>
30 #include <sys/time.h>
31 #include <sys/types.h>
32
33 #include "pvscsi.h"
34 #include "pvscsi_var.h"
35
36 int pvscsi_enable_msi = 1;
37 int pvscsi_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
38 int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
39
40 static int pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt);
41
42 static void *pvscsi_sstate;
43
44 /* HBA DMA attributes */
45 static ddi_dma_attr_t pvscsi_hba_dma_attr = {
46 .dma_attr_version = DMA_ATTR_V0,
47 .dma_attr_addr_lo = 0x0000000000000000ull,
48 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
49 .dma_attr_count_max = 0x000000007FFFFFFFull,
50 .dma_attr_align = 0x0000000000000001ull,
51 .dma_attr_burstsizes = 0x7ff,
52 .dma_attr_minxfer = 0x00000001u,
53 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
54 .dma_attr_seg = 0x00000000FFFFFFFFull,
55 .dma_attr_sgllen = 1,
56 .dma_attr_granular = 0x00000200u,
57 .dma_attr_flags = 0
58 };
59
60 /* DMA attributes for req/comp rings */
61 static ddi_dma_attr_t pvscsi_ring_dma_attr = {
62 .dma_attr_version = DMA_ATTR_V0,
63 .dma_attr_addr_lo = 0x0000000000000000ull,
64 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
65 .dma_attr_count_max = 0x000000007FFFFFFFull,
66 .dma_attr_align = 0x0000000000000001ull,
67 .dma_attr_burstsizes = 0x7ff,
68 .dma_attr_minxfer = 0x00000001u,
69 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
70 .dma_attr_seg = 0x00000000FFFFFFFFull,
71 .dma_attr_sgllen = 1,
72 .dma_attr_granular = 0x00000001u,
73 .dma_attr_flags = 0
74 };
75
76 /* DMA attributes for buffer I/O */
77 static ddi_dma_attr_t pvscsi_io_dma_attr = {
78 .dma_attr_version = DMA_ATTR_V0,
79 .dma_attr_addr_lo = 0x0000000000000000ull,
80 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
81 .dma_attr_count_max = 0x000000007FFFFFFFull,
82 .dma_attr_align = 0x0000000000000001ull,
83 .dma_attr_burstsizes = 0x7ff,
84 .dma_attr_minxfer = 0x00000001u,
85 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
86 .dma_attr_seg = 0x00000000FFFFFFFFull,
87 .dma_attr_sgllen = PVSCSI_MAX_SG_SIZE,
88 .dma_attr_granular = 0x00000200u,
89 .dma_attr_flags = 0
90 };
91
92 static ddi_device_acc_attr_t pvscsi_mmio_attr = {
93 DDI_DEVICE_ATTR_V1,
94 DDI_STRUCTURE_LE_ACC,
95 DDI_STRICTORDER_ACC,
96 DDI_DEFAULT_ACC
97 };
98
99 static ddi_device_acc_attr_t pvscsi_dma_attrs = {
100 DDI_DEVICE_ATTR_V0,
101 DDI_STRUCTURE_LE_ACC,
102 DDI_STRICTORDER_ACC,
103 DDI_DEFAULT_ACC,
104 };
105
106 static void
107 pvscsi_add_to_queue(pvscsi_cmd_t *cmd)
108 {
109 pvscsi_softc_t *pvs = cmd->cmd_pvs;
110
111 ASSERT(pvs != NULL);
112 ASSERT(mutex_owned(&pvs->mutex));
113 ASSERT(!list_link_active(&(cmd)->cmd_queue_node));
114
115 list_insert_tail(&pvs->cmd_queue, cmd);
116 pvs->cmd_queue_len++;
117 }
118
119 static void
120 pvscsi_remove_from_queue(pvscsi_cmd_t *cmd)
121 {
122 pvscsi_softc_t *pvs = cmd->cmd_pvs;
123
124 ASSERT(pvs != NULL);
125 ASSERT(mutex_owned(&pvs->mutex));
126 ASSERT(list_link_active(&cmd->cmd_queue_node));
127 ASSERT(pvs->cmd_queue_len > 0);
128
129 if (list_link_active(&cmd->cmd_queue_node)) {
130 list_remove(&pvs->cmd_queue, cmd);
131 pvs->cmd_queue_len--;
132 }
133 }
134
135 static uint64_t
136 pvscsi_map_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_ctx_t *io_ctx)
137 {
138 return (io_ctx - pvs->cmd_ctx + 1);
139 }
140
141 static pvscsi_cmd_ctx_t *
142 pvscsi_lookup_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
143 {
144 pvscsi_cmd_ctx_t *ctx, *end;
145
146 end = &pvs->cmd_ctx[pvs->req_depth];
147 for (ctx = pvs->cmd_ctx; ctx < end; ctx++) {
148 if (ctx->cmd == cmd)
149 return (ctx);
150 }
151
152 return (NULL);
153 }
154
155 static pvscsi_cmd_ctx_t *
156 pvscsi_resolve_ctx(pvscsi_softc_t *pvs, uint64_t ctx)
157 {
158 if (ctx > 0 && ctx <= pvs->req_depth)
159 return (&pvs->cmd_ctx[ctx - 1]);
160 else
161 return (NULL);
162 }
163
164 static boolean_t
165 pvscsi_acquire_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
166 {
167 pvscsi_cmd_ctx_t *ctx;
168
169 if (list_is_empty(&pvs->cmd_ctx_pool))
170 return (B_FALSE);
171
172 ctx = (pvscsi_cmd_ctx_t *)list_remove_head(&pvs->cmd_ctx_pool);
173 ASSERT(ctx != NULL);
174
175 ctx->cmd = cmd;
176 cmd->ctx = ctx;
177
178 return (B_TRUE);
179 }
180
181 static void
182 pvscsi_release_ctx(pvscsi_cmd_t *cmd)
183 {
184 pvscsi_softc_t *pvs = cmd->cmd_pvs;
185
186 ASSERT(mutex_owned(&pvs->mutex));
187
188 cmd->ctx->cmd = NULL;
189 list_insert_tail(&pvs->cmd_ctx_pool, cmd->ctx);
190 cmd->ctx = NULL;
191 }
192
193 static uint32_t
194 pvscsi_reg_read(pvscsi_softc_t *pvs, uint32_t offset)
195 {
196 uint32_t ret;
197
198 ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
199
200 ret = ddi_get32(pvs->mmio_handle,
201 (uint32_t *)(pvs->mmio_base + offset));
202
203 return (ret);
204 }
205
206 static void
207 pvscsi_reg_write(pvscsi_softc_t *pvs, uint32_t offset, uint32_t value)
208 {
209 ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
210
211 ddi_put32(pvs->mmio_handle, (uint32_t *)(pvs->mmio_base + offset),
212 value);
213 }
214
215 static void
216 pvscsi_write_cmd_desc(pvscsi_softc_t *pvs, uint32_t cmd, void *desc, size_t len)
217 {
218 len /= sizeof (uint32_t);
219 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_COMMAND, cmd);
220 ddi_rep_put32(pvs->mmio_handle, (uint32_t *)desc,
221 (uint32_t *)(pvs->mmio_base + PVSCSI_REG_OFFSET_COMMAND_DATA),
222 len, DDI_DEV_NO_AUTOINCR);
223 }
224
225 static uint32_t
226 pvscsi_read_intr_status(pvscsi_softc_t *pvs)
227 {
228 return (pvscsi_reg_read(pvs, PVSCSI_REG_OFFSET_INTR_STATUS));
229 }
230
231 static void
232 pvscsi_write_intr_status(pvscsi_softc_t *pvs, uint32_t val)
233 {
234 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_STATUS, val);
235 }
236
237 static void
238 pvscsi_mask_intr(pvscsi_softc_t *pvs)
239 {
240 mutex_enter(&pvs->intr_mutex);
241
242 VERIFY(pvs->intr_lock_counter >= 0);
243
244 if (++pvs->intr_lock_counter == 1)
245 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
246
247 mutex_exit(&pvs->intr_mutex);
248 }
249
250 static void
251 pvscsi_unmask_intr(pvscsi_softc_t *pvs)
252 {
253 mutex_enter(&pvs->intr_mutex);
254
255 VERIFY(pvs->intr_lock_counter > 0);
256
257 if (--pvs->intr_lock_counter == 0) {
258 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
259 PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
260 }
261
262 mutex_exit(&pvs->intr_mutex);
263 }
264
265 static void
266 pvscsi_reset_hba(pvscsi_softc_t *pvs)
267 {
268 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
269 }
270
271 static void
272 pvscsi_reset_bus(pvscsi_softc_t *pvs)
273 {
274 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_BUS, NULL, 0);
275 }
276
277 static void
278 pvscsi_submit_nonrw_io(pvscsi_softc_t *pvs)
279 {
280 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
281 }
282
283 static void
284 pvscsi_submit_rw_io(pvscsi_softc_t *pvs)
285 {
286 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
287 }
288
289
290 static int
291 pvscsi_inquiry_target(pvscsi_softc_t *pvs, int target, struct scsi_inquiry *inq)
292 {
293 int len = sizeof (struct scsi_inquiry);
294 int ret = -1;
295 struct buf *b;
296 struct scsi_address ap;
297 struct scsi_pkt *pkt;
298 uint8_t cdb[CDB_GROUP0];
299
300 ap.a_hba_tran = pvs->tran;
301 ap.a_target = (ushort_t)target;
302 ap.a_lun = (uchar_t)0;
303
304 if ((b = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL, len,
305 B_READ, NULL_FUNC, NULL)) == NULL)
306 return (-1);
307
308 if ((pkt = scsi_init_pkt(&ap, (struct scsi_pkt *)NULL, b,
309 CDB_GROUP0, sizeof (struct scsi_arq_status), 0, 0,
310 NULL_FUNC, NULL)) == NULL)
311 goto free_buf;
312
313 cdb[0] = SCMD_INQUIRY;
314 cdb[1] = 0;
315 cdb[2] = 0;
316 cdb[3] = (len & 0xff00) >> 8;
317 cdb[4] = (len & 0x00ff);
318 cdb[5] = 0;
319
320 if (inq != NULL)
321 bzero(inq, sizeof (*inq));
322 bcopy(cdb, pkt->pkt_cdbp, CDB_GROUP0);
323 bzero((struct scsi_inquiry *)b->b_un.b_addr, sizeof (*inq));
324
325 if ((ret = scsi_poll(pkt)) == 0 && inq != NULL)
326 bcopy(b->b_un.b_addr, inq, sizeof (*inq));
327
328 scsi_destroy_pkt(pkt);
329
330 free_buf:
331 scsi_free_consistent_buf(b);
332
333 return (ret);
334 }
335
336 static int
337 pvscsi_config_one(dev_info_t *pdip, pvscsi_softc_t *pvs, int target,
338 dev_info_t **childp)
339 {
340 char **compatible = NULL;
341 char *nodename = NULL;
342 dev_info_t *dip;
343 int inqrc;
344 int ncompatible = 0;
345 pvscsi_device_t *devnode;
346 struct scsi_inquiry inq;
347
348 ASSERT(DEVI_BUSY_OWNED(pdip));
349
350 /* Inquiry target */
351 inqrc = pvscsi_inquiry_target(pvs, target, &inq);
352
353 /* Find devnode */
354 for (devnode = list_head(&pvs->devnodes); devnode != NULL;
355 devnode = list_next(&pvs->devnodes, devnode)) {
356 if (devnode->target == target)
357 break;
358 }
359
360 if (devnode != NULL) {
361 if (inqrc != 0) {
362 (void) ndi_devi_offline(devnode->pdip,
363 NDI_DEVFS_CLEAN | NDI_DEVI_REMOVE);
364 list_remove(&pvs->devnodes, devnode);
365 kmem_free(devnode, sizeof (*devnode));
366 } else if (childp != NULL) {
367 /* Target exists */
368 *childp = devnode->pdip;
369 }
370 return (NDI_SUCCESS);
371 } else if (inqrc != 0) {
372 /* Target doesn't exist */
373 return (NDI_FAILURE);
374 }
375
376 scsi_hba_nodename_compatible_get(&inq, NULL, inq.inq_dtype, NULL,
377 &nodename, &compatible, &ncompatible);
378 if (nodename == NULL)
379 goto free_nodename;
380
381 if (ndi_devi_alloc(pdip, nodename, DEVI_SID_NODEID,
382 &dip) != NDI_SUCCESS) {
383 dev_err(pvs->dip, CE_WARN, "!failed to alloc device instance");
384 goto free_nodename;
385 }
386
387 if (ndi_prop_update_string(DDI_DEV_T_NONE, dip,
388 "device-type", "scsi") != DDI_PROP_SUCCESS ||
389 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
390 "target", target) != DDI_PROP_SUCCESS ||
391 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
392 "lun", 0) != DDI_PROP_SUCCESS ||
393 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
394 "pm-capable", 1) != DDI_PROP_SUCCESS ||
395 ndi_prop_update_string_array(DDI_DEV_T_NONE, dip,
396 "compatible", compatible, ncompatible) != DDI_PROP_SUCCESS) {
397 dev_err(pvs->dip, CE_WARN,
398 "!failed to update props for target %d", target);
399 goto free_devi;
400 }
401
402 if ((devnode = kmem_zalloc(sizeof (*devnode), KM_NOSLEEP)) == NULL)
403 goto free_devi;
404
405 if (ndi_devi_online(dip, NDI_ONLINE_ATTACH) != NDI_SUCCESS) {
406 dev_err(pvs->dip, CE_WARN, "!failed to online target %d",
407 target);
408 kmem_free(devnode, sizeof (*devnode));
409 goto free_devi;
410 }
411
412 devnode->target = target;
413 devnode->pdip = dip;
414 devnode->parent = pdip;
415 list_insert_tail(&pvs->devnodes, devnode);
416
417 if (childp != NULL)
418 *childp = dip;
419
420 scsi_hba_nodename_compatible_free(nodename, compatible);
421
422 return (NDI_SUCCESS);
423
424 free_devi:
425 ndi_prop_remove_all(dip);
426 (void) ndi_devi_free(dip);
427 free_nodename:
428 scsi_hba_nodename_compatible_free(nodename, compatible);
429
430 return (NDI_FAILURE);
431 }
432
433 static int
434 pvscsi_config_all(dev_info_t *pdip, pvscsi_softc_t *pvs)
435 {
436 int target;
437
438 for (target = 0; target < PVSCSI_MAXTGTS; target++) {
439 /* ndi_devi_enter is done in pvscsi_bus_config */
440 (void) pvscsi_config_one(pdip, pvs, target, NULL);
441 }
442
443 return (NDI_SUCCESS);
444 }
445
446 static pvscsi_cmd_t *
447 pvscsi_process_comp_ring(pvscsi_softc_t *pvs)
448 {
449 pvscsi_cmd_t **pnext_cmd = NULL;
450 pvscsi_cmd_t *cmd;
451 pvscsi_cmd_t *head = NULL;
452 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
453 uint32_t cmp_ne = sdesc->cmpNumEntriesLog2;
454
455 ASSERT(mutex_owned(&pvs->rx_mutex));
456
457 while (sdesc->cmpConsIdx != sdesc->cmpProdIdx) {
458 pvscsi_cmd_ctx_t *ctx;
459 struct PVSCSIRingCmpDesc *cdesc;
460
461 cdesc = CMP_RING(pvs) + (sdesc->cmpConsIdx & MASK(cmp_ne));
462 membar_consumer();
463
464 ctx = pvscsi_resolve_ctx(pvs, cdesc->context);
465 ASSERT(ctx != NULL);
466
467 if ((cmd = ctx->cmd) != NULL) {
468 cmd->next_cmd = NULL;
469
470 /* Save command status for further processing */
471 cmd->cmp_stat.host_status = cdesc->hostStatus;
472 cmd->cmp_stat.scsi_status = cdesc->scsiStatus;
473 cmd->cmp_stat.data_len = cdesc->dataLen;
474
475 /* Mark this command as arrived from hardware */
476 cmd->flags |= PVSCSI_FLAG_HW_STATUS;
477
478 if (head == NULL) {
479 head = cmd;
480 head->tail_cmd = cmd;
481 } else {
482 head->tail_cmd = cmd;
483 }
484
485 if (pnext_cmd == NULL) {
486 pnext_cmd = &cmd->next_cmd;
487 } else {
488 *pnext_cmd = cmd;
489 pnext_cmd = &cmd->next_cmd;
490 }
491 }
492
493 membar_consumer();
494 sdesc->cmpConsIdx++;
495 }
496
497 return (head);
498 }
499
500 static pvscsi_msg_t *
501 pvscsi_process_msg_ring(pvscsi_softc_t *pvs)
502 {
503 pvscsi_msg_t *msg;
504 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
505 struct PVSCSIRingMsgDesc *mdesc;
506 struct PVSCSIMsgDescDevStatusChanged *desc;
507 uint32_t msg_ne = sdesc->msgNumEntriesLog2;
508
509 ASSERT(mutex_owned(&pvs->rx_mutex));
510
511 if (sdesc->msgProdIdx == sdesc->msgConsIdx)
512 return (NULL);
513
514 mdesc = MSG_RING(pvs) + (sdesc->msgConsIdx & MASK(msg_ne));
515 membar_consumer();
516
517 switch (mdesc->type) {
518 case PVSCSI_MSG_DEV_ADDED:
519 case PVSCSI_MSG_DEV_REMOVED:
520 desc = (struct PVSCSIMsgDescDevStatusChanged *)mdesc;
521 msg = kmem_alloc(sizeof (pvscsi_msg_t), KM_NOSLEEP);
522 if (msg == NULL)
523 return (NULL);
524 msg->msg_pvs = pvs;
525 msg->type = mdesc->type;
526 msg->target = desc->target;
527 break;
528 default:
529 dev_err(pvs->dip, CE_WARN, "!unknown msg type: %d",
530 mdesc->type);
531 return (NULL);
532 }
533
534 membar_consumer();
535 sdesc->msgConsIdx++;
536
537 return (msg);
538 }
539
540 static void
541 pvscsi_handle_msg(void *arg)
542 {
543 pvscsi_msg_t *msg = (pvscsi_msg_t *)arg;
544 dev_info_t *dip = msg->msg_pvs->dip;
545 int circ;
546
547 ndi_devi_enter(dip, &circ);
548 (void) pvscsi_config_one(dip, msg->msg_pvs, msg->target, NULL);
549 ndi_devi_exit(dip, circ);
550
551 kmem_free(msg, sizeof (pvscsi_msg_t));
552 }
553
554 static int
555 pvscsi_abort_cmd(pvscsi_cmd_t *cmd, pvscsi_cmd_t **pending)
556 {
557 pvscsi_softc_t *pvs = cmd->cmd_pvs;
558 pvscsi_cmd_t *c;
559 pvscsi_cmd_t *done;
560 struct PVSCSICmdDescAbortCmd acmd;
561
562 dev_err(pvs->dip, CE_WARN, "!aborting command %p", (void *)cmd);
563
564 ASSERT(mutex_owned(&pvs->rx_mutex));
565 ASSERT(mutex_owned(&pvs->tx_mutex));
566
567 /* Check if the cmd was already completed by the HBA */
568 *pending = done = pvscsi_process_comp_ring(pvs);
569 for (c = done; c != NULL; c = c->next_cmd) {
570 if (c == cmd)
571 return (CMD_CMPLT);
572 }
573
574 /* Check if cmd was really scheduled by the HBA */
575 if (pvscsi_lookup_ctx(pvs, cmd) == NULL)
576 return (CMD_CMPLT);
577
578 /* Abort cmd in the HBA */
579 bzero(&acmd, sizeof (acmd));
580 acmd.target = cmd->cmd_target;
581 acmd.context = pvscsi_map_ctx(pvs, cmd->ctx);
582 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ABORT_CMD, &acmd, sizeof (acmd));
583
584 /* Check if cmd was completed by the HBA before it could be aborted */
585 if ((done = pvscsi_process_comp_ring(pvs)) != NULL) {
586 done->tail_cmd->next_cmd = *pending;
587 *pending = done;
588 for (c = done; c != NULL; c = c->next_cmd) {
589 if (c == cmd)
590 return (CMD_CMPLT);
591 }
592 }
593
594 /* Release I/O ctx */
595 mutex_enter(&pvs->mutex);
596 if (cmd->ctx != NULL)
597 pvscsi_release_ctx(cmd);
598 /* Remove cmd from the queue */
599 pvscsi_remove_from_queue(cmd);
600 mutex_exit(&pvs->mutex);
601
602 /* Insert cmd at the beginning of the list */
603 cmd->next_cmd = *pending;
604 *pending = cmd;
605
606 dev_err(pvs->dip, CE_WARN, "!command %p aborted", (void *)cmd);
607
608 return (CMD_ABORTED);
609 }
610
611 static void
612 pvscsi_map_buffers(pvscsi_cmd_t *cmd, struct PVSCSIRingReqDesc *rdesc)
613 {
614 int i;
615
616 ASSERT(cmd->ctx);
617 ASSERT(cmd->cmd_dmaccount > 0 && cmd->cmd_dmaccount <=
618 PVSCSI_MAX_SG_SIZE);
619
620 rdesc->dataLen = cmd->cmd_dma_count;
621 rdesc->dataAddr = 0;
622
623 if (cmd->cmd_dma_count == 0)
624 return;
625
626 if (cmd->cmd_dmaccount > 1) {
627 struct PVSCSISGElement *sgl = CMD_CTX_SGLIST_VA(cmd->ctx);
628
629 for (i = 0; i < cmd->cmd_dmaccount; i++) {
630 sgl[i].addr = cmd->cached_cookies[i].dmac_laddress;
631 sgl[i].length = cmd->cached_cookies[i].dmac_size;
632 sgl[i].flags = 0;
633 }
634 rdesc->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
635 rdesc->dataAddr = (uint64_t)CMD_CTX_SGLIST_PA(cmd->ctx);
636 } else {
637 rdesc->dataAddr = cmd->cached_cookies[0].dmac_laddress;
638 }
639 }
640
641 static void
642 pvscsi_comp_cmd(pvscsi_cmd_t *cmd, uint8_t status)
643 {
644 struct scsi_pkt *pkt = CMD2PKT(cmd);
645
646 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
647 STATE_GOT_STATUS);
648 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0)
649 pkt->pkt_state |= STATE_XFERRED_DATA;
650 pkt->pkt_reason = CMD_CMPLT;
651 pkt->pkt_resid = 0;
652 *(pkt->pkt_scbp) = status;
653 }
654
655 static void
656 pvscsi_set_status(pvscsi_cmd_t *cmd)
657 {
658 pvscsi_softc_t *pvs = cmd->cmd_pvs;
659 struct scsi_pkt *pkt = CMD2PKT(cmd);
660 uchar_t scsi_status = cmd->cmp_stat.scsi_status;
661 uint32_t host_status = cmd->cmp_stat.host_status;
662
663 if (scsi_status != STATUS_GOOD &&
664 (host_status == BTSTAT_SUCCESS ||
665 (host_status == BTSTAT_LINKED_COMMAND_COMPLETED) ||
666 (host_status == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG))) {
667 if (scsi_status == STATUS_CHECK) {
668 struct scsi_arq_status *astat = (void*)(pkt->pkt_scbp);
669 uint8_t *sensedata;
670 int arq_size;
671
672 *pkt->pkt_scbp = scsi_status;
673 pkt->pkt_state |= STATE_ARQ_DONE;
674
675 if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
676 arq_size = (cmd->cmd_rqslen >=
677 SENSE_BUFFER_SIZE) ? SENSE_BUFFER_SIZE :
678 cmd->cmd_rqslen;
679
680 astat->sts_rqpkt_resid = SENSE_BUFFER_SIZE -
681 arq_size;
682 sensedata = (uint8_t *)&astat->sts_sensedata;
683 bcopy(cmd->arqbuf->b_un.b_addr, sensedata,
684 arq_size);
685
686 pkt->pkt_state |= STATE_XARQ_DONE;
687 } else {
688 astat->sts_rqpkt_resid = 0;
689 }
690
691 astat->sts_rqpkt_statistics = 0;
692 astat->sts_rqpkt_reason = CMD_CMPLT;
693 (*(uint8_t *)&astat->sts_rqpkt_status) = STATUS_GOOD;
694 astat->sts_rqpkt_state = STATE_GOT_BUS |
695 STATE_GOT_TARGET | STATE_SENT_CMD |
696 STATE_XFERRED_DATA | STATE_GOT_STATUS;
697 }
698 pvscsi_comp_cmd(cmd, scsi_status);
699
700 return;
701 }
702
703 switch (host_status) {
704 case BTSTAT_SUCCESS:
705 case BTSTAT_LINKED_COMMAND_COMPLETED:
706 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
707 pvscsi_comp_cmd(cmd, STATUS_GOOD);
708 break;
709 case BTSTAT_DATARUN:
710 pkt->pkt_reason = CMD_DATA_OVR;
711 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
712 STATE_SENT_CMD | STATE_GOT_STATUS |
713 STATE_XFERRED_DATA);
714 pkt->pkt_resid = 0;
715 break;
716 case BTSTAT_DATA_UNDERRUN:
717 pkt->pkt_reason = pkt->pkt_state |= (STATE_GOT_BUS |
718 STATE_GOT_TARGET | STATE_SENT_CMD | STATE_GOT_STATUS);
719 pkt->pkt_resid = cmd->dma_count - cmd->cmp_stat.data_len;
720 if (pkt->pkt_resid != cmd->dma_count)
721 pkt->pkt_state |= STATE_XFERRED_DATA;
722 break;
723 case BTSTAT_SELTIMEO:
724 pkt->pkt_reason = CMD_DEV_GONE;
725 pkt->pkt_state |= STATE_GOT_BUS;
726 break;
727 case BTSTAT_TAGREJECT:
728 pkt->pkt_reason = CMD_TAG_REJECT;
729 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
730 STATE_SENT_CMD | STATE_GOT_STATUS);
731 break;
732 case BTSTAT_BADMSG:
733 pkt->pkt_reason = CMD_BADMSG;
734 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
735 STATE_SENT_CMD | STATE_GOT_STATUS);
736 break;
737 case BTSTAT_SENTRST:
738 case BTSTAT_RECVRST:
739 case BTSTAT_BUSRESET:
740 pkt->pkt_reason = CMD_RESET;
741 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
742 STATE_SENT_CMD | STATE_GOT_STATUS);
743 break;
744 case BTSTAT_ABORTQUEUE:
745 pkt->pkt_reason = CMD_ABORTED;
746 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
747 STATE_SENT_CMD | STATE_GOT_STATUS);
748 break;
749 case BTSTAT_HAHARDWARE:
750 case BTSTAT_INVPHASE:
751 case BTSTAT_HATIMEOUT:
752 case BTSTAT_NORESPONSE:
753 case BTSTAT_DISCONNECT:
754 case BTSTAT_HASOFTWARE:
755 case BTSTAT_BUSFREE:
756 case BTSTAT_SENSFAILED:
757 pkt->pkt_reason = CMD_TRAN_ERR;
758 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
759 STATE_SENT_CMD | STATE_GOT_STATUS);
760 break;
761 default:
762 dev_err(pvs->dip, CE_WARN,
763 "!unknown host status code: %d", host_status);
764 pkt->pkt_reason = CMD_TRAN_ERR;
765 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
766 STATE_SENT_CMD | STATE_GOT_STATUS);
767 break;
768 }
769 }
770
771 static void
772 pvscsi_complete_chained(void *arg)
773 {
774 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)arg;
775 pvscsi_cmd_t *c;
776 struct scsi_pkt *pkt;
777
778 while (cmd != NULL) {
779 pvscsi_softc_t *pvs = cmd->cmd_pvs;
780
781 c = cmd->next_cmd;
782 cmd->next_cmd = NULL;
783
784 pkt = CMD2PKT(cmd);
785 if (pkt == NULL)
786 return;
787
788 if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0 &&
789 (cmd->flags & PVSCSI_FLAG_IO_READ) != 0) {
790 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
791 DDI_DMA_SYNC_FORCPU);
792 }
793
794 mutex_enter(&pvs->mutex);
795 /* Release I/O ctx */
796 if (cmd->ctx != NULL)
797 pvscsi_release_ctx(cmd);
798 /* Remove command from queue */
799 pvscsi_remove_from_queue(cmd);
800 mutex_exit(&pvs->mutex);
801
802 if ((cmd->flags & PVSCSI_FLAG_HW_STATUS) != 0) {
803 pvscsi_set_status(cmd);
804 } else {
805 ASSERT((cmd->flags & PVSCSI_FLAGS_NON_HW_COMPLETION) !=
806 0);
807
808 if ((cmd->flags & PVSCSI_FLAG_TIMED_OUT) != 0) {
809 cmd->pkt->pkt_reason = CMD_TIMEOUT;
810 cmd->pkt->pkt_statistics |=
811 (STAT_TIMEOUT | STAT_ABORTED);
812 } else if ((cmd->flags & PVSCSI_FLAG_ABORTED) != 0) {
813 cmd->pkt->pkt_reason = CMD_ABORTED;
814 cmd->pkt->pkt_statistics |=
815 (STAT_TIMEOUT | STAT_ABORTED);
816 } else if ((cmd->flags & PVSCSI_FLAGS_RESET) != 0) {
817 cmd->pkt->pkt_reason = CMD_RESET;
818 if ((cmd->flags & PVSCSI_FLAG_RESET_BUS) != 0) {
819 cmd->pkt->pkt_statistics |=
820 STAT_BUS_RESET;
821 } else {
822 cmd->pkt->pkt_statistics |=
823 STAT_DEV_RESET;
824 }
825 }
826 }
827
828 cmd->flags |= PVSCSI_FLAG_DONE;
829 cmd->flags &= ~PVSCSI_FLAG_TRANSPORT;
830
831 if ((pkt->pkt_flags & FLAG_NOINTR) == 0 &&
832 pkt->pkt_comp != NULL)
833 (*pkt->pkt_comp)(pkt);
834
835 cmd = c;
836 }
837 }
838
839 static void
840 pvscsi_dev_reset(pvscsi_softc_t *pvs, int target)
841 {
842 struct PVSCSICmdDescResetDevice cmd = { 0 };
843
844 cmd.target = target;
845 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof (cmd));
846 }
847
848 static int
849 pvscsi_poll_cmd(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
850 {
851 boolean_t seen_intr;
852 int cycles = (cmd->pkt->pkt_time * 1000000) / USECS_TO_WAIT;
853 int i;
854 pvscsi_cmd_t *dcmd;
855 struct scsi_pkt *pkt = CMD2PKT(cmd);
856
857 /*
858 * Make sure we're not missing any commands completed
859 * concurrently before we have actually disabled interrupts.
860 */
861 mutex_enter(&pvs->rx_mutex);
862 dcmd = pvscsi_process_comp_ring(pvs);
863 mutex_exit(&pvs->rx_mutex);
864
865 pvscsi_complete_chained(dcmd);
866
867 while ((cmd->flags & PVSCSI_FLAG_DONE) == 0) {
868 seen_intr = B_FALSE;
869
870 /* Disable interrupts from H/W */
871 pvscsi_mask_intr(pvs);
872
873 /* Wait for interrupt to arrive */
874 for (i = 0; i < cycles; i++) {
875 uint32_t status;
876
877 mutex_enter(&pvs->rx_mutex);
878 mutex_enter(&pvs->intr_mutex);
879 status = pvscsi_read_intr_status(pvs);
880 if ((status & PVSCSI_INTR_ALL_SUPPORTED) != 0) {
881 /* Check completion ring */
882 mutex_exit(&pvs->intr_mutex);
883 dcmd = pvscsi_process_comp_ring(pvs);
884 mutex_exit(&pvs->rx_mutex);
885 seen_intr = B_TRUE;
886 break;
887 } else {
888 mutex_exit(&pvs->intr_mutex);
889 mutex_exit(&pvs->rx_mutex);
890 drv_usecwait(USECS_TO_WAIT);
891 }
892 }
893
894 /* Enable interrupts from H/W */
895 pvscsi_unmask_intr(pvs);
896
897 if (!seen_intr) {
898 /* No interrupts seen from device during the timeout */
899 mutex_enter(&pvs->tx_mutex);
900 mutex_enter(&pvs->rx_mutex);
901 if ((cmd->flags & PVSCSI_FLAGS_COMPLETION) != 0) {
902 /* Command was cancelled asynchronously */
903 dcmd = NULL;
904 } else if ((pvscsi_abort_cmd(cmd,
905 &dcmd)) == CMD_ABORTED) {
906 /* Command was cancelled in hardware */
907 pkt->pkt_state |= (STAT_TIMEOUT | STAT_ABORTED);
908 pkt->pkt_statistics |= (STAT_TIMEOUT |
909 STAT_ABORTED);
910 pkt->pkt_reason = CMD_TIMEOUT;
911 }
912 mutex_exit(&pvs->rx_mutex);
913 mutex_exit(&pvs->tx_mutex);
914
915 /*
916 * Complete commands that might be on completion list.
917 * Target command can also be on the list in case it was
918 * completed before it could be actually cancelled.
919 */
920 break;
921 }
922
923 pvscsi_complete_chained(dcmd);
924
925 if (!seen_intr)
926 break;
927 }
928
929 return (TRAN_ACCEPT);
930 }
931
932 static void
933 pvscsi_abort_all(struct scsi_address *ap, pvscsi_softc_t *pvs,
934 pvscsi_cmd_t **pending, int marker_flag)
935 {
936 int qlen = pvs->cmd_queue_len;
937 pvscsi_cmd_t *cmd, *pcmd, *phead = NULL;
938
939 ASSERT(mutex_owned(&pvs->rx_mutex));
940 ASSERT(mutex_owned(&pvs->tx_mutex));
941
942 /*
943 * Try to abort all queued commands, merging commands waiting
944 * for completion into a single list to complete them at one
945 * time when mutex is released.
946 */
947 while (qlen > 0) {
948 mutex_enter(&pvs->mutex);
949 cmd = list_remove_head(&pvs->cmd_queue);
950 ASSERT(cmd != NULL);
951
952 qlen--;
953
954 if (ap == NULL || ap->a_target == cmd->cmd_target) {
955 int c = --pvs->cmd_queue_len;
956
957 mutex_exit(&pvs->mutex);
958
959 if (pvscsi_abort_cmd(cmd, &pcmd) == CMD_ABORTED) {
960 /*
961 * Assume command is completely cancelled now,
962 * so mark it as requested.
963 */
964 cmd->flags |= marker_flag;
965 }
966
967 qlen -= (c - pvs->cmd_queue_len);
968
969 /*
970 * Now merge current pending commands with
971 * previous ones.
972 */
973 if (phead == NULL) {
974 phead = pcmd;
975 } else if (pcmd != NULL) {
976 phead->tail_cmd->next_cmd = pcmd;
977 phead->tail_cmd = pcmd->tail_cmd;
978 }
979 } else {
980 list_insert_tail(&pvs->cmd_queue, cmd);
981 mutex_exit(&pvs->mutex);
982 }
983 }
984
985 *pending = phead;
986 }
987
988 static void
989 pvscsi_quiesce_notify(pvscsi_softc_t *pvs)
990 {
991 mutex_enter(&pvs->mutex);
992 if (pvs->cmd_queue_len == 0 &&
993 (pvs->flags & PVSCSI_HBA_QUIESCE_PENDING) != 0) {
994 pvs->flags &= ~PVSCSI_HBA_QUIESCE_PENDING;
995 cv_broadcast(&pvs->quiescevar);
996 }
997 mutex_exit(&pvs->mutex);
998 }
999
1000 static int
1001 pvscsi_transport_command(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
1002 {
1003 struct PVSCSIRingReqDesc *rdesc;
1004 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
1005 struct scsi_pkt *pkt = CMD2PKT(cmd);
1006 uint32_t req_ne = sdesc->reqNumEntriesLog2;
1007
1008 mutex_enter(&pvs->tx_mutex);
1009 mutex_enter(&pvs->mutex);
1010 if (!pvscsi_acquire_ctx(pvs, cmd)) {
1011 mutex_exit(&pvs->mutex);
1012 mutex_exit(&pvs->tx_mutex);
1013 dev_err(pvs->dip, CE_WARN, "!no free ctx available");
1014 return (TRAN_BUSY);
1015 }
1016
1017 if ((sdesc->reqProdIdx - sdesc->cmpConsIdx) >= (1 << req_ne)) {
1018 pvscsi_release_ctx(cmd);
1019 mutex_exit(&pvs->mutex);
1020 mutex_exit(&pvs->tx_mutex);
1021 dev_err(pvs->dip, CE_WARN, "!no free I/O slots available");
1022 return (TRAN_BUSY);
1023 }
1024 mutex_exit(&pvs->mutex);
1025
1026 cmd->flags |= PVSCSI_FLAG_TRANSPORT;
1027
1028 rdesc = REQ_RING(pvs) + (sdesc->reqProdIdx & MASK(req_ne));
1029
1030 bzero(&rdesc->lun, sizeof (rdesc->lun));
1031
1032 rdesc->bus = 0;
1033 rdesc->target = cmd->cmd_target;
1034
1035 if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
1036 bzero((void*)cmd->arqbuf->b_un.b_addr, SENSE_BUFFER_SIZE);
1037 rdesc->senseLen = SENSE_BUFFER_SIZE;
1038 rdesc->senseAddr = cmd->arqc.dmac_laddress;
1039 } else {
1040 rdesc->senseLen = 0;
1041 rdesc->senseAddr = 0;
1042 }
1043
1044 rdesc->vcpuHint = CPU->cpu_id;
1045 rdesc->cdbLen = cmd->cmdlen;
1046 bcopy(cmd->cmd_cdb, rdesc->cdb, cmd->cmdlen);
1047
1048 /* Setup tag info */
1049 if ((cmd->flags & PVSCSI_FLAG_TAG) != 0)
1050 rdesc->tag = cmd->tag;
1051 else
1052 rdesc->tag = MSG_SIMPLE_QTAG;
1053
1054 /* Setup I/O direction and map data buffers */
1055 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1056 if ((cmd->flags & PVSCSI_FLAG_IO_READ) != 0)
1057 rdesc->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
1058 else
1059 rdesc->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
1060 pvscsi_map_buffers(cmd, rdesc);
1061 } else {
1062 rdesc->flags = 0;
1063 }
1064
1065 rdesc->context = pvscsi_map_ctx(pvs, cmd->ctx);
1066 membar_producer();
1067
1068 sdesc->reqProdIdx++;
1069 membar_producer();
1070
1071 mutex_enter(&pvs->mutex);
1072 cmd->timeout_lbolt = ddi_get_lbolt() + SEC_TO_TICK(pkt->pkt_time);
1073 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD);
1074 pvscsi_add_to_queue(cmd);
1075
1076 switch (cmd->pkt->pkt_cdbp[0]) {
1077 case SCMD_READ:
1078 case SCMD_WRITE:
1079 case SCMD_READ_G1:
1080 case SCMD_WRITE_G1:
1081 case SCMD_READ_G4:
1082 case SCMD_WRITE_G4:
1083 case SCMD_READ_G5:
1084 case SCMD_WRITE_G5:
1085 ASSERT((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0);
1086 pvscsi_submit_rw_io(pvs);
1087 break;
1088 default:
1089 pvscsi_submit_nonrw_io(pvs);
1090 break;
1091 }
1092 mutex_exit(&pvs->mutex);
1093 mutex_exit(&pvs->tx_mutex);
1094
1095 return (TRAN_ACCEPT);
1096 }
1097
1098 static int
1099 pvscsi_reset_generic(pvscsi_softc_t *pvs, struct scsi_address *ap)
1100 {
1101 boolean_t bus_reset = (ap == NULL);
1102 int flags;
1103 pvscsi_cmd_t *done, *aborted;
1104
1105 flags = bus_reset ? PVSCSI_FLAG_RESET_BUS : PVSCSI_FLAG_RESET_DEV;
1106
1107 mutex_enter(&pvs->tx_mutex);
1108 mutex_enter(&pvs->rx_mutex);
1109 /* Try to process pending requests */
1110 done = pvscsi_process_comp_ring(pvs);
1111
1112 /* Abort all pending requests */
1113 pvscsi_abort_all(ap, pvs, &aborted, flags);
1114
1115 /* Reset at hardware level */
1116 if (bus_reset) {
1117 pvscsi_reset_bus(pvs);
1118 /* Should never happen after bus reset */
1119 ASSERT(pvscsi_process_comp_ring(pvs) == NULL);
1120 } else {
1121 pvscsi_dev_reset(pvs, ap->a_target);
1122 }
1123 mutex_exit(&pvs->rx_mutex);
1124 mutex_exit(&pvs->tx_mutex);
1125
1126 pvscsi_complete_chained(done);
1127 pvscsi_complete_chained(aborted);
1128
1129 return (1);
1130 }
1131
1132 static void
1133 pvscsi_cmd_ext_free(pvscsi_cmd_t *cmd)
1134 {
1135 struct scsi_pkt *pkt = CMD2PKT(cmd);
1136
1137 if ((cmd->flags & PVSCSI_FLAG_CDB_EXT) != 0) {
1138 kmem_free(pkt->pkt_cdbp, cmd->cmdlen);
1139 cmd->flags &= ~PVSCSI_FLAG_CDB_EXT;
1140 }
1141 if ((cmd->flags & PVSCSI_FLAG_SCB_EXT) != 0) {
1142 kmem_free(pkt->pkt_scbp, cmd->statuslen);
1143 cmd->flags &= ~PVSCSI_FLAG_SCB_EXT;
1144 }
1145 if ((cmd->flags & PVSCSI_FLAG_PRIV_EXT) != 0) {
1146 kmem_free(pkt->pkt_private, cmd->tgtlen);
1147 cmd->flags &= ~PVSCSI_FLAG_PRIV_EXT;
1148 }
1149 }
1150
1151 /* ARGSUSED pvs */
1152 static int
1153 pvscsi_cmd_ext_alloc(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd, int kf)
1154 {
1155 struct scsi_pkt *pkt = CMD2PKT(cmd);
1156 void *buf;
1157
1158 if (cmd->cmdlen > sizeof (cmd->cmd_cdb)) {
1159 if ((buf = kmem_zalloc(cmd->cmdlen, kf)) == NULL)
1160 return (DDI_FAILURE);
1161 pkt->pkt_cdbp = buf;
1162 cmd->flags |= PVSCSI_FLAG_CDB_EXT;
1163 }
1164
1165 if (cmd->statuslen > sizeof (cmd->cmd_scb)) {
1166 if ((buf = kmem_zalloc(cmd->statuslen, kf)) == NULL)
1167 goto out;
1168 pkt->pkt_scbp = buf;
1169 cmd->flags |= PVSCSI_FLAG_SCB_EXT;
1170 cmd->cmd_rqslen = (cmd->statuslen - sizeof (cmd->cmd_scb));
1171 }
1172
1173 if (cmd->tgtlen > sizeof (cmd->tgt_priv)) {
1174 if ((buf = kmem_zalloc(cmd->tgtlen, kf)) == NULL)
1175 goto out;
1176 pkt->pkt_private = buf;
1177 cmd->flags |= PVSCSI_FLAG_PRIV_EXT;
1178 }
1179
1180 return (DDI_SUCCESS);
1181
1182 out:
1183 pvscsi_cmd_ext_free(cmd);
1184
1185 return (DDI_FAILURE);
1186 }
1187
1188 static int
1189 pvscsi_setup_dma_buffer(pvscsi_softc_t *pvs, size_t length,
1190 pvscsi_dma_buf_t *buf)
1191 {
1192 ddi_dma_cookie_t cookie;
1193 uint_t ccount;
1194
1195 if ((ddi_dma_alloc_handle(pvs->dip, &pvscsi_ring_dma_attr,
1196 DDI_DMA_SLEEP, NULL, &buf->dma_handle)) != DDI_SUCCESS) {
1197 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1198 return (DDI_FAILURE);
1199 }
1200
1201 if ((ddi_dma_mem_alloc(buf->dma_handle, length, &pvscsi_dma_attrs,
1202 DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &buf->addr,
1203 &buf->real_length, &buf->acc_handle)) != DDI_SUCCESS) {
1204 dev_err(pvs->dip, CE_WARN,
1205 "!failed to allocate %ld bytes for DMA buffer", length);
1206 ddi_dma_free_handle(&buf->dma_handle);
1207 return (DDI_FAILURE);
1208 }
1209
1210 if ((ddi_dma_addr_bind_handle(buf->dma_handle, NULL, buf->addr,
1211 buf->real_length, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, DDI_DMA_SLEEP,
1212 NULL, &cookie, &ccount)) != DDI_SUCCESS) {
1213 dev_err(pvs->dip, CE_WARN, "!failed to bind DMA buffer");
1214 ddi_dma_free_handle(&buf->dma_handle);
1215 ddi_dma_mem_free(&buf->acc_handle);
1216 return (DDI_FAILURE);
1217 }
1218
1219 /* TODO Support multipart SG regions */
1220 ASSERT(ccount == 1);
1221
1222 buf->pa = cookie.dmac_laddress;
1223
1224 return (DDI_SUCCESS);
1225 }
1226
1227 static void
1228 pvscsi_free_dma_buffer(pvscsi_dma_buf_t *buf)
1229 {
1230 ddi_dma_free_handle(&buf->dma_handle);
1231 ddi_dma_mem_free(&buf->acc_handle);
1232 }
1233
1234 static int
1235 pvscsi_setup_sg(pvscsi_softc_t *pvs)
1236 {
1237 int i;
1238 pvscsi_cmd_ctx_t *ctx;
1239 size_t size = pvs->req_depth * sizeof (pvscsi_cmd_ctx_t);
1240
1241 ctx = pvs->cmd_ctx = kmem_zalloc(size, KM_SLEEP);
1242
1243 for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1244 list_insert_tail(&pvs->cmd_ctx_pool, ctx);
1245 if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1246 &ctx->dma_buf) != DDI_SUCCESS)
1247 goto cleanup;
1248 }
1249
1250 return (DDI_SUCCESS);
1251
1252 cleanup:
1253 for (; i >= 0; --i, --ctx) {
1254 list_remove(&pvs->cmd_ctx_pool, ctx);
1255 pvscsi_free_dma_buffer(&ctx->dma_buf);
1256 }
1257 kmem_free(pvs->cmd_ctx, size);
1258
1259 return (DDI_FAILURE);
1260 }
1261
1262 static void
1263 pvscsi_free_sg(pvscsi_softc_t *pvs)
1264 {
1265 int i;
1266 pvscsi_cmd_ctx_t *ctx = pvs->cmd_ctx;
1267
1268 for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1269 list_remove(&pvs->cmd_ctx_pool, ctx);
1270 pvscsi_free_dma_buffer(&ctx->dma_buf);
1271 }
1272
1273 kmem_free(pvs->cmd_ctx, pvs->req_pages << PAGE_SHIFT);
1274 }
1275
1276 static int
1277 pvscsi_allocate_rings(pvscsi_softc_t *pvs)
1278 {
1279 /* Allocate DMA buffer for rings state */
1280 if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1281 &pvs->rings_state_buf) != DDI_SUCCESS)
1282 return (DDI_FAILURE);
1283
1284 /* Allocate DMA buffer for request ring */
1285 pvs->req_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_REQ_RING);
1286 pvs->req_depth = pvs->req_pages * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
1287 if (pvscsi_setup_dma_buffer(pvs, pvs->req_pages * PAGE_SIZE,
1288 &pvs->req_ring_buf) != DDI_SUCCESS)
1289 goto free_rings_state;
1290
1291 /* Allocate completion ring */
1292 pvs->cmp_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_CMP_RING);
1293 if (pvscsi_setup_dma_buffer(pvs, pvs->cmp_pages * PAGE_SIZE,
1294 &pvs->cmp_ring_buf) != DDI_SUCCESS)
1295 goto free_req_buf;
1296
1297 /* Allocate message ring */
1298 pvs->msg_pages = MIN(pvscsi_msg_ring_pages,
1299 PVSCSI_MAX_NUM_PAGES_MSG_RING);
1300 if (pvscsi_setup_dma_buffer(pvs, pvs->msg_pages * PAGE_SIZE,
1301 &pvs->msg_ring_buf) != DDI_SUCCESS)
1302 goto free_cmp_buf;
1303
1304 return (DDI_SUCCESS);
1305
1306 free_cmp_buf:
1307 pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1308 free_req_buf:
1309 pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1310 free_rings_state:
1311 pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1312
1313 return (DDI_FAILURE);
1314 }
1315
1316 static void
1317 pvscsi_free_rings(pvscsi_softc_t *pvs)
1318 {
1319 pvscsi_free_dma_buffer(&pvs->msg_ring_buf);
1320 pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1321 pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1322 pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1323 }
1324
1325 static void
1326 pvscsi_setup_rings(pvscsi_softc_t *pvs)
1327 {
1328 int i;
1329 struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
1330 struct PVSCSICmdDescSetupRings cmd = { 0 };
1331 uint64_t base;
1332
1333 cmd.ringsStatePPN = pvs->rings_state_buf.pa >> PAGE_SHIFT;
1334 cmd.reqRingNumPages = pvs->req_pages;
1335 cmd.cmpRingNumPages = pvs->cmp_pages;
1336
1337 /* Setup request ring */
1338 base = pvs->req_ring_buf.pa;
1339 for (i = 0; i < pvs->req_pages; i++) {
1340 cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
1341 base += PAGE_SIZE;
1342 }
1343
1344 /* Setup completion ring */
1345 base = pvs->cmp_ring_buf.pa;
1346 for (i = 0; i < pvs->cmp_pages; i++) {
1347 cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
1348 base += PAGE_SIZE;
1349 }
1350
1351 bzero(RINGS_STATE(pvs), PAGE_SIZE);
1352 bzero(REQ_RING(pvs), pvs->req_pages * PAGE_SIZE);
1353 bzero(CMP_RING(pvs), pvs->cmp_pages * PAGE_SIZE);
1354
1355 /* Issue SETUP command */
1356 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof (cmd));
1357
1358 /* Setup message ring */
1359 cmd_msg.numPages = pvs->msg_pages;
1360 base = pvs->msg_ring_buf.pa;
1361
1362 for (i = 0; i < pvs->msg_pages; i++) {
1363 cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
1364 base += PAGE_SIZE;
1365 }
1366 bzero(MSG_RING(pvs), pvs->msg_pages * PAGE_SIZE);
1367
1368 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_MSG_RING, &cmd_msg,
1369 sizeof (cmd_msg));
1370 }
1371
1372 static int
1373 pvscsi_setup_io(pvscsi_softc_t *pvs)
1374 {
1375 int offset, rcount, rn, type;
1376 int ret = DDI_FAILURE;
1377 off_t regsize;
1378 pci_regspec_t *regs;
1379 uint_t regs_length;
1380
1381 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, pvs->dip,
1382 DDI_PROP_DONTPASS, "reg", (int **)®s,
1383 ®s_length) != DDI_PROP_SUCCESS) {
1384 dev_err(pvs->dip, CE_WARN, "!failed to lookup 'reg' property");
1385 return (DDI_FAILURE);
1386 }
1387
1388 rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t);
1389
1390 for (offset = PCI_CONF_BASE0; offset <= PCI_CONF_BASE5; offset += 4) {
1391 for (rn = 0; rn < rcount; ++rn) {
1392 if (PCI_REG_REG_G(regs[rn].pci_phys_hi) == offset) {
1393 type = regs[rn].pci_phys_hi & PCI_ADDR_MASK;
1394 break;
1395 }
1396 }
1397
1398 if (rn >= rcount)
1399 continue;
1400
1401 if (type != PCI_ADDR_IO) {
1402 if (ddi_dev_regsize(pvs->dip, rn,
1403 ®size) != DDI_SUCCESS) {
1404 dev_err(pvs->dip, CE_WARN,
1405 "!failed to get size of reg %d", rn);
1406 goto out;
1407 }
1408 if (regsize == PVSCSI_MEM_SPACE_SIZE) {
1409 if (ddi_regs_map_setup(pvs->dip, rn,
1410 &pvs->mmio_base, 0, 0,
1411 &pvscsi_mmio_attr,
1412 &pvs->mmio_handle) != DDI_SUCCESS) {
1413 dev_err(pvs->dip, CE_WARN,
1414 "!failed to map MMIO BAR");
1415 goto out;
1416 }
1417 ret = DDI_SUCCESS;
1418 break;
1419 }
1420 }
1421 }
1422
1423 out:
1424 ddi_prop_free(regs);
1425
1426 return (ret);
1427 }
1428
1429 static void
1430 pvscsi_free_io(pvscsi_softc_t *pvs)
1431 {
1432 ddi_regs_map_free(&pvs->mmio_handle);
1433 }
1434
1435 static int
1436 pvscsi_enable_intrs(pvscsi_softc_t *pvs)
1437 {
1438 int i, rc, intr_caps;
1439
1440 if ((rc = ddi_intr_get_cap(pvs->intr_htable[0], &intr_caps)) !=
1441 DDI_SUCCESS) {
1442 dev_err(pvs->dip, CE_WARN, "!failed to get interrupt caps");
1443 return (DDI_FAILURE);
1444 }
1445
1446 if ((intr_caps & DDI_INTR_FLAG_BLOCK) != 0) {
1447 if ((rc = ddi_intr_block_enable(pvs->intr_htable,
1448 pvs->intr_cnt)) != DDI_SUCCESS) {
1449 dev_err(pvs->dip, CE_WARN,
1450 "!failed to enable interrupt block");
1451 }
1452 } else {
1453 for (i = 0; i < pvs->intr_cnt; i++) {
1454 if ((rc = ddi_intr_enable(pvs->intr_htable[i])) ==
1455 DDI_SUCCESS)
1456 continue;
1457 dev_err(pvs->dip, CE_WARN,
1458 "!failed to enable interrupt");
1459 while (--i >= 0)
1460 (void) ddi_intr_disable(pvs->intr_htable[i]);
1461 break;
1462 }
1463 }
1464
1465 /* Unmask interrupts */
1466 if (rc == DDI_SUCCESS) {
1467 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
1468 PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
1469 }
1470
1471 return (rc);
1472 }
1473
1474 /* ARGSUSED arg2 */
1475 static uint32_t
1476 pvscsi_intr_handler(caddr_t arg1, caddr_t arg2)
1477 {
1478 boolean_t handled;
1479 pvscsi_softc_t *pvs = (pvscsi_softc_t *)arg1;
1480 uint32_t status;
1481
1482 mutex_enter(&pvs->intr_mutex);
1483 if (pvs->num_pollers > 0) {
1484 mutex_exit(&pvs->intr_mutex);
1485 return (DDI_INTR_CLAIMED);
1486 }
1487
1488 if (pvscsi_enable_msi) {
1489 handled = B_TRUE;
1490 } else {
1491 status = pvscsi_read_intr_status(pvs);
1492 handled = (status & PVSCSI_INTR_ALL_SUPPORTED) != 0;
1493 if (handled)
1494 pvscsi_write_intr_status(pvs, status);
1495 }
1496 mutex_exit(&pvs->intr_mutex);
1497
1498 if (handled) {
1499 boolean_t qnotify;
1500 pvscsi_cmd_t *pending;
1501 pvscsi_msg_t *msg;
1502
1503 mutex_enter(&pvs->rx_mutex);
1504 pending = pvscsi_process_comp_ring(pvs);
1505 msg = pvscsi_process_msg_ring(pvs);
1506 mutex_exit(&pvs->rx_mutex);
1507
1508 mutex_enter(&pvs->mutex);
1509 qnotify = HBA_QUIESCE_PENDING(pvs);
1510 mutex_exit(&pvs->mutex);
1511
1512 if (pending != NULL && ddi_taskq_dispatch(pvs->comp_tq,
1513 pvscsi_complete_chained, pending,
1514 DDI_NOSLEEP) == DDI_FAILURE)
1515 pvscsi_complete_chained(pending);
1516
1517 if (msg != NULL && ddi_taskq_dispatch(pvs->msg_tq,
1518 pvscsi_handle_msg, msg, DDI_NOSLEEP) == DDI_FAILURE) {
1519 dev_err(pvs->dip, CE_WARN,
1520 "!failed to process msg type %d for target %d",
1521 msg->type, msg->target);
1522 kmem_free(msg, sizeof (pvscsi_msg_t));
1523 }
1524
1525 if (qnotify)
1526 pvscsi_quiesce_notify(pvs);
1527 }
1528
1529 return (handled ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
1530 }
1531
1532 static int
1533 pvscsi_register_isr(pvscsi_softc_t *pvs, int type)
1534 {
1535 int navail, nactual;
1536 int i;
1537
1538 if (ddi_intr_get_navail(pvs->dip, type, &navail) != DDI_SUCCESS ||
1539 navail == 0) {
1540 dev_err(pvs->dip, CE_WARN,
1541 "!failed to get number of available interrupts of type %d",
1542 type);
1543 return (DDI_FAILURE);
1544 }
1545 navail = MIN(navail, PVSCSI_MAX_INTRS);
1546
1547 pvs->intr_size = navail * sizeof (ddi_intr_handle_t);
1548 if ((pvs->intr_htable = kmem_alloc(pvs->intr_size, KM_SLEEP)) == NULL) {
1549 dev_err(pvs->dip, CE_WARN,
1550 "!failed to allocate %d bytes for interrupt hashtable",
1551 pvs->intr_size);
1552 return (DDI_FAILURE);
1553 }
1554
1555 if (ddi_intr_alloc(pvs->dip, pvs->intr_htable, type, 0, navail,
1556 &nactual, DDI_INTR_ALLOC_NORMAL) != DDI_SUCCESS || nactual == 0) {
1557 dev_err(pvs->dip, CE_WARN, "!failed to allocate %d interrupts",
1558 navail);
1559 goto free_htable;
1560 }
1561
1562 pvs->intr_cnt = nactual;
1563
1564 if (ddi_intr_get_pri(pvs->intr_htable[0],
1565 (uint_t *)&pvs->intr_pri) != DDI_SUCCESS) {
1566 dev_err(pvs->dip, CE_WARN, "!failed to get interrupt priority");
1567 goto free_intrs;
1568 }
1569
1570 for (i = 0; i < nactual; i++) {
1571 if (ddi_intr_add_handler(pvs->intr_htable[i],
1572 pvscsi_intr_handler, (caddr_t)pvs, NULL) != DDI_SUCCESS) {
1573 dev_err(pvs->dip, CE_WARN,
1574 "!failed to add interrupt handler");
1575 goto free_intrs;
1576 }
1577 }
1578
1579 return (DDI_SUCCESS);
1580
1581 free_intrs:
1582 for (i = 0; i < nactual; i++)
1583 (void) ddi_intr_free(pvs->intr_htable[i]);
1584 free_htable:
1585 kmem_free(pvs->intr_htable, pvs->intr_size);
1586
1587 return (DDI_FAILURE);
1588 }
1589
1590 static void
1591 pvscsi_free_intr_resources(pvscsi_softc_t *pvs)
1592 {
1593 int i;
1594
1595 for (i = 0; i < pvs->intr_cnt; i++) {
1596 (void) ddi_intr_disable(pvs->intr_htable[i]);
1597 (void) ddi_intr_remove_handler(pvs->intr_htable[i]);
1598 (void) ddi_intr_free(pvs->intr_htable[i]);
1599 }
1600 kmem_free(pvs->intr_htable, pvs->intr_size);
1601 }
1602
1603 static int
1604 pvscsi_setup_isr(pvscsi_softc_t *pvs)
1605 {
1606 int intr_types;
1607
1608 if (ddi_intr_get_supported_types(pvs->dip,
1609 &intr_types) != DDI_SUCCESS) {
1610 dev_err(pvs->dip, CE_WARN,
1611 "!failed to get supported interrupt types");
1612 return (DDI_FAILURE);
1613 }
1614
1615 if ((intr_types & DDI_INTR_TYPE_MSIX) != 0 && pvscsi_enable_msi) {
1616 if (pvscsi_register_isr(pvs,
1617 DDI_INTR_TYPE_MSIX) == DDI_SUCCESS) {
1618 pvs->intr_type = DDI_INTR_TYPE_MSIX;
1619 } else {
1620 dev_err(pvs->dip, CE_WARN,
1621 "!failed to install MSI-X interrupt handler");
1622 }
1623 } else if ((intr_types & DDI_INTR_TYPE_MSI) != 0 && pvscsi_enable_msi) {
1624 if (pvscsi_register_isr(pvs,
1625 DDI_INTR_TYPE_MSI) == DDI_SUCCESS) {
1626 pvs->intr_type = DDI_INTR_TYPE_MSI;
1627 } else {
1628 dev_err(pvs->dip, CE_WARN,
1629 "!failed to install MSI interrupt handler");
1630 }
1631 } else if ((intr_types & DDI_INTR_TYPE_FIXED) != 0) {
1632 if (pvscsi_register_isr(pvs,
1633 DDI_INTR_TYPE_FIXED) == DDI_SUCCESS) {
1634 pvs->intr_type = DDI_INTR_TYPE_FIXED;
1635 } else {
1636 dev_err(pvs->dip, CE_WARN,
1637 "!failed to install FIXED interrupt handler");
1638 }
1639 }
1640
1641 return (pvs->intr_type == 0 ? DDI_FAILURE : DDI_SUCCESS);
1642 }
1643
1644 static void
1645 pvscsi_wd_thread(pvscsi_softc_t *pvs)
1646 {
1647 clock_t now;
1648 pvscsi_cmd_t *expired, *c, *cn, **pnext;
1649
1650 mutex_enter(&pvs->mutex);
1651 for (;;) {
1652 expired = NULL;
1653 pnext = NULL;
1654 now = ddi_get_lbolt();
1655
1656 for (c = list_head(&pvs->cmd_queue); c != NULL; ) {
1657 cn = list_next(&pvs->cmd_queue, c);
1658
1659 /*
1660 * Commands with 'FLAG_NOINTR' are watched using their
1661 * own timeouts, so we should not touch them.
1662 */
1663 if ((c->pkt->pkt_flags & FLAG_NOINTR) == 0 &&
1664 now > c->timeout_lbolt) {
1665 dev_err(pvs->dip, CE_WARN,
1666 "!expired command: %p (%ld > %ld)",
1667 (void *)c, now, c->timeout_lbolt);
1668 pvscsi_remove_from_queue(c);
1669 if (expired == NULL)
1670 expired = c;
1671 if (pnext == NULL) {
1672 pnext = &c->next_cmd;
1673 } else {
1674 *pnext = c;
1675 pnext = &c->next_cmd;
1676 }
1677 }
1678 c = cn;
1679 }
1680 mutex_exit(&pvs->mutex);
1681
1682 /* Now cancel all expired commands */
1683 if (expired != NULL) {
1684 struct scsi_address sa = {0};
1685 /* Build a fake SCSI address */
1686 sa.a_hba_tran = pvs->tran;
1687 while (expired != NULL) {
1688 c = expired->next_cmd;
1689 sa.a_target = expired->cmd_target;
1690 sa.a_lun = 0;
1691 (void) pvscsi_abort(&sa, CMD2PKT(expired));
1692 expired = c;
1693 }
1694 }
1695
1696 mutex_enter(&pvs->mutex);
1697 if ((pvs->flags & PVSCSI_DRIVER_SHUTDOWN) != 0) {
1698 /* Finish job */
1699 break;
1700 }
1701 if (cv_reltimedwait(&pvs->wd_condvar, &pvs->mutex,
1702 SEC_TO_TICK(1), TR_CLOCK_TICK) > 0) {
1703 /* Explicitly woken up, finish job */
1704 break;
1705 }
1706 }
1707
1708 /* Confirm thread termination */
1709 cv_signal(&pvs->syncvar);
1710 mutex_exit(&pvs->mutex);
1711 }
1712
1713 static int
1714 pvscsi_ccache_constructor(void *buf, void *cdrarg, int kmflags)
1715 {
1716 int (*callback)(caddr_t);
1717 uint_t cookiec;
1718 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)buf;
1719 pvscsi_softc_t *pvs = cdrarg;
1720 struct scsi_address ap;
1721
1722 callback = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
1723 ap.a_hba_tran = pvs->tran;
1724 ap.a_target = 0;
1725 ap.a_lun = 0;
1726
1727 /* Allocate a DMA handle for data transfers */
1728 if ((ddi_dma_alloc_handle(pvs->dip, &pvs->io_dma_attr, callback,
1729 NULL, &cmd->cmd_dmahdl)) != DDI_SUCCESS) {
1730 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1731 return (-1);
1732 }
1733
1734 /* Setup ARQ buffer */
1735 if ((cmd->arqbuf = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL,
1736 SENSE_BUFFER_SIZE, B_READ, callback, NULL)) == NULL) {
1737 dev_err(pvs->dip, CE_WARN, "!failed to allocate ARQ buffer");
1738 goto free_handle;
1739 }
1740
1741 if (ddi_dma_alloc_handle(pvs->dip, &pvs->hba_dma_attr,
1742 callback, NULL, &cmd->arqhdl) != DDI_SUCCESS) {
1743 dev_err(pvs->dip, CE_WARN,
1744 "!failed to allocate DMA handle for ARQ buffer");
1745 goto free_arqbuf;
1746 }
1747
1748 if (ddi_dma_buf_bind_handle(cmd->arqhdl, cmd->arqbuf,
1749 (DDI_DMA_READ | DDI_DMA_CONSISTENT), callback, NULL,
1750 &cmd->arqc, &cookiec) != DDI_SUCCESS) {
1751 dev_err(pvs->dip, CE_WARN, "!failed to bind ARQ buffer");
1752 goto free_arqhdl;
1753 }
1754
1755 return (0);
1756
1757 free_arqhdl:
1758 ddi_dma_free_handle(&cmd->arqhdl);
1759 free_arqbuf:
1760 scsi_free_consistent_buf(cmd->arqbuf);
1761 free_handle:
1762 ddi_dma_free_handle(&cmd->cmd_dmahdl);
1763
1764 return (-1);
1765 }
1766
1767 /* ARGSUSED cdrarg */
1768 static void
1769 pvscsi_ccache_destructor(void *buf, void *cdrarg)
1770 {
1771 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)buf;
1772
1773 if (cmd->cmd_dmahdl != NULL) {
1774 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1775 ddi_dma_free_handle(&cmd->cmd_dmahdl);
1776 cmd->cmd_dmahdl = NULL;
1777 }
1778
1779 if (cmd->arqhdl != NULL) {
1780 (void) ddi_dma_unbind_handle(cmd->arqhdl);
1781 ddi_dma_free_handle(&cmd->arqhdl);
1782 cmd->arqhdl = NULL;
1783 }
1784
1785 if (cmd->arqbuf != NULL) {
1786 scsi_free_consistent_buf(cmd->arqbuf);
1787 cmd->arqbuf = NULL;
1788 }
1789 }
1790
1791 /* tran_* entry points and setup */
1792 /* ARGSUSED hba_dip tgt_dip hba_tran */
1793 static int
1794 pvscsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
1795 scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
1796 {
1797 pvscsi_softc_t *pvs = SDEV2PRIV(sd);
1798
1799 ASSERT(pvs != NULL);
1800
1801 if (sd->sd_address.a_target >= PVSCSI_MAXTGTS)
1802 return (DDI_FAILURE);
1803
1804 return (DDI_SUCCESS);
1805 }
1806
1807 static int
1808 pvscsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
1809 {
1810 boolean_t poll = ((pkt->pkt_flags & FLAG_NOINTR) != 0);
1811 int rc;
1812 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1813 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1814
1815 ASSERT(cmd->pkt == pkt);
1816 ASSERT(cmd->cmd_pvs == pvs);
1817
1818 /*
1819 * Reinitialize some fields because the packet may
1820 * have been resubmitted.
1821 */
1822 pkt->pkt_reason = CMD_CMPLT;
1823 pkt->pkt_state = 0;
1824 pkt->pkt_statistics = 0;
1825
1826 /* Zero status byte */
1827 *(pkt->pkt_scbp) = 0;
1828
1829 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1830 ASSERT(cmd->cmd_dma_count != 0);
1831 pkt->pkt_resid = cmd->cmd_dma_count;
1832
1833 /*
1834 * Consistent packets need to be synced first
1835 * (only for data going out).
1836 */
1837 if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0) {
1838 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
1839 DDI_DMA_SYNC_FORDEV);
1840 }
1841 }
1842
1843 cmd->cmd_target = ap->a_target;
1844
1845 mutex_enter(&pvs->mutex);
1846 if (HBA_IS_QUIESCED(pvs) && !poll) {
1847 mutex_exit(&pvs->mutex);
1848 return (TRAN_BUSY);
1849 }
1850 mutex_exit(&pvs->mutex);
1851
1852 rc = pvscsi_transport_command(pvs, cmd);
1853
1854 if (poll) {
1855 pvscsi_cmd_t *dcmd;
1856 boolean_t qnotify;
1857
1858 if (rc == TRAN_ACCEPT)
1859 rc = pvscsi_poll_cmd(pvs, cmd);
1860
1861 mutex_enter(&pvs->rx_mutex);
1862 dcmd = pvscsi_process_comp_ring(pvs);
1863 mutex_exit(&pvs->rx_mutex);
1864
1865 mutex_enter(&pvs->mutex);
1866 qnotify = HBA_QUIESCE_PENDING(pvs);
1867 mutex_exit(&pvs->mutex);
1868
1869 pvscsi_complete_chained(dcmd);
1870
1871 if (qnotify)
1872 pvscsi_quiesce_notify(pvs);
1873 }
1874
1875 return (rc);
1876 }
1877
1878 static int
1879 pvscsi_reset(struct scsi_address *ap, int level)
1880 {
1881 pvscsi_softc_t *pvs = AP2PRIV(ap);
1882
1883 switch (level) {
1884 case RESET_ALL:
1885 return (pvscsi_reset_generic(pvs, NULL));
1886 case RESET_TARGET:
1887 ASSERT(ap != NULL);
1888 return (pvscsi_reset_generic(pvs, ap));
1889 default:
1890 return (0);
1891 }
1892 }
1893
1894 static int
1895 pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
1896 {
1897 boolean_t qnotify = B_FALSE;
1898 pvscsi_cmd_t *pending;
1899 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1900
1901 mutex_enter(&pvs->tx_mutex);
1902 mutex_enter(&pvs->rx_mutex);
1903 if (pkt != NULL) {
1904 /* Abort single command */
1905 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1906
1907 if (pvscsi_abort_cmd(cmd, &pending) == CMD_ABORTED) {
1908 /* Assume command is completely cancelled now */
1909 cmd->flags |= PVSCSI_FLAG_ABORTED;
1910 }
1911 } else {
1912 /* Abort all commands on the bus */
1913 pvscsi_abort_all(ap, pvs, &pending, PVSCSI_FLAG_ABORTED);
1914 }
1915 qnotify = HBA_QUIESCE_PENDING(pvs);
1916 mutex_exit(&pvs->rx_mutex);
1917 mutex_exit(&pvs->tx_mutex);
1918
1919 pvscsi_complete_chained(pending);
1920
1921 if (qnotify)
1922 pvscsi_quiesce_notify(pvs);
1923
1924 return (1);
1925 }
1926
1927 /* ARGSUSED tgtonly */
1928 static int
1929 pvscsi_getcap(struct scsi_address *ap, char *cap, int tgtonly)
1930 {
1931 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1932
1933 if (cap == NULL)
1934 return (-1);
1935
1936 switch (scsi_hba_lookup_capstr(cap)) {
1937 case SCSI_CAP_ARQ:
1938 return ((pvs->flags & PVSCSI_HBA_AUTO_REQUEST_SENSE) != 0);
1939 case SCSI_CAP_UNTAGGED_QING:
1940 return (1);
1941 default:
1942 return (-1);
1943 }
1944 }
1945
1946 /* ARGSUSED tgtonly */
1947 static int
1948 pvscsi_setcap(struct scsi_address *ap, char *cap, int value, int tgtonly)
1949 {
1950 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1951
1952 if (cap == NULL)
1953 return (-1);
1954
1955 switch (scsi_hba_lookup_capstr(cap)) {
1956 case SCSI_CAP_ARQ:
1957 mutex_enter(&pvs->mutex);
1958 if (value == 0)
1959 pvs->flags &= ~PVSCSI_HBA_AUTO_REQUEST_SENSE;
1960 else
1961 pvs->flags |= PVSCSI_HBA_AUTO_REQUEST_SENSE;
1962 mutex_exit(&pvs->mutex);
1963 return (1);
1964 default:
1965 return (0);
1966 }
1967 }
1968
1969 static void
1970 pvscsi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
1971 {
1972 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1973 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1974
1975 ASSERT(cmd->cmd_pvs == pvs);
1976
1977 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1978 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
1979 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1980 }
1981
1982 if (cmd->ctx != NULL) {
1983 mutex_enter(&pvs->mutex);
1984 pvscsi_release_ctx(cmd);
1985 mutex_exit(&pvs->mutex);
1986 }
1987
1988 if ((cmd->flags & PVSCSI_FLAGS_EXT) != 0)
1989 pvscsi_cmd_ext_free(cmd);
1990
1991 kmem_cache_free(pvs->cmd_cache, cmd);
1992 }
1993
1994 static struct scsi_pkt *
1995 pvscsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt, struct buf *bp,
1996 int cmdlen, int statuslen, int tgtlen, int flags, int (*callback)(),
1997 caddr_t arg)
1998 {
1999 boolean_t is_new;
2000 int kf = (callback == SLEEP_FUNC) ? KM_SLEEP: KM_NOSLEEP;
2001 int rc, i;
2002 pvscsi_cmd_t *cmd;
2003 pvscsi_softc_t *pvs;
2004
2005 pvs = ap->a_hba_tran->tran_hba_private;
2006 ASSERT(pvs != NULL);
2007
2008 /* Allocate a new SCSI packet */
2009 if (pkt == NULL) {
2010 ddi_dma_handle_t saved_dmahdl, saved_arqhdl;
2011 struct buf *saved_arqbuf;
2012 ddi_dma_cookie_t saved_arqc;
2013
2014 is_new = B_TRUE;
2015
2016 if ((cmd = kmem_cache_alloc(pvs->cmd_cache, kf)) == NULL)
2017 return (NULL);
2018
2019 saved_dmahdl = cmd->cmd_dmahdl;
2020 saved_arqhdl = cmd->arqhdl;
2021 saved_arqbuf = cmd->arqbuf;
2022 saved_arqc = cmd->arqc;
2023
2024 bzero(cmd, sizeof (pvscsi_cmd_t) -
2025 sizeof (cmd->cached_cookies));
2026
2027 cmd->cmd_pvs = pvs;
2028 cmd->cmd_dmahdl = saved_dmahdl;
2029 cmd->arqhdl = saved_arqhdl;
2030 cmd->arqbuf = saved_arqbuf;
2031 cmd->arqc = saved_arqc;
2032
2033 pkt = &cmd->cached_pkt;
2034 pkt->pkt_ha_private = (opaque_t)cmd;
2035 pkt->pkt_address = *ap;
2036 pkt->pkt_scbp = (uint8_t *)&cmd->cmd_scb;
2037 pkt->pkt_cdbp = (uint8_t *)&cmd->cmd_cdb;
2038 pkt->pkt_private = (opaque_t)&cmd->tgt_priv;
2039
2040 cmd->tgtlen = tgtlen;
2041 cmd->statuslen = statuslen;
2042 cmd->cmdlen = cmdlen;
2043 cmd->pkt = pkt;
2044 cmd->ctx = NULL;
2045
2046 /* Allocate extended buffers */
2047 if ((cmdlen > sizeof (cmd->cmd_cdb)) ||
2048 (statuslen > sizeof (cmd->cmd_scb)) ||
2049 (tgtlen > sizeof (cmd->tgt_priv))) {
2050 if (pvscsi_cmd_ext_alloc(pvs, cmd, kf) != DDI_SUCCESS) {
2051 dev_err(pvs->dip, CE_WARN,
2052 "!extent allocation failed");
2053 goto out;
2054 }
2055 }
2056 } else {
2057 is_new = B_FALSE;
2058
2059 cmd = PKT2CMD(pkt);
2060 cmd->flags &= PVSCSI_FLAGS_PERSISTENT;
2061 }
2062
2063 ASSERT((cmd->flags & PVSCSI_FLAG_TRANSPORT) == 0);
2064
2065 if ((flags & PKT_XARQ) != 0)
2066 cmd->flags |= PVSCSI_FLAG_XARQ;
2067
2068 /* Handle partial DMA transfers */
2069 if (cmd->cmd_nwin > 0) {
2070 if (++cmd->cmd_winindex >= cmd->cmd_nwin)
2071 return (NULL);
2072 if (ddi_dma_getwin(cmd->cmd_dmahdl, cmd->cmd_winindex,
2073 &cmd->cmd_dma_offset, &cmd->cmd_dma_len,
2074 &cmd->cmd_dmac, &cmd->cmd_dmaccount) == DDI_FAILURE)
2075 return (NULL);
2076 goto handle_dma_cookies;
2077 }
2078
2079 /* Setup data buffer */
2080 if (bp != NULL && bp->b_bcount > 0 &&
2081 (cmd->flags & PVSCSI_FLAG_DMA_VALID) == 0) {
2082 int dma_flags;
2083
2084 ASSERT(cmd->cmd_dmahdl != NULL);
2085
2086 if ((bp->b_flags & B_READ) != 0) {
2087 cmd->flags |= PVSCSI_FLAG_IO_READ;
2088 dma_flags = DDI_DMA_READ;
2089 } else {
2090 cmd->flags &= ~PVSCSI_FLAG_IO_READ;
2091 dma_flags = DDI_DMA_WRITE;
2092 }
2093 if ((flags & PKT_CONSISTENT) != 0) {
2094 cmd->flags |= PVSCSI_FLAG_IO_IOPB;
2095 dma_flags |= DDI_DMA_CONSISTENT;
2096 }
2097 if ((flags & PKT_DMA_PARTIAL) != 0)
2098 dma_flags |= DDI_DMA_PARTIAL;
2099
2100 rc = ddi_dma_buf_bind_handle(cmd->cmd_dmahdl, bp,
2101 dma_flags, callback, arg, &cmd->cmd_dmac,
2102 &cmd->cmd_dmaccount);
2103 if (rc == DDI_DMA_PARTIAL_MAP) {
2104 (void) ddi_dma_numwin(cmd->cmd_dmahdl,
2105 &cmd->cmd_nwin);
2106 cmd->cmd_winindex = 0;
2107 (void) ddi_dma_getwin(cmd->cmd_dmahdl,
2108 cmd->cmd_winindex, &cmd->cmd_dma_offset,
2109 &cmd->cmd_dma_len, &cmd->cmd_dmac,
2110 &cmd->cmd_dmaccount);
2111 } else if (rc != 0 && rc != DDI_DMA_MAPPED) {
2112 switch (rc) {
2113 case DDI_DMA_NORESOURCES:
2114 bioerror(bp, 0);
2115 break;
2116 case DDI_DMA_BADATTR:
2117 case DDI_DMA_NOMAPPING:
2118 bioerror(bp, EFAULT);
2119 break;
2120 case DDI_DMA_TOOBIG:
2121 default:
2122 bioerror(bp, EINVAL);
2123 break;
2124 }
2125 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2126 goto out;
2127 }
2128
2129 handle_dma_cookies:
2130 ASSERT(cmd->cmd_dmaccount > 0);
2131 if (cmd->cmd_dmaccount > PVSCSI_MAX_SG_SIZE) {
2132 dev_err(pvs->dip, CE_WARN,
2133 "!invalid cookie count: %d (max %d)",
2134 cmd->cmd_dmaccount, PVSCSI_MAX_SG_SIZE);
2135 bioerror(bp, EINVAL);
2136 goto out;
2137 }
2138
2139 cmd->flags |= PVSCSI_FLAG_DMA_VALID;
2140 cmd->cmd_dma_count = cmd->cmd_dmac.dmac_size;
2141 cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2142
2143 cmd->cached_cookies[0] = cmd->cmd_dmac;
2144
2145 /*
2146 * Calculate total amount of bytes for this I/O and
2147 * store cookies for further processing.
2148 */
2149 for (i = 1; i < cmd->cmd_dmaccount; i++) {
2150 ddi_dma_nextcookie(cmd->cmd_dmahdl, &cmd->cmd_dmac);
2151 cmd->cached_cookies[i] = cmd->cmd_dmac;
2152 cmd->cmd_dma_count += cmd->cmd_dmac.dmac_size;
2153 cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2154 }
2155
2156 pkt->pkt_resid = (bp->b_bcount - cmd->cmd_total_dma_count);
2157 }
2158
2159 return (pkt);
2160
2161 out:
2162 if (is_new)
2163 pvscsi_destroy_pkt(ap, pkt);
2164
2165 return (NULL);
2166 }
2167
2168 /* ARGSUSED ap */
2169 static void
2170 pvscsi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt)
2171 {
2172 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
2173
2174 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
2175 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
2176 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2177 }
2178 }
2179
2180 /* ARGSUSED ap */
2181 static void
2182 pvscsi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
2183 {
2184 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
2185
2186 if (cmd->cmd_dmahdl != NULL) {
2187 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
2188 (cmd->flags & PVSCSI_FLAG_IO_READ) ?
2189 DDI_DMA_SYNC_FORCPU : DDI_DMA_SYNC_FORDEV);
2190 }
2191
2192 }
2193
2194 /* ARGSUSED ap flag callback arg */
2195 static int
2196 pvscsi_reset_notify(struct scsi_address *ap, int flag,
2197 void (*callback)(caddr_t), caddr_t arg)
2198 {
2199 return (DDI_FAILURE);
2200 }
2201
2202 static int
2203 pvscsi_quiesce_hba(dev_info_t *dip)
2204 {
2205 pvscsi_softc_t *pvs;
2206 scsi_hba_tran_t *tran;
2207
2208 if ((tran = ddi_get_driver_private(dip)) == NULL ||
2209 (pvs = TRAN2PRIV(tran)) == NULL)
2210 return (-1);
2211
2212 mutex_enter(&pvs->mutex);
2213 if (!HBA_IS_QUIESCED(pvs))
2214 pvs->flags |= PVSCSI_HBA_QUIESCED;
2215
2216 if (pvs->cmd_queue_len != 0) {
2217 /* Outstanding commands present, wait */
2218 pvs->flags |= PVSCSI_HBA_QUIESCE_PENDING;
2219 cv_wait(&pvs->quiescevar, &pvs->mutex);
2220 ASSERT(pvs->cmd_queue_len == 0);
2221 }
2222 mutex_exit(&pvs->mutex);
2223
2224 /* Suspend taskq delivery and complete all scheduled tasks */
2225 ddi_taskq_suspend(pvs->msg_tq);
2226 ddi_taskq_wait(pvs->msg_tq);
2227 ddi_taskq_suspend(pvs->comp_tq);
2228 ddi_taskq_wait(pvs->comp_tq);
2229
2230 return (0);
2231 }
2232
2233 static int
2234 pvscsi_unquiesce_hba(dev_info_t *dip)
2235 {
2236 pvscsi_softc_t *pvs;
2237 scsi_hba_tran_t *tran;
2238
2239 if ((tran = ddi_get_driver_private(dip)) == NULL ||
2240 (pvs = TRAN2PRIV(tran)) == NULL)
2241 return (-1);
2242
2243 mutex_enter(&pvs->mutex);
2244 if (!HBA_IS_QUIESCED(pvs)) {
2245 mutex_exit(&pvs->mutex);
2246 return (0);
2247 }
2248 ASSERT(pvs->cmd_queue_len == 0);
2249 pvs->flags &= ~PVSCSI_HBA_QUIESCED;
2250 mutex_exit(&pvs->mutex);
2251
2252 /* Resume taskq delivery */
2253 ddi_taskq_resume(pvs->msg_tq);
2254 ddi_taskq_resume(pvs->comp_tq);
2255
2256 return (0);
2257 }
2258
2259 static int
2260 pvscsi_bus_config(dev_info_t *pdip, uint_t flags, ddi_bus_config_op_t op,
2261 void *arg, dev_info_t **childp)
2262 {
2263 char *p;
2264 int circ;
2265 int ret = NDI_FAILURE;
2266 long target = 0;
2267 pvscsi_softc_t *pvs;
2268 scsi_hba_tran_t *tran;
2269
2270 tran = ddi_get_driver_private(pdip);
2271 pvs = tran->tran_hba_private;
2272
2273 ndi_devi_enter(pdip, &circ);
2274 switch (op) {
2275 case BUS_CONFIG_ONE:
2276 if ((p = strrchr((char *)arg, '@')) != NULL &&
2277 ddi_strtol(p + 1, NULL, 16, &target) == 0)
2278 ret = pvscsi_config_one(pdip, pvs, (int)target, childp);
2279 break;
2280 case BUS_CONFIG_DRIVER:
2281 case BUS_CONFIG_ALL:
2282 ret = pvscsi_config_all(pdip, pvs);
2283 break;
2284 default:
2285 break;
2286 }
2287
2288 if (ret == NDI_SUCCESS)
2289 ret = ndi_busop_bus_config(pdip, flags, op, arg, childp, 0);
2290 ndi_devi_exit(pdip, circ);
2291
2292 return (ret);
2293 }
2294
2295 static int
2296 pvscsi_hba_setup(pvscsi_softc_t *pvs)
2297 {
2298 scsi_hba_tran_t *hba_tran;
2299
2300 hba_tran = pvs->tran = scsi_hba_tran_alloc(pvs->dip,
2301 SCSI_HBA_CANSLEEP);
2302 ASSERT(pvs->tran != NULL);
2303
2304 hba_tran->tran_hba_private = pvs;
2305 hba_tran->tran_tgt_private = NULL;
2306
2307 hba_tran->tran_tgt_init = pvscsi_tgt_init;
2308 hba_tran->tran_tgt_free = NULL;
2309 hba_tran->tran_tgt_probe = scsi_hba_probe;
2310
2311 hba_tran->tran_start = pvscsi_start;
2312 hba_tran->tran_reset = pvscsi_reset;
2313 hba_tran->tran_abort = pvscsi_abort;
2314 hba_tran->tran_getcap = pvscsi_getcap;
2315 hba_tran->tran_setcap = pvscsi_setcap;
2316 hba_tran->tran_init_pkt = pvscsi_init_pkt;
2317 hba_tran->tran_destroy_pkt = pvscsi_destroy_pkt;
2318
2319 hba_tran->tran_dmafree = pvscsi_dmafree;
2320 hba_tran->tran_sync_pkt = pvscsi_sync_pkt;
2321 hba_tran->tran_reset_notify = pvscsi_reset_notify;
2322
2323 hba_tran->tran_quiesce = pvscsi_quiesce_hba;
2324 hba_tran->tran_unquiesce = pvscsi_unquiesce_hba;
2325 hba_tran->tran_bus_reset = NULL;
2326
2327 hba_tran->tran_add_eventcall = NULL;
2328 hba_tran->tran_get_eventcookie = NULL;
2329 hba_tran->tran_post_event = NULL;
2330 hba_tran->tran_remove_eventcall = NULL;
2331
2332 hba_tran->tran_bus_config = pvscsi_bus_config;
2333
2334 hba_tran->tran_interconnect_type = INTERCONNECT_SAS;
2335
2336 if (scsi_hba_attach_setup(pvs->dip, &pvs->hba_dma_attr, hba_tran,
2337 SCSI_HBA_TRAN_CDB | SCSI_HBA_TRAN_SCB | SCSI_HBA_TRAN_CLONE) !=
2338 DDI_SUCCESS) {
2339 dev_err(pvs->dip, CE_WARN, "!failed to attach HBA");
2340 scsi_hba_tran_free(hba_tran);
2341 pvs->tran = NULL;
2342 return (-1);
2343 }
2344
2345 return (0);
2346 }
2347
2348 static int
2349 pvscsi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2350 {
2351 int instance;
2352 pvscsi_softc_t *pvs;
2353 char buf[32];
2354
2355 ASSERT(scsi_hba_iport_unit_address(dip) == NULL);
2356
2357 switch (cmd) {
2358 case DDI_ATTACH:
2359 case DDI_RESUME:
2360 break;
2361 default:
2362 return (DDI_FAILURE);
2363 }
2364
2365 instance = ddi_get_instance(dip);
2366
2367 /* Allocate softstate information */
2368 if (ddi_soft_state_zalloc(pvscsi_sstate, instance) != DDI_SUCCESS) {
2369 cmn_err(CE_WARN,
2370 "!ddi_soft_state_zalloc() failed for instance %d",
2371 instance);
2372 return (DDI_FAILURE);
2373 }
2374
2375 if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2376 cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2377 instance);
2378 goto fail;
2379 }
2380
2381 /*
2382 * Indicate that we are 'sizeof (scsi_*(9S))' clean, we use
2383 * scsi_pkt_size() instead.
2384 */
2385 scsi_size_clean(dip);
2386
2387 /* Setup HBA instance */
2388 pvs->instance = instance;
2389 pvs->dip = dip;
2390 pvs->hba_dma_attr = pvscsi_hba_dma_attr;
2391 pvs->ring_dma_attr = pvscsi_ring_dma_attr;
2392 pvs->io_dma_attr = pvscsi_io_dma_attr;
2393 mutex_init(&pvs->mutex, "pvscsi instance mutex", MUTEX_DRIVER, NULL);
2394 mutex_init(&pvs->intr_mutex, "pvscsi instance interrupt mutex",
2395 MUTEX_DRIVER, NULL);
2396 mutex_init(&pvs->rx_mutex, "pvscsi rx ring mutex", MUTEX_DRIVER, NULL);
2397 mutex_init(&pvs->tx_mutex, "pvscsi tx ring mutex", MUTEX_DRIVER, NULL);
2398 list_create(&pvs->cmd_ctx_pool, sizeof (pvscsi_cmd_ctx_t),
2399 offsetof(pvscsi_cmd_ctx_t, list));
2400 list_create(&pvs->devnodes, sizeof (pvscsi_device_t),
2401 offsetof(pvscsi_device_t, list));
2402 list_create(&pvs->cmd_queue, sizeof (pvscsi_cmd_t),
2403 offsetof(pvscsi_cmd_t, cmd_queue_node));
2404 cv_init(&pvs->syncvar, "pvscsi synchronization cv", CV_DRIVER, NULL);
2405 cv_init(&pvs->wd_condvar, "pvscsi watchdog cv", CV_DRIVER, NULL);
2406 cv_init(&pvs->quiescevar, "pvscsi quiesce cv", CV_DRIVER, NULL);
2407
2408 (void) sprintf(buf, "pvscsi%d_cache", instance);
2409 pvs->cmd_cache = kmem_cache_create(buf, sizeof (pvscsi_cmd_t), 0,
2410 pvscsi_ccache_constructor, pvscsi_ccache_destructor, NULL,
2411 (void *)pvs, NULL, 0);
2412 if (pvs->cmd_cache == NULL) {
2413 dev_err(pvs->dip, CE_WARN,
2414 "!failed to create a cache for SCSI commands");
2415 goto fail;
2416 }
2417
2418 if ((pvscsi_setup_io(pvs)) != DDI_SUCCESS) {
2419 dev_err(pvs->dip, CE_WARN, "!failed to setup I/O region");
2420 goto free_cache;
2421 }
2422
2423 pvscsi_reset_hba(pvs);
2424
2425 if ((pvscsi_allocate_rings(pvs)) != DDI_SUCCESS) {
2426 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA rings");
2427 goto free_io;
2428 }
2429
2430 pvscsi_setup_rings(pvs);
2431
2432 if (pvscsi_setup_isr(pvs) != DDI_SUCCESS) {
2433 dev_err(pvs->dip, CE_WARN, "!failed to setup ISR");
2434 goto free_rings;
2435 }
2436
2437 if (pvscsi_setup_sg(pvs) != DDI_SUCCESS) {
2438 dev_err(pvs->dip, CE_WARN, "!failed to setup S/G");
2439 goto free_intr;
2440 }
2441
2442 if (pvscsi_hba_setup(pvs) != 0) {
2443 dev_err(pvs->dip, CE_WARN, "!failed to setup HBA");
2444 goto free_sg;
2445 }
2446
2447 if ((pvs->comp_tq = ddi_taskq_create(pvs->dip, "comp_tq",
2448 MIN(UINT16_MAX, ncpus), TASKQ_DEFAULTPRI, 0)) == NULL) {
2449 dev_err(pvs->dip, CE_WARN,
2450 "!failed to create completion taskq");
2451 goto free_sg;
2452 }
2453
2454 if ((pvs->msg_tq = ddi_taskq_create(pvs->dip, "msg_tq",
2455 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
2456 dev_err(pvs->dip, CE_WARN,
2457 "!failed to create message taskq");
2458 goto free_comp_tq;
2459 }
2460
2461 if (pvscsi_enable_intrs(pvs) != DDI_SUCCESS) {
2462 dev_err(pvs->dip, CE_WARN, "!failed to enable interrupts");
2463 goto free_msg_tq;
2464 }
2465
2466 /* Launch watchdog thread */
2467 pvs->wd_thread = thread_create(NULL, 0, pvscsi_wd_thread, pvs, 0, &p0,
2468 TS_RUN, minclsyspri);
2469
2470 return (DDI_SUCCESS);
2471
2472 free_msg_tq:
2473 ddi_taskq_destroy(pvs->msg_tq);
2474 free_comp_tq:
2475 ddi_taskq_destroy(pvs->comp_tq);
2476 free_sg:
2477 pvscsi_free_sg(pvs);
2478 free_intr:
2479 pvscsi_free_intr_resources(pvs);
2480 free_rings:
2481 pvscsi_reset_hba(pvs);
2482 pvscsi_free_rings(pvs);
2483 free_io:
2484 pvscsi_free_io(pvs);
2485 free_cache:
2486 kmem_cache_destroy(pvs->cmd_cache);
2487 fail:
2488 ddi_soft_state_free(pvscsi_sstate, instance);
2489
2490 return (DDI_FAILURE);
2491 }
2492
2493 static int
2494 pvscsi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2495 {
2496 int instance;
2497 pvscsi_softc_t *pvs;
2498
2499 switch (cmd) {
2500 case DDI_DETACH:
2501 break;
2502 default:
2503 return (DDI_FAILURE);
2504 }
2505
2506 instance = ddi_get_instance(dip);
2507 if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2508 cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2509 instance);
2510 return (DDI_FAILURE);
2511 }
2512
2513 pvscsi_reset_hba(pvs);
2514 pvscsi_free_intr_resources(pvs);
2515
2516 /* Shutdown message taskq */
2517 ddi_taskq_wait(pvs->msg_tq);
2518 ddi_taskq_destroy(pvs->msg_tq);
2519
2520 /* Shutdown completion taskq */
2521 ddi_taskq_wait(pvs->comp_tq);
2522 ddi_taskq_destroy(pvs->comp_tq);
2523
2524 /* Shutdown watchdog thread */
2525 mutex_enter(&pvs->mutex);
2526 pvs->flags |= PVSCSI_DRIVER_SHUTDOWN;
2527 cv_signal(&pvs->wd_condvar);
2528 cv_wait(&pvs->syncvar, &pvs->mutex);
2529 mutex_exit(&pvs->mutex);
2530
2531 pvscsi_free_sg(pvs);
2532 pvscsi_free_rings(pvs);
2533 pvscsi_free_io(pvs);
2534
2535 kmem_cache_destroy(pvs->cmd_cache);
2536
2537 mutex_destroy(&pvs->mutex);
2538 mutex_destroy(&pvs->intr_mutex);
2539 mutex_destroy(&pvs->rx_mutex);
2540
2541 cv_destroy(&pvs->syncvar);
2542 cv_destroy(&pvs->wd_condvar);
2543 cv_destroy(&pvs->quiescevar);
2544
2545 ddi_soft_state_free(pvscsi_sstate, instance);
2546 ddi_prop_remove_all(dip);
2547
2548 return (DDI_SUCCESS);
2549 }
2550
2551 static int
2552 pvscsi_ioctl(dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp,
2553 int *rval)
2554 {
2555 int ret;
2556
2557 if (ddi_get_soft_state(pvscsi_sstate, getminor(dev)) == NULL) {
2558 cmn_err(CE_WARN, "!invalid device instance: %d", getminor(dev));
2559 return (ENXIO);
2560 }
2561
2562 /* Try to handle command in a common way */
2563 if ((ret = scsi_hba_ioctl(dev, cmd, data, mode, credp, rval)) != ENOTTY)
2564 return (ret);
2565
2566 cmn_err(CE_WARN, "!unsupported IOCTL command: 0x%X", cmd);
2567
2568 return (ENXIO);
2569 }
2570
2571 static int
2572 pvscsi_quiesce(dev_info_t *devi)
2573 {
2574 scsi_hba_tran_t *tran;
2575 pvscsi_softc_t *pvs;
2576
2577 if ((tran = ddi_get_driver_private(devi)) == NULL)
2578 return (DDI_SUCCESS);
2579
2580 if ((pvs = tran->tran_hba_private) == NULL)
2581 return (DDI_SUCCESS);
2582
2583 /* Mask all interrupts from device */
2584 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
2585
2586 /* Reset the HBA */
2587 pvscsi_reset_hba(pvs);
2588
2589 return (DDI_SUCCESS);
2590 }
2591
2592 /* module */
2593
2594 static struct cb_ops pvscsi_cb_ops = {
2595 .cb_open = scsi_hba_open,
2596 .cb_close = scsi_hba_close,
2597 .cb_strategy = nodev,
2598 .cb_print = nodev,
2599 .cb_dump = nodev,
2600 .cb_read = nodev,
2601 .cb_write = nodev,
2602 .cb_ioctl = pvscsi_ioctl,
2603 .cb_devmap = nodev,
2604 .cb_mmap = nodev,
2605 .cb_segmap = nodev,
2606 .cb_chpoll = nochpoll,
2607 .cb_prop_op = ddi_prop_op,
2608 .cb_str = NULL,
2609 .cb_flag = D_MP,
2610 .cb_rev = CB_REV,
2611 .cb_aread = nodev,
2612 .cb_awrite = nodev
2613 };
2614
2615 static struct dev_ops pvscsi_ops = {
2616 .devo_rev = DEVO_REV,
2617 .devo_refcnt = 0,
2618 .devo_getinfo = ddi_no_info,
2619 .devo_identify = nulldev,
2620 .devo_probe = nulldev,
2621 .devo_attach = pvscsi_attach,
2622 .devo_detach = pvscsi_detach,
2623 .devo_reset = nodev,
2624 .devo_cb_ops = &pvscsi_cb_ops,
2625 .devo_bus_ops = NULL,
2626 .devo_power = NULL,
2627 .devo_quiesce = pvscsi_quiesce
2628 };
2629
2630 #define PVSCSI_IDENT "VMware PVSCSI"
2631
2632 static struct modldrv modldrv = {
2633 &mod_driverops,
2634 PVSCSI_IDENT,
2635 &pvscsi_ops,
2636 };
2637
2638 static struct modlinkage modlinkage = {
2639 MODREV_1,
2640 &modldrv,
2641 NULL
2642 };
2643
2644 int
2645 _init(void)
2646 {
2647 int ret;
2648
2649 if ((ret = ddi_soft_state_init(&pvscsi_sstate,
2650 sizeof (struct pvscsi_softc), PVSCSI_INITIAL_SSTATE_ITEMS)) != 0) {
2651 cmn_err(CE_WARN, "!ddi_soft_state_init() failed");
2652 return (ret);
2653 }
2654
2655 if ((ret = scsi_hba_init(&modlinkage)) != 0) {
2656 cmn_err(CE_WARN, "!scsi_hba_init() failed");
2657 ddi_soft_state_fini(&pvscsi_sstate);
2658 return (ret);
2659 }
2660
2661 if ((ret = mod_install(&modlinkage)) != 0) {
2662 cmn_err(CE_WARN, "!mod_install() failed");
2663 ddi_soft_state_fini(&pvscsi_sstate);
2664 scsi_hba_fini(&modlinkage);
2665 }
2666
2667 return (ret);
2668 }
2669
2670 int
2671 _info(struct modinfo *modinfop)
2672 {
2673 return (mod_info(&modlinkage, modinfop));
2674 }
2675
2676 int
2677 _fini(void)
2678 {
2679 int ret;
2680
2681 if ((ret = mod_remove(&modlinkage)) == 0) {
2682 ddi_soft_state_fini(&pvscsi_sstate);
2683 scsi_hba_fini(&modlinkage);
2684 }
2685
2686 return (ret);
2687 }