1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * dcopy.c
29 * dcopy misc module
30 */
31
32 #include <sys/conf.h>
33 #include <sys/kmem.h>
34 #include <sys/ddi.h>
35 #include <sys/sunddi.h>
36 #include <sys/modctl.h>
37 #include <sys/sysmacros.h>
38 #include <sys/atomic.h>
39
40
41 #include <sys/dcopy.h>
42 #include <sys/dcopy_device.h>
43
44
45 /* Number of entries per channel to allocate */
46 uint_t dcopy_channel_size = 1024;
47
48
49 typedef struct dcopy_list_s {
50 list_t dl_list;
51 kmutex_t dl_mutex;
52 uint_t dl_cnt; /* num entries on list */
53 } dcopy_list_t;
54
55 /* device state for register/unregister */
56 struct dcopy_device_s {
57 /* DMA device drivers private pointer */
58 void *dc_device_private;
59
60 /* to track list of channels from this DMA device */
61 dcopy_list_t dc_devchan_list;
62 list_node_t dc_device_list_node;
63
64 /*
65 * dc_removing_cnt track how many channels still have to be freed up
66 * before it's safe to allow the DMA device driver to detach.
67 */
68 uint_t dc_removing_cnt;
69 dcopy_device_cb_t *dc_cb;
70
71 dcopy_device_info_t dc_info;
72
73 };
74
75 typedef struct dcopy_stats_s {
76 kstat_named_t cs_bytes_xfer;
77 kstat_named_t cs_cmd_alloc;
78 kstat_named_t cs_cmd_post;
79 kstat_named_t cs_cmd_poll;
80 kstat_named_t cs_notify_poll;
81 kstat_named_t cs_notify_pending;
82 kstat_named_t cs_id;
83 kstat_named_t cs_capabilities;
84 } dcopy_stats_t;
85
86 /* DMA channel state */
87 struct dcopy_channel_s {
88 /* DMA driver channel private pointer */
89 void *ch_channel_private;
90
91 /* shortcut to device callbacks */
92 dcopy_device_cb_t *ch_cb;
93
94 /*
95 * number of outstanding allocs for this channel. used to track when
96 * it's safe to free up this channel so the DMA device driver can
97 * detach.
98 */
99 uint64_t ch_ref_cnt;
100
101 /* state for if channel needs to be removed when ch_ref_cnt gets to 0 */
102 boolean_t ch_removing;
103
104 list_node_t ch_devchan_list_node;
105 list_node_t ch_globalchan_list_node;
106
107 /*
108 * per channel list of commands actively blocking waiting for
109 * completion.
110 */
111 dcopy_list_t ch_poll_list;
112
113 /* pointer back to our device */
114 struct dcopy_device_s *ch_device;
115
116 dcopy_query_channel_t ch_info;
117
118 kstat_t *ch_kstat;
119 dcopy_stats_t ch_stat;
120 };
121
122 /*
123 * If grabbing both device_list mutex & globalchan_list mutex,
124 * Always grab globalchan_list mutex before device_list mutex
125 */
126 typedef struct dcopy_state_s {
127 dcopy_list_t d_device_list;
128 dcopy_list_t d_globalchan_list;
129 } dcopy_state_t;
130 dcopy_state_t *dcopy_statep;
131
132
133 /* Module Driver Info */
134 static struct modlmisc dcopy_modlmisc = {
135 &mod_miscops,
136 "dcopy kernel module"
137 };
138
139 /* Module Linkage */
140 static struct modlinkage dcopy_modlinkage = {
141 MODREV_1,
142 &dcopy_modlmisc,
143 NULL
144 };
145
146 static int dcopy_init();
147 static void dcopy_fini();
148
149 static int dcopy_list_init(dcopy_list_t *list, size_t node_size,
150 offset_t link_offset);
151 static void dcopy_list_fini(dcopy_list_t *list);
152 static void dcopy_list_push(dcopy_list_t *list, void *list_node);
153 static void *dcopy_list_pop(dcopy_list_t *list);
154
155 static void dcopy_device_cleanup(dcopy_device_handle_t device,
156 boolean_t do_callback);
157
158 static int dcopy_stats_init(dcopy_handle_t channel);
159 static void dcopy_stats_fini(dcopy_handle_t channel);
160
161
162 /*
163 * _init()
164 */
165 int
166 _init()
167 {
168 int e;
169
170 e = dcopy_init();
171 if (e != 0) {
172 return (e);
173 }
174
175 return (mod_install(&dcopy_modlinkage));
176 }
177
178
179 /*
180 * _info()
181 */
182 int
183 _info(struct modinfo *modinfop)
184 {
185 return (mod_info(&dcopy_modlinkage, modinfop));
186 }
187
188
189 /*
190 * _fini()
191 */
192 int
193 _fini()
194 {
195 int e;
196
197 e = mod_remove(&dcopy_modlinkage);
198 if (e != 0) {
199 return (e);
200 }
201
202 dcopy_fini();
203
204 return (e);
205 }
206
207 /*
208 * dcopy_init()
209 */
210 static int
211 dcopy_init()
212 {
213 int e;
214
215
216 dcopy_statep = kmem_zalloc(sizeof (*dcopy_statep), KM_SLEEP);
217
218 /* Initialize the list we use to track device register/unregister */
219 e = dcopy_list_init(&dcopy_statep->d_device_list,
220 sizeof (struct dcopy_device_s),
221 offsetof(struct dcopy_device_s, dc_device_list_node));
222 if (e != DCOPY_SUCCESS) {
223 goto dcopyinitfail_device;
224 }
225
226 /* Initialize the list we use to track all DMA channels */
227 e = dcopy_list_init(&dcopy_statep->d_globalchan_list,
228 sizeof (struct dcopy_channel_s),
229 offsetof(struct dcopy_channel_s, ch_globalchan_list_node));
230 if (e != DCOPY_SUCCESS) {
231 goto dcopyinitfail_global;
232 }
233
234 return (0);
235
236 dcopyinitfail_cback:
237 dcopy_list_fini(&dcopy_statep->d_globalchan_list);
238 dcopyinitfail_global:
239 dcopy_list_fini(&dcopy_statep->d_device_list);
240 dcopyinitfail_device:
241 kmem_free(dcopy_statep, sizeof (*dcopy_statep));
242
243 return (-1);
244 }
245
246
247 /*
248 * dcopy_fini()
249 */
250 static void
251 dcopy_fini()
252 {
253 /*
254 * if mod_remove was successfull, we shouldn't have any
255 * devices/channels to worry about.
256 */
257 ASSERT(list_head(&dcopy_statep->d_globalchan_list.dl_list) == NULL);
258 ASSERT(list_head(&dcopy_statep->d_device_list.dl_list) == NULL);
259
260 dcopy_list_fini(&dcopy_statep->d_globalchan_list);
261 dcopy_list_fini(&dcopy_statep->d_device_list);
262 kmem_free(dcopy_statep, sizeof (*dcopy_statep));
263 }
264
265
266 /* *** EXTERNAL INTERFACE *** */
267 /*
268 * dcopy_query()
269 */
270 void
271 dcopy_query(dcopy_query_t *query)
272 {
273 query->dq_version = DCOPY_QUERY_V0;
274 query->dq_num_channels = dcopy_statep->d_globalchan_list.dl_cnt;
275 }
276
277
278 /*
279 * dcopy_alloc()
280 */
281 /*ARGSUSED*/
282 int
283 dcopy_alloc(int flags, dcopy_handle_t *handle)
284 {
285 dcopy_handle_t channel;
286 dcopy_list_t *list;
287
288
289 /*
290 * we don't use the dcopy_list_* code here because we need to due
291 * some non-standard stuff.
292 */
293
294 list = &dcopy_statep->d_globalchan_list;
295
296 /*
297 * if nothing is on the channel list, return DCOPY_NORESOURCES. This
298 * can happen if there aren't any DMA device registered.
299 */
300 mutex_enter(&list->dl_mutex);
301 channel = list_head(&list->dl_list);
302 if (channel == NULL) {
303 mutex_exit(&list->dl_mutex);
304 return (DCOPY_NORESOURCES);
305 }
306
307 /*
308 * increment the reference count, and pop the channel off the head and
309 * push it on the tail. This ensures we rotate through the channels.
310 * DMA channels are shared.
311 */
312 channel->ch_ref_cnt++;
313 list_remove(&list->dl_list, channel);
314 list_insert_tail(&list->dl_list, channel);
315 mutex_exit(&list->dl_mutex);
316
317 *handle = (dcopy_handle_t)channel;
318 return (DCOPY_SUCCESS);
319 }
320
321
322 /*
323 * dcopy_free()
324 */
325 void
326 dcopy_free(dcopy_handle_t *channel)
327 {
328 dcopy_device_handle_t device;
329 dcopy_list_t *list;
330 boolean_t cleanup = B_FALSE;
331
332
333 ASSERT(*channel != NULL);
334
335 /*
336 * we don't need to add the channel back to the list since we never
337 * removed it. decrement the reference count.
338 */
339 list = &dcopy_statep->d_globalchan_list;
340 mutex_enter(&list->dl_mutex);
341 (*channel)->ch_ref_cnt--;
342
343 /*
344 * if we need to remove this channel, and the reference count is down
345 * to 0, decrement the number of channels which still need to be
346 * removed on the device.
347 */
348 if ((*channel)->ch_removing && ((*channel)->ch_ref_cnt == 0)) {
349 device = (*channel)->ch_device;
350 mutex_enter(&device->dc_devchan_list.dl_mutex);
351 device->dc_removing_cnt--;
352 if (device->dc_removing_cnt == 0) {
353 cleanup = B_TRUE;
354 }
355 mutex_exit(&device->dc_devchan_list.dl_mutex);
356 }
357 mutex_exit(&list->dl_mutex);
358
359 /*
360 * if there are no channels which still need to be removed, cleanup the
361 * device state and call back into the DMA device driver to tell them
362 * the device is free.
363 */
364 if (cleanup) {
365 dcopy_device_cleanup(device, B_TRUE);
366 }
367
368 *channel = NULL;
369 }
370
371
372 /*
373 * dcopy_query_channel()
374 */
375 void
376 dcopy_query_channel(dcopy_handle_t channel, dcopy_query_channel_t *query)
377 {
378 *query = channel->ch_info;
379 }
380
381
382 /*
383 * dcopy_cmd_alloc()
384 */
385 int
386 dcopy_cmd_alloc(dcopy_handle_t handle, int flags, dcopy_cmd_t *cmd)
387 {
388 dcopy_handle_t channel;
389 dcopy_cmd_priv_t priv;
390 int e;
391
392
393 channel = handle;
394
395 atomic_inc_64(&channel->ch_stat.cs_cmd_alloc.value.ui64);
396 e = channel->ch_cb->cb_cmd_alloc(channel->ch_channel_private, flags,
397 cmd);
398 if (e == DCOPY_SUCCESS) {
399 priv = (*cmd)->dp_private;
400 priv->pr_channel = channel;
401 /*
402 * we won't initialize the blocking state until we actually
403 * need to block.
404 */
405 priv->pr_block_init = B_FALSE;
406 }
407
408 return (e);
409 }
410
411
412 /*
413 * dcopy_cmd_free()
414 */
415 void
416 dcopy_cmd_free(dcopy_cmd_t *cmd)
417 {
418 dcopy_handle_t channel;
419 dcopy_cmd_priv_t priv;
420
421
422 ASSERT(*cmd != NULL);
423
424 priv = (*cmd)->dp_private;
425 channel = priv->pr_channel;
426
427 /* if we initialized the blocking state, clean it up too */
428 if (priv->pr_block_init) {
429 cv_destroy(&priv->pr_cv);
430 mutex_destroy(&priv->pr_mutex);
431 }
432
433 channel->ch_cb->cb_cmd_free(channel->ch_channel_private, cmd);
434 }
435
436
437 /*
438 * dcopy_cmd_post()
439 */
440 int
441 dcopy_cmd_post(dcopy_cmd_t cmd)
442 {
443 dcopy_handle_t channel;
444 int e;
445
446
447 channel = cmd->dp_private->pr_channel;
448
449 atomic_inc_64(&channel->ch_stat.cs_cmd_post.value.ui64);
450 if (cmd->dp_cmd == DCOPY_CMD_COPY) {
451 atomic_add_64(&channel->ch_stat.cs_bytes_xfer.value.ui64,
452 cmd->dp.copy.cc_size);
453 }
454 e = channel->ch_cb->cb_cmd_post(channel->ch_channel_private, cmd);
455 if (e != DCOPY_SUCCESS) {
456 return (e);
457 }
458
459 return (DCOPY_SUCCESS);
460 }
461
462
463 /*
464 * dcopy_cmd_poll()
465 */
466 int
467 dcopy_cmd_poll(dcopy_cmd_t cmd, int flags)
468 {
469 dcopy_handle_t channel;
470 dcopy_cmd_priv_t priv;
471 int e;
472
473
474 priv = cmd->dp_private;
475 channel = priv->pr_channel;
476
477 /*
478 * if the caller is trying to block, they needed to post the
479 * command with DCOPY_CMD_INTR set.
480 */
481 if ((flags & DCOPY_POLL_BLOCK) && !(cmd->dp_flags & DCOPY_CMD_INTR)) {
482 return (DCOPY_FAILURE);
483 }
484
485 atomic_inc_64(&channel->ch_stat.cs_cmd_poll.value.ui64);
486
487 repoll:
488 e = channel->ch_cb->cb_cmd_poll(channel->ch_channel_private, cmd);
489 if (e == DCOPY_PENDING) {
490 /*
491 * if the command is still active, and the blocking flag
492 * is set.
493 */
494 if (flags & DCOPY_POLL_BLOCK) {
495
496 /*
497 * if we haven't initialized the state, do it now. A
498 * command can be re-used, so it's possible it's
499 * already been initialized.
500 */
501 if (!priv->pr_block_init) {
502 priv->pr_block_init = B_TRUE;
503 mutex_init(&priv->pr_mutex, NULL, MUTEX_DRIVER,
504 NULL);
505 cv_init(&priv->pr_cv, NULL, CV_DRIVER, NULL);
506 priv->pr_cmd = cmd;
507 }
508
509 /* push it on the list for blocking commands */
510 priv->pr_wait = B_TRUE;
511 dcopy_list_push(&channel->ch_poll_list, priv);
512
513 mutex_enter(&priv->pr_mutex);
514 /*
515 * it's possible we already cleared pr_wait before we
516 * grabbed the mutex.
517 */
518 if (priv->pr_wait) {
519 cv_wait(&priv->pr_cv, &priv->pr_mutex);
520 }
521 mutex_exit(&priv->pr_mutex);
522
523 /*
524 * the command has completed, go back and poll so we
525 * get the status.
526 */
527 goto repoll;
528 }
529 }
530
531 return (e);
532 }
533
534 /* *** END OF EXTERNAL INTERFACE *** */
535
536 /*
537 * dcopy_list_init()
538 */
539 static int
540 dcopy_list_init(dcopy_list_t *list, size_t node_size, offset_t link_offset)
541 {
542 mutex_init(&list->dl_mutex, NULL, MUTEX_DRIVER, NULL);
543 list_create(&list->dl_list, node_size, link_offset);
544 list->dl_cnt = 0;
545
546 return (DCOPY_SUCCESS);
547 }
548
549
550 /*
551 * dcopy_list_fini()
552 */
553 static void
554 dcopy_list_fini(dcopy_list_t *list)
555 {
556 list_destroy(&list->dl_list);
557 mutex_destroy(&list->dl_mutex);
558 }
559
560
561 /*
562 * dcopy_list_push()
563 */
564 static void
565 dcopy_list_push(dcopy_list_t *list, void *list_node)
566 {
567 mutex_enter(&list->dl_mutex);
568 list_insert_tail(&list->dl_list, list_node);
569 list->dl_cnt++;
570 mutex_exit(&list->dl_mutex);
571 }
572
573
574 /*
575 * dcopy_list_pop()
576 */
577 static void *
578 dcopy_list_pop(dcopy_list_t *list)
579 {
580 list_node_t *list_node;
581
582 mutex_enter(&list->dl_mutex);
583 list_node = list_head(&list->dl_list);
584 if (list_node == NULL) {
585 mutex_exit(&list->dl_mutex);
586 return (list_node);
587 }
588 list->dl_cnt--;
589 list_remove(&list->dl_list, list_node);
590 mutex_exit(&list->dl_mutex);
591
592 return (list_node);
593 }
594
595
596 /* *** DEVICE INTERFACE *** */
597 /*
598 * dcopy_device_register()
599 */
600 int
601 dcopy_device_register(void *device_private, dcopy_device_info_t *info,
602 dcopy_device_handle_t *handle)
603 {
604 struct dcopy_channel_s *channel;
605 struct dcopy_device_s *device;
606 int e;
607 int i;
608
609
610 /* initialize the per device state */
611 device = kmem_zalloc(sizeof (*device), KM_SLEEP);
612 device->dc_device_private = device_private;
613 device->dc_info = *info;
614 device->dc_removing_cnt = 0;
615 device->dc_cb = info->di_cb;
616
617 /*
618 * we have a per device channel list so we can remove a device in the
619 * future.
620 */
621 e = dcopy_list_init(&device->dc_devchan_list,
622 sizeof (struct dcopy_channel_s),
623 offsetof(struct dcopy_channel_s, ch_devchan_list_node));
624 if (e != DCOPY_SUCCESS) {
625 goto registerfail_devchan;
626 }
627
628 /*
629 * allocate state for each channel, allocate the channel, and then add
630 * the devices dma channels to the devices channel list.
631 */
632 for (i = 0; i < info->di_num_dma; i++) {
633 channel = kmem_zalloc(sizeof (*channel), KM_SLEEP);
634 channel->ch_device = device;
635 channel->ch_removing = B_FALSE;
636 channel->ch_ref_cnt = 0;
637 channel->ch_cb = info->di_cb;
638
639 e = info->di_cb->cb_channel_alloc(device_private, channel,
640 DCOPY_SLEEP, dcopy_channel_size, &channel->ch_info,
641 &channel->ch_channel_private);
642 if (e != DCOPY_SUCCESS) {
643 kmem_free(channel, sizeof (*channel));
644 goto registerfail_alloc;
645 }
646
647 e = dcopy_stats_init(channel);
648 if (e != DCOPY_SUCCESS) {
649 info->di_cb->cb_channel_free(
650 &channel->ch_channel_private);
651 kmem_free(channel, sizeof (*channel));
652 goto registerfail_alloc;
653 }
654
655 e = dcopy_list_init(&channel->ch_poll_list,
656 sizeof (struct dcopy_cmd_priv_s),
657 offsetof(struct dcopy_cmd_priv_s, pr_poll_list_node));
658 if (e != DCOPY_SUCCESS) {
659 dcopy_stats_fini(channel);
660 info->di_cb->cb_channel_free(
661 &channel->ch_channel_private);
662 kmem_free(channel, sizeof (*channel));
663 goto registerfail_alloc;
664 }
665
666 dcopy_list_push(&device->dc_devchan_list, channel);
667 }
668
669 /* add the device to device list */
670 dcopy_list_push(&dcopy_statep->d_device_list, device);
671
672 /*
673 * add the device's dma channels to the global channel list (where
674 * dcopy_alloc's come from)
675 */
676 mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
677 mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
678 channel = list_head(&device->dc_devchan_list.dl_list);
679 while (channel != NULL) {
680 list_insert_tail(&dcopy_statep->d_globalchan_list.dl_list,
681 channel);
682 dcopy_statep->d_globalchan_list.dl_cnt++;
683 channel = list_next(&device->dc_devchan_list.dl_list, channel);
684 }
685 mutex_exit(&dcopy_statep->d_device_list.dl_mutex);
686 mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);
687
688 *handle = device;
689
690 /* last call-back into kernel for dcopy KAPI enabled */
691 uioa_dcopy_enable();
692
693 return (DCOPY_SUCCESS);
694
695 registerfail_alloc:
696 channel = list_head(&device->dc_devchan_list.dl_list);
697 while (channel != NULL) {
698 /* remove from the list */
699 channel = dcopy_list_pop(&device->dc_devchan_list);
700 ASSERT(channel != NULL);
701
702 dcopy_list_fini(&channel->ch_poll_list);
703 dcopy_stats_fini(channel);
704 info->di_cb->cb_channel_free(&channel->ch_channel_private);
705 kmem_free(channel, sizeof (*channel));
706 }
707
708 dcopy_list_fini(&device->dc_devchan_list);
709 registerfail_devchan:
710 kmem_free(device, sizeof (*device));
711
712 return (DCOPY_FAILURE);
713 }
714
715
716 /*
717 * dcopy_device_unregister()
718 */
719 /*ARGSUSED*/
720 int
721 dcopy_device_unregister(dcopy_device_handle_t *handle)
722 {
723 struct dcopy_channel_s *channel;
724 dcopy_device_handle_t device;
725 boolean_t device_busy;
726
727 /* first call-back into kernel for dcopy KAPI disable */
728 uioa_dcopy_disable();
729
730 device = *handle;
731 device_busy = B_FALSE;
732
733 /*
734 * remove the devices dma channels from the global channel list (where
735 * dcopy_alloc's come from)
736 */
737 mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
738 mutex_enter(&device->dc_devchan_list.dl_mutex);
739 channel = list_head(&device->dc_devchan_list.dl_list);
740 while (channel != NULL) {
741 /*
742 * if the channel has outstanding allocs, mark it as having
743 * to be removed and increment the number of channels which
744 * need to be removed in the device state too.
745 */
746 if (channel->ch_ref_cnt != 0) {
747 channel->ch_removing = B_TRUE;
748 device_busy = B_TRUE;
749 device->dc_removing_cnt++;
750 }
751 dcopy_statep->d_globalchan_list.dl_cnt--;
752 list_remove(&dcopy_statep->d_globalchan_list.dl_list, channel);
753 channel = list_next(&device->dc_devchan_list.dl_list, channel);
754 }
755 mutex_exit(&device->dc_devchan_list.dl_mutex);
756 mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);
757
758 /*
759 * if there are channels which still need to be removed, we will clean
760 * up the device state after they are freed up.
761 */
762 if (device_busy) {
763 return (DCOPY_PENDING);
764 }
765
766 dcopy_device_cleanup(device, B_FALSE);
767
768 *handle = NULL;
769 return (DCOPY_SUCCESS);
770 }
771
772
773 /*
774 * dcopy_device_cleanup()
775 */
776 static void
777 dcopy_device_cleanup(dcopy_device_handle_t device, boolean_t do_callback)
778 {
779 struct dcopy_channel_s *channel;
780
781 /*
782 * remove all the channels in the device list, free them, and clean up
783 * the state.
784 */
785 mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
786 channel = list_head(&device->dc_devchan_list.dl_list);
787 while (channel != NULL) {
788 device->dc_devchan_list.dl_cnt--;
789 list_remove(&device->dc_devchan_list.dl_list, channel);
790 dcopy_list_fini(&channel->ch_poll_list);
791 dcopy_stats_fini(channel);
792 channel->ch_cb->cb_channel_free(&channel->ch_channel_private);
793 kmem_free(channel, sizeof (*channel));
794 channel = list_head(&device->dc_devchan_list.dl_list);
795 }
796
797 /* remove it from the list of devices */
798 list_remove(&dcopy_statep->d_device_list.dl_list, device);
799
800 mutex_exit(&dcopy_statep->d_device_list.dl_mutex);
801
802 /*
803 * notify the DMA device driver that the device is free to be
804 * detached.
805 */
806 if (do_callback) {
807 device->dc_cb->cb_unregister_complete(
808 device->dc_device_private, DCOPY_SUCCESS);
809 }
810
811 dcopy_list_fini(&device->dc_devchan_list);
812 kmem_free(device, sizeof (*device));
813 }
814
815
816 /*
817 * dcopy_device_channel_notify()
818 */
819 /*ARGSUSED*/
820 void
821 dcopy_device_channel_notify(dcopy_handle_t handle, int status)
822 {
823 struct dcopy_channel_s *channel;
824 dcopy_list_t *poll_list;
825 dcopy_cmd_priv_t priv;
826 int e;
827
828
829 ASSERT(status == DCOPY_COMPLETION);
830 channel = handle;
831
832 poll_list = &channel->ch_poll_list;
833
834 /*
835 * when we get a completion notification from the device, go through
836 * all of the commands blocking on this channel and see if they have
837 * completed. Remove the command and wake up the block thread if they
838 * have. Once we hit a command which is still pending, we are done
839 * polling since commands in a channel complete in order.
840 */
841 mutex_enter(&poll_list->dl_mutex);
842 if (poll_list->dl_cnt != 0) {
843 priv = list_head(&poll_list->dl_list);
844 while (priv != NULL) {
845 atomic_inc_64(&channel->
846 ch_stat.cs_notify_poll.value.ui64);
847 e = channel->ch_cb->cb_cmd_poll(
848 channel->ch_channel_private,
849 priv->pr_cmd);
850 if (e == DCOPY_PENDING) {
851 atomic_inc_64(&channel->
852 ch_stat.cs_notify_pending.value.ui64);
853 break;
854 }
855
856 poll_list->dl_cnt--;
857 list_remove(&poll_list->dl_list, priv);
858
859 mutex_enter(&priv->pr_mutex);
860 priv->pr_wait = B_FALSE;
861 cv_signal(&priv->pr_cv);
862 mutex_exit(&priv->pr_mutex);
863
864 priv = list_head(&poll_list->dl_list);
865 }
866 }
867
868 mutex_exit(&poll_list->dl_mutex);
869 }
870
871
872 /*
873 * dcopy_stats_init()
874 */
875 static int
876 dcopy_stats_init(dcopy_handle_t channel)
877 {
878 #define CHANSTRSIZE 20
879 char chanstr[CHANSTRSIZE];
880 dcopy_stats_t *stats;
881 int instance;
882 char *name;
883
884
885 stats = &channel->ch_stat;
886 name = (char *)ddi_driver_name(channel->ch_device->dc_info.di_dip);
887 instance = ddi_get_instance(channel->ch_device->dc_info.di_dip);
888
889 (void) snprintf(chanstr, CHANSTRSIZE, "channel%d",
890 (uint32_t)channel->ch_info.qc_chan_num);
891
892 channel->ch_kstat = kstat_create(name, instance, chanstr, "misc",
893 KSTAT_TYPE_NAMED, sizeof (dcopy_stats_t) / sizeof (kstat_named_t),
894 KSTAT_FLAG_VIRTUAL);
895 if (channel->ch_kstat == NULL) {
896 return (DCOPY_FAILURE);
897 }
898 channel->ch_kstat->ks_data = stats;
899
900 kstat_named_init(&stats->cs_bytes_xfer, "bytes_xfer",
901 KSTAT_DATA_UINT64);
902 kstat_named_init(&stats->cs_cmd_alloc, "cmd_alloc",
903 KSTAT_DATA_UINT64);
904 kstat_named_init(&stats->cs_cmd_post, "cmd_post",
905 KSTAT_DATA_UINT64);
906 kstat_named_init(&stats->cs_cmd_poll, "cmd_poll",
907 KSTAT_DATA_UINT64);
908 kstat_named_init(&stats->cs_notify_poll, "notify_poll",
909 KSTAT_DATA_UINT64);
910 kstat_named_init(&stats->cs_notify_pending, "notify_pending",
911 KSTAT_DATA_UINT64);
912 kstat_named_init(&stats->cs_id, "id",
913 KSTAT_DATA_UINT64);
914 kstat_named_init(&stats->cs_capabilities, "capabilities",
915 KSTAT_DATA_UINT64);
916
917 kstat_install(channel->ch_kstat);
918
919 channel->ch_stat.cs_id.value.ui64 = channel->ch_info.qc_id;
920 channel->ch_stat.cs_capabilities.value.ui64 =
921 channel->ch_info.qc_capabilities;
922
923 return (DCOPY_SUCCESS);
924 }
925
926
927 /*
928 * dcopy_stats_fini()
929 */
930 static void
931 dcopy_stats_fini(dcopy_handle_t channel)
932 {
933 kstat_delete(channel->ch_kstat);
934 }
935 /* *** END OF DEVICE INTERFACE *** */