1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/errno.h>
28 #include <sys/types.h>
29 #include <sys/conf.h>
30 #include <sys/kmem.h>
31 #include <sys/ddi.h>
32 #include <sys/stat.h>
33 #include <sys/sunddi.h>
34 #include <sys/file.h>
35 #include <sys/open.h>
36 #include <sys/modctl.h>
37 #include <sys/ddi_impldefs.h>
38 #include <sys/sysmacros.h>
39
40 #include <sys/ioat.h>
41
42 static int ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred);
43 static int ioat_close(dev_t devp, int flag, int otyp, cred_t *cred);
44 static int ioat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
45 static int ioat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
46 static int ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
47 void **result);
48 static int ioat_quiesce(dev_info_t *dip);
49
50 static struct cb_ops ioat_cb_ops = {
51 ioat_open, /* cb_open */
52 ioat_close, /* cb_close */
53 nodev, /* cb_strategy */
54 nodev, /* cb_print */
55 nodev, /* cb_dump */
56 nodev, /* cb_read */
57 nodev, /* cb_write */
58 ioat_ioctl, /* cb_ioctl */
59 nodev, /* cb_devmap */
60 nodev, /* cb_mmap */
61 nodev, /* cb_segmap */
62 nochpoll, /* cb_chpoll */
63 ddi_prop_op, /* cb_prop_op */
64 NULL, /* cb_stream */
65 D_NEW | D_MP | D_64BIT | D_DEVMAP, /* cb_flag */
66 CB_REV
67 };
68
69 static struct dev_ops ioat_dev_ops = {
70 DEVO_REV, /* devo_rev */
71 0, /* devo_refcnt */
72 ioat_getinfo, /* devo_getinfo */
73 nulldev, /* devo_identify */
74 nulldev, /* devo_probe */
75 ioat_attach, /* devo_attach */
76 ioat_detach, /* devo_detach */
77 nodev, /* devo_reset */
78 &ioat_cb_ops, /* devo_cb_ops */
79 NULL, /* devo_bus_ops */
80 NULL, /* devo_power */
81 ioat_quiesce, /* devo_quiesce */
82 };
83
84 static struct modldrv ioat_modldrv = {
85 &mod_driverops, /* Type of module. This one is a driver */
86 "ioat driver", /* Name of the module. */
87 &ioat_dev_ops, /* driver ops */
88 };
89
90 static struct modlinkage ioat_modlinkage = {
91 MODREV_1,
92 (void *) &ioat_modldrv,
93 NULL
94 };
95
96
97 void *ioat_statep;
98
99 static int ioat_chip_init(ioat_state_t *state);
100 static void ioat_chip_fini(ioat_state_t *state);
101 static int ioat_drv_init(ioat_state_t *state);
102 static void ioat_drv_fini(ioat_state_t *state);
103 static uint_t ioat_isr(caddr_t parm);
104 static void ioat_intr_enable(ioat_state_t *state);
105 static void ioat_intr_disable(ioat_state_t *state);
106 void ioat_detach_finish(ioat_state_t *state);
107
108
109 ddi_device_acc_attr_t ioat_acc_attr = {
110 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
111 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
112 DDI_STORECACHING_OK_ACC, /* devacc_attr_dataorder */
113 DDI_DEFAULT_ACC /* devacc_attr_access */
114 };
115
116 /* dcopy callback interface */
117 dcopy_device_cb_t ioat_cb = {
118 DCOPY_DEVICECB_V0,
119 0, /* reserved */
120 ioat_channel_alloc,
121 ioat_channel_free,
122 ioat_cmd_alloc,
123 ioat_cmd_free,
124 ioat_cmd_post,
125 ioat_cmd_poll,
126 ioat_unregister_complete
127 };
128
129 /*
130 * _init()
131 */
132 int
133 _init(void)
134 {
135 int e;
136
137 e = ddi_soft_state_init(&ioat_statep, sizeof (ioat_state_t), 1);
138 if (e != 0) {
139 return (e);
140 }
141
142 e = mod_install(&ioat_modlinkage);
143 if (e != 0) {
144 ddi_soft_state_fini(&ioat_statep);
145 return (e);
146 }
147
148 return (0);
149 }
150
151 /*
152 * _info()
153 */
154 int
155 _info(struct modinfo *modinfop)
156 {
157 return (mod_info(&ioat_modlinkage, modinfop));
158 }
159
160 /*
161 * _fini()
162 */
163 int
164 _fini(void)
165 {
166 int e;
167
168 e = mod_remove(&ioat_modlinkage);
169 if (e != 0) {
170 return (e);
171 }
172
173 ddi_soft_state_fini(&ioat_statep);
174
175 return (0);
176 }
177
178 /*
179 * ioat_attach()
180 */
181 static int
182 ioat_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
183 {
184 ioat_state_t *state;
185 int instance;
186 int e;
187
188
189 switch (cmd) {
190 case DDI_ATTACH:
191 break;
192
193 case DDI_RESUME:
194 instance = ddi_get_instance(dip);
195 state = ddi_get_soft_state(ioat_statep, instance);
196 if (state == NULL) {
197 return (DDI_FAILURE);
198 }
199 e = ioat_channel_resume(state);
200 if (e != DDI_SUCCESS) {
201 return (DDI_FAILURE);
202 }
203 ioat_intr_enable(state);
204 return (DDI_SUCCESS);
205
206 default:
207 return (DDI_FAILURE);
208 }
209
210 instance = ddi_get_instance(dip);
211 e = ddi_soft_state_zalloc(ioat_statep, instance);
212 if (e != DDI_SUCCESS) {
213 return (DDI_FAILURE);
214 }
215 state = ddi_get_soft_state(ioat_statep, instance);
216 if (state == NULL) {
217 goto attachfail_get_soft_state;
218 }
219
220 state->is_dip = dip;
221 state->is_instance = instance;
222
223 /* setup the registers, save away some device info */
224 e = ioat_chip_init(state);
225 if (e != DDI_SUCCESS) {
226 goto attachfail_chip_init;
227 }
228
229 /* initialize driver state, must be after chip init */
230 e = ioat_drv_init(state);
231 if (e != DDI_SUCCESS) {
232 goto attachfail_drv_init;
233 }
234
235 /* create the minor node (for the ioctl) */
236 e = ddi_create_minor_node(dip, "ioat", S_IFCHR, instance, DDI_PSEUDO,
237 0);
238 if (e != DDI_SUCCESS) {
239 goto attachfail_minor_node;
240 }
241
242 /* Enable device interrupts */
243 ioat_intr_enable(state);
244
245 /* Report that driver was loaded */
246 ddi_report_dev(dip);
247
248 /* register with dcopy */
249 e = dcopy_device_register(state, &state->is_deviceinfo,
250 &state->is_device_handle);
251 if (e != DCOPY_SUCCESS) {
252 goto attachfail_register;
253 }
254
255 return (DDI_SUCCESS);
256
257 attachfail_register:
258 ioat_intr_disable(state);
259 ddi_remove_minor_node(dip, NULL);
260 attachfail_minor_node:
261 ioat_drv_fini(state);
262 attachfail_drv_init:
263 ioat_chip_fini(state);
264 attachfail_chip_init:
265 attachfail_get_soft_state:
266 (void) ddi_soft_state_free(ioat_statep, instance);
267
268 return (DDI_FAILURE);
269 }
270
271 /*
272 * ioat_detach()
273 */
274 static int
275 ioat_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
276 {
277 ioat_state_t *state;
278 int instance;
279 int e;
280
281
282 instance = ddi_get_instance(dip);
283 state = ddi_get_soft_state(ioat_statep, instance);
284 if (state == NULL) {
285 return (DDI_FAILURE);
286 }
287
288 switch (cmd) {
289 case DDI_DETACH:
290 break;
291
292 case DDI_SUSPEND:
293 ioat_channel_suspend(state);
294 return (DDI_SUCCESS);
295
296 default:
297 return (DDI_FAILURE);
298 }
299
300 /*
301 * try to unregister from dcopy. Since this driver doesn't follow the
302 * traditional parent/child model, we may still be in use so we can't
303 * detach yet.
304 */
305 e = dcopy_device_unregister(&state->is_device_handle);
306 if (e != DCOPY_SUCCESS) {
307 if (e == DCOPY_PENDING) {
308 cmn_err(CE_NOTE, "device busy, performing asynchronous"
309 " detach\n");
310 }
311 return (DDI_FAILURE);
312 }
313
314 ioat_detach_finish(state);
315
316 return (DDI_SUCCESS);
317 }
318
319 /*
320 * ioat_getinfo()
321 */
322 /*ARGSUSED*/
323 static int
324 ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
325 {
326 ioat_state_t *state;
327 int instance;
328 dev_t dev;
329 int e;
330
331
332 dev = (dev_t)arg;
333 instance = getminor(dev);
334
335 switch (cmd) {
336 case DDI_INFO_DEVT2DEVINFO:
337 state = ddi_get_soft_state(ioat_statep, instance);
338 if (state == NULL) {
339 return (DDI_FAILURE);
340 }
341 *result = (void *)state->is_dip;
342 e = DDI_SUCCESS;
343 break;
344
345 case DDI_INFO_DEVT2INSTANCE:
346 *result = (void *)(uintptr_t)instance;
347 e = DDI_SUCCESS;
348 break;
349
350 default:
351 e = DDI_FAILURE;
352 break;
353 }
354
355 return (e);
356 }
357
358
359 /*
360 * ioat_open()
361 */
362 /*ARGSUSED*/
363 static int
364 ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred)
365 {
366 ioat_state_t *state;
367 int instance;
368
369 instance = getminor(*devp);
370 state = ddi_get_soft_state(ioat_statep, instance);
371 if (state == NULL) {
372 return (ENXIO);
373 }
374
375 return (0);
376 }
377
378
379 /*
380 * ioat_close()
381 */
382 /*ARGSUSED*/
383 static int
384 ioat_close(dev_t devp, int flag, int otyp, cred_t *cred)
385 {
386 return (0);
387 }
388
389
390 /*
391 * ioat_chip_init()
392 */
393 static int
394 ioat_chip_init(ioat_state_t *state)
395 {
396 ddi_device_acc_attr_t attr;
397 int e;
398
399
400 attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
401 attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
402 attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
403
404 e = ddi_regs_map_setup(state->is_dip, 1, (caddr_t *)&state->is_genregs,
405 0, 0, &attr, &state->is_reg_handle);
406 if (e != DDI_SUCCESS) {
407 goto chipinitfail_regsmap;
408 }
409
410 /* save away ioat chip info */
411 state->is_num_channels = (uint_t)ddi_get8(state->is_reg_handle,
412 &state->is_genregs[IOAT_CHANCNT]);
413
414 /*
415 * If we get a bogus value, something is wrong with the H/W, fail to
416 * attach.
417 */
418 if (state->is_num_channels == 0) {
419 goto chipinitfail_numchan;
420 }
421
422 state->is_maxxfer = (uint_t)ddi_get8(state->is_reg_handle,
423 &state->is_genregs[IOAT_XFERCAP]);
424 state->is_chanoff = (uintptr_t)ddi_get16(state->is_reg_handle,
425 (uint16_t *)&state->is_genregs[IOAT_PERPORT_OFF]);
426 state->is_cbver = (uint_t)ddi_get8(state->is_reg_handle,
427 &state->is_genregs[IOAT_CBVER]);
428 state->is_intrdelay = (uint_t)ddi_get16(state->is_reg_handle,
429 (uint16_t *)&state->is_genregs[IOAT_INTRDELAY]);
430 state->is_status = (uint_t)ddi_get16(state->is_reg_handle,
431 (uint16_t *)&state->is_genregs[IOAT_CSSTATUS]);
432 state->is_capabilities = (uint_t)ddi_get32(state->is_reg_handle,
433 (uint32_t *)&state->is_genregs[IOAT_DMACAPABILITY]);
434
435 if (state->is_cbver & 0x10) {
436 state->is_ver = IOAT_CBv1;
437 } else if (state->is_cbver & 0x20) {
438 state->is_ver = IOAT_CBv2;
439 } else {
440 goto chipinitfail_version;
441 }
442
443 return (DDI_SUCCESS);
444
445 chipinitfail_version:
446 chipinitfail_numchan:
447 ddi_regs_map_free(&state->is_reg_handle);
448 chipinitfail_regsmap:
449 return (DDI_FAILURE);
450 }
451
452
453 /*
454 * ioat_chip_fini()
455 */
456 static void
457 ioat_chip_fini(ioat_state_t *state)
458 {
459 ddi_regs_map_free(&state->is_reg_handle);
460 }
461
462
463 /*
464 * ioat_drv_init()
465 */
466 static int
467 ioat_drv_init(ioat_state_t *state)
468 {
469 ddi_acc_handle_t handle;
470 int e;
471
472
473 mutex_init(&state->is_mutex, NULL, MUTEX_DRIVER, NULL);
474
475 state->is_deviceinfo.di_dip = state->is_dip;
476 state->is_deviceinfo.di_num_dma = state->is_num_channels;
477 state->is_deviceinfo.di_maxxfer = state->is_maxxfer;
478 state->is_deviceinfo.di_capabilities = state->is_capabilities;
479 state->is_deviceinfo.di_cb = &ioat_cb;
480
481 e = pci_config_setup(state->is_dip, &handle);
482 if (e != DDI_SUCCESS) {
483 goto drvinitfail_config_setup;
484 }
485
486 /* read in Vendor ID */
487 state->is_deviceinfo.di_id = (uint64_t)pci_config_get16(handle, 0);
488 state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 16;
489
490 /* read in Device ID */
491 state->is_deviceinfo.di_id |= (uint64_t)pci_config_get16(handle, 2);
492 state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 32;
493
494 /* Add in chipset version */
495 state->is_deviceinfo.di_id |= (uint64_t)state->is_cbver;
496 pci_config_teardown(&handle);
497
498 e = ddi_intr_hilevel(state->is_dip, 0);
499 if (e != 0) {
500 cmn_err(CE_WARN, "hilevel interrupt not supported\n");
501 goto drvinitfail_hilevel;
502 }
503
504 /* we don't support MSIs for v2 yet */
505 e = ddi_add_intr(state->is_dip, 0, NULL, NULL, ioat_isr,
506 (caddr_t)state);
507 if (e != DDI_SUCCESS) {
508 goto drvinitfail_add_intr;
509 }
510
511 e = ddi_get_iblock_cookie(state->is_dip, 0, &state->is_iblock_cookie);
512 if (e != DDI_SUCCESS) {
513 goto drvinitfail_iblock_cookie;
514 }
515
516 e = ioat_channel_init(state);
517 if (e != DDI_SUCCESS) {
518 goto drvinitfail_channel_init;
519 }
520
521 return (DDI_SUCCESS);
522
523 drvinitfail_channel_init:
524 drvinitfail_iblock_cookie:
525 ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
526 drvinitfail_add_intr:
527 drvinitfail_hilevel:
528 drvinitfail_config_setup:
529 mutex_destroy(&state->is_mutex);
530
531 return (DDI_FAILURE);
532 }
533
534
535 /*
536 * ioat_drv_fini()
537 */
538 static void
539 ioat_drv_fini(ioat_state_t *state)
540 {
541 ioat_channel_fini(state);
542 ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
543 mutex_destroy(&state->is_mutex);
544 }
545
546
547 /*
548 * ioat_unregister_complete()
549 */
550 void
551 ioat_unregister_complete(void *device_private, int status)
552 {
553 ioat_state_t *state;
554
555
556 state = device_private;
557
558 if (status != DCOPY_SUCCESS) {
559 cmn_err(CE_WARN, "asynchronous detach aborted\n");
560 return;
561 }
562
563 cmn_err(CE_CONT, "detach completing\n");
564 ioat_detach_finish(state);
565 }
566
567
568 /*
569 * ioat_detach_finish()
570 */
571 void
572 ioat_detach_finish(ioat_state_t *state)
573 {
574 ioat_intr_disable(state);
575 ddi_remove_minor_node(state->is_dip, NULL);
576 ioat_drv_fini(state);
577 ioat_chip_fini(state);
578 (void) ddi_soft_state_free(ioat_statep, state->is_instance);
579 }
580
581
582 /*
583 * ioat_intr_enable()
584 */
585 static void
586 ioat_intr_enable(ioat_state_t *state)
587 {
588 uint32_t intr_status;
589
590
591 /* Clear any pending interrupts */
592 intr_status = ddi_get32(state->is_reg_handle,
593 (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
594 if (intr_status != 0) {
595 ddi_put32(state->is_reg_handle,
596 (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS],
597 intr_status);
598 }
599
600 /* Enable interrupts on the device */
601 ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
602 IOAT_INTRCTL_MASTER_EN);
603 }
604
605
606 /*
607 * ioat_intr_disable()
608 */
609 static void
610 ioat_intr_disable(ioat_state_t *state)
611 {
612 /*
613 * disable interrupts on the device. A read of the interrupt control
614 * register clears the enable bit.
615 */
616 (void) ddi_get8(state->is_reg_handle,
617 &state->is_genregs[IOAT_INTRCTL]);
618 }
619
620
621 /*
622 * ioat_isr()
623 */
624 static uint_t
625 ioat_isr(caddr_t parm)
626 {
627 uint32_t intr_status;
628 ioat_state_t *state;
629 uint8_t intrctrl;
630 uint32_t chan;
631 uint_t r;
632 int i;
633
634 state = (ioat_state_t *)parm;
635
636 intrctrl = ddi_get8(state->is_reg_handle,
637 &state->is_genregs[IOAT_INTRCTL]);
638 /* master interrupt enable should always be set */
639 ASSERT(intrctrl & IOAT_INTRCTL_MASTER_EN);
640
641 /* If the interrupt status bit isn't set, it's not ours */
642 if (!(intrctrl & IOAT_INTRCTL_INTR_STAT)) {
643 /* re-set master interrupt enable (since it clears on read) */
644 ddi_put8(state->is_reg_handle,
645 &state->is_genregs[IOAT_INTRCTL], intrctrl);
646 return (DDI_INTR_UNCLAIMED);
647 }
648
649 /* see which channels generated the interrupt */
650 intr_status = ddi_get32(state->is_reg_handle,
651 (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
652
653 /* call the intr handler for the channels */
654 r = DDI_INTR_UNCLAIMED;
655 chan = 1;
656 for (i = 0; i < state->is_num_channels; i++) {
657 if (intr_status & chan) {
658 ioat_channel_intr(&state->is_channel[i]);
659 r = DDI_INTR_CLAIMED;
660 }
661 chan = chan << 1;
662 }
663
664 /*
665 * if interrupt status bit was set, there should have been an
666 * attention status bit set too.
667 */
668 ASSERT(r == DDI_INTR_CLAIMED);
669
670 /* re-set master interrupt enable (since it clears on read) */
671 ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
672 intrctrl);
673
674 return (r);
675 }
676
677 static int
678 ioat_quiesce(dev_info_t *dip)
679 {
680 ioat_state_t *state;
681 int instance;
682
683 instance = ddi_get_instance(dip);
684 state = ddi_get_soft_state(ioat_statep, instance);
685 if (state == NULL) {
686 return (DDI_FAILURE);
687 }
688
689 ioat_intr_disable(state);
690 ioat_channel_quiesce(state);
691
692 return (DDI_SUCCESS);
693 }