1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/errno.h>
28 #include <sys/types.h>
29 #include <sys/conf.h>
30 #include <sys/kmem.h>
31 #include <sys/ddi.h>
32 #include <sys/stat.h>
33 #include <sys/sunddi.h>
34 #include <sys/file.h>
35 #include <sys/open.h>
36 #include <sys/modctl.h>
37 #include <sys/ddi_impldefs.h>
38 #include <sys/sysmacros.h>
39
40 #include <sys/ioat.h>
41
42 static int ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred);
43 static int ioat_close(dev_t devp, int flag, int otyp, cred_t *cred);
44 static int ioat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
45 static int ioat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
46 static int ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
47 void **result);
48 static int ioat_quiesce(dev_info_t *dip);
49
50 static struct cb_ops ioat_cb_ops = {
51 ioat_open, /* cb_open */
52 ioat_close, /* cb_close */
53 nodev, /* cb_strategy */
54 nodev, /* cb_print */
55 nodev, /* cb_dump */
56 nodev, /* cb_read */
57 nodev, /* cb_write */
58 ioat_ioctl, /* cb_ioctl */
59 nodev, /* cb_devmap */
60 nodev, /* cb_mmap */
61 nodev, /* cb_segmap */
62 nochpoll, /* cb_chpoll */
63 ddi_prop_op, /* cb_prop_op */
64 NULL, /* cb_stream */
65 D_NEW | D_MP | D_64BIT | D_DEVMAP, /* cb_flag */
66 CB_REV
67 };
68
69 static struct dev_ops ioat_dev_ops = {
70 DEVO_REV, /* devo_rev */
71 0, /* devo_refcnt */
72 ioat_getinfo, /* devo_getinfo */
73 nulldev, /* devo_identify */
74 nulldev, /* devo_probe */
75 ioat_attach, /* devo_attach */
76 ioat_detach, /* devo_detach */
77 nodev, /* devo_reset */
78 &ioat_cb_ops, /* devo_cb_ops */
79 NULL, /* devo_bus_ops */
80 NULL, /* devo_power */
81 ioat_quiesce, /* devo_quiesce */
82 };
83
84 static struct modldrv ioat_modldrv = {
85 &mod_driverops, /* Type of module. This one is a driver */
86 "ioat driver", /* Name of the module. */
87 &ioat_dev_ops, /* driver ops */
88 };
89
90 static struct modlinkage ioat_modlinkage = {
91 MODREV_1,
92 { (void *) &ioat_modldrv, NULL }
93 };
94
95
96 void *ioat_statep;
97
98 static int ioat_chip_init(ioat_state_t *state);
99 static void ioat_chip_fini(ioat_state_t *state);
100 static int ioat_drv_init(ioat_state_t *state);
101 static void ioat_drv_fini(ioat_state_t *state);
102 static uint_t ioat_isr(caddr_t parm);
103 static void ioat_intr_enable(ioat_state_t *state);
104 static void ioat_intr_disable(ioat_state_t *state);
105 void ioat_detach_finish(ioat_state_t *state);
106
107
108 ddi_device_acc_attr_t ioat_acc_attr = {
109 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
110 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
111 DDI_STORECACHING_OK_ACC, /* devacc_attr_dataorder */
112 DDI_DEFAULT_ACC /* devacc_attr_access */
113 };
114
115 /* dcopy callback interface */
116 dcopy_device_cb_t ioat_cb = {
117 DCOPY_DEVICECB_V0,
118 0, /* reserved */
119 ioat_channel_alloc,
120 ioat_channel_free,
121 ioat_cmd_alloc,
122 ioat_cmd_free,
123 ioat_cmd_post,
124 ioat_cmd_poll,
125 ioat_unregister_complete
126 };
127
128 /*
129 * _init()
130 */
131 int
132 _init(void)
133 {
134 int e;
135
136 e = ddi_soft_state_init(&ioat_statep, sizeof (ioat_state_t), 1);
137 if (e != 0) {
138 return (e);
139 }
140
141 e = mod_install(&ioat_modlinkage);
142 if (e != 0) {
143 ddi_soft_state_fini(&ioat_statep);
144 return (e);
145 }
146
147 return (0);
148 }
149
150 /*
151 * _info()
152 */
153 int
154 _info(struct modinfo *modinfop)
155 {
156 return (mod_info(&ioat_modlinkage, modinfop));
157 }
158
159 /*
160 * _fini()
161 */
162 int
163 _fini(void)
164 {
165 int e;
166
167 e = mod_remove(&ioat_modlinkage);
168 if (e != 0) {
169 return (e);
170 }
171
172 ddi_soft_state_fini(&ioat_statep);
173
174 return (0);
175 }
176
177 /*
178 * ioat_attach()
179 */
180 static int
181 ioat_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
182 {
183 ioat_state_t *state;
184 int instance;
185 int e;
186
187
188 switch (cmd) {
189 case DDI_ATTACH:
190 break;
191
192 case DDI_RESUME:
193 instance = ddi_get_instance(dip);
194 state = ddi_get_soft_state(ioat_statep, instance);
195 if (state == NULL) {
196 return (DDI_FAILURE);
197 }
198 e = ioat_channel_resume(state);
199 if (e != DDI_SUCCESS) {
200 return (DDI_FAILURE);
201 }
202 ioat_intr_enable(state);
203 return (DDI_SUCCESS);
204
205 default:
206 return (DDI_FAILURE);
207 }
208
209 instance = ddi_get_instance(dip);
210 e = ddi_soft_state_zalloc(ioat_statep, instance);
211 if (e != DDI_SUCCESS) {
212 return (DDI_FAILURE);
213 }
214 state = ddi_get_soft_state(ioat_statep, instance);
215 if (state == NULL) {
216 goto attachfail_get_soft_state;
217 }
218
219 state->is_dip = dip;
220 state->is_instance = instance;
221
222 /* setup the registers, save away some device info */
223 e = ioat_chip_init(state);
224 if (e != DDI_SUCCESS) {
225 goto attachfail_chip_init;
226 }
227
228 /* initialize driver state, must be after chip init */
229 e = ioat_drv_init(state);
230 if (e != DDI_SUCCESS) {
231 goto attachfail_drv_init;
232 }
233
234 /* create the minor node (for the ioctl) */
235 e = ddi_create_minor_node(dip, "ioat", S_IFCHR, instance, DDI_PSEUDO,
236 0);
237 if (e != DDI_SUCCESS) {
238 goto attachfail_minor_node;
239 }
240
241 /* Enable device interrupts */
242 ioat_intr_enable(state);
243
244 /* Report that driver was loaded */
245 ddi_report_dev(dip);
246
247 /* register with dcopy */
248 e = dcopy_device_register(state, &state->is_deviceinfo,
249 &state->is_device_handle);
250 if (e != DCOPY_SUCCESS) {
251 goto attachfail_register;
252 }
253
254 return (DDI_SUCCESS);
255
256 attachfail_register:
257 ioat_intr_disable(state);
258 ddi_remove_minor_node(dip, NULL);
259 attachfail_minor_node:
260 ioat_drv_fini(state);
261 attachfail_drv_init:
262 ioat_chip_fini(state);
263 attachfail_chip_init:
264 attachfail_get_soft_state:
265 (void) ddi_soft_state_free(ioat_statep, instance);
266
267 return (DDI_FAILURE);
268 }
269
270 /*
271 * ioat_detach()
272 */
273 static int
274 ioat_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
275 {
276 ioat_state_t *state;
277 int instance;
278 int e;
279
280
281 instance = ddi_get_instance(dip);
282 state = ddi_get_soft_state(ioat_statep, instance);
283 if (state == NULL) {
284 return (DDI_FAILURE);
285 }
286
287 switch (cmd) {
288 case DDI_DETACH:
289 break;
290
291 case DDI_SUSPEND:
292 ioat_channel_suspend(state);
293 return (DDI_SUCCESS);
294
295 default:
296 return (DDI_FAILURE);
297 }
298
299 /*
300 * try to unregister from dcopy. Since this driver doesn't follow the
301 * traditional parent/child model, we may still be in use so we can't
302 * detach yet.
303 */
304 e = dcopy_device_unregister(&state->is_device_handle);
305 if (e != DCOPY_SUCCESS) {
306 if (e == DCOPY_PENDING) {
307 cmn_err(CE_NOTE, "device busy, performing asynchronous"
308 " detach\n");
309 }
310 return (DDI_FAILURE);
311 }
312
313 ioat_detach_finish(state);
314
315 return (DDI_SUCCESS);
316 }
317
318 /*
319 * ioat_getinfo()
320 */
321 /*ARGSUSED*/
322 static int
323 ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
324 {
325 ioat_state_t *state;
326 int instance;
327 dev_t dev;
328 int e;
329
330
331 dev = (dev_t)arg;
332 instance = getminor(dev);
333
334 switch (cmd) {
335 case DDI_INFO_DEVT2DEVINFO:
336 state = ddi_get_soft_state(ioat_statep, instance);
337 if (state == NULL) {
338 return (DDI_FAILURE);
339 }
340 *result = (void *)state->is_dip;
341 e = DDI_SUCCESS;
342 break;
343
344 case DDI_INFO_DEVT2INSTANCE:
345 *result = (void *)(uintptr_t)instance;
346 e = DDI_SUCCESS;
347 break;
348
349 default:
350 e = DDI_FAILURE;
351 break;
352 }
353
354 return (e);
355 }
356
357
358 /*
359 * ioat_open()
360 */
361 /*ARGSUSED*/
362 static int
363 ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred)
364 {
365 ioat_state_t *state;
366 int instance;
367
368 instance = getminor(*devp);
369 state = ddi_get_soft_state(ioat_statep, instance);
370 if (state == NULL) {
371 return (ENXIO);
372 }
373
374 return (0);
375 }
376
377
378 /*
379 * ioat_close()
380 */
381 /*ARGSUSED*/
382 static int
383 ioat_close(dev_t devp, int flag, int otyp, cred_t *cred)
384 {
385 return (0);
386 }
387
388
389 /*
390 * ioat_chip_init()
391 */
392 static int
393 ioat_chip_init(ioat_state_t *state)
394 {
395 ddi_device_acc_attr_t attr;
396 int e;
397
398
399 attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
400 attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
401 attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
402
403 e = ddi_regs_map_setup(state->is_dip, 1, (caddr_t *)&state->is_genregs,
404 0, 0, &attr, &state->is_reg_handle);
405 if (e != DDI_SUCCESS) {
406 goto chipinitfail_regsmap;
407 }
408
409 /* save away ioat chip info */
410 state->is_num_channels = (uint_t)ddi_get8(state->is_reg_handle,
411 &state->is_genregs[IOAT_CHANCNT]);
412
413 /*
414 * If we get a bogus value, something is wrong with the H/W, fail to
415 * attach.
416 */
417 if (state->is_num_channels == 0) {
418 goto chipinitfail_numchan;
419 }
420
421 state->is_maxxfer = (uint_t)ddi_get8(state->is_reg_handle,
422 &state->is_genregs[IOAT_XFERCAP]);
423 state->is_chanoff = (uintptr_t)ddi_get16(state->is_reg_handle,
424 (uint16_t *)&state->is_genregs[IOAT_PERPORT_OFF]);
425 state->is_cbver = (uint_t)ddi_get8(state->is_reg_handle,
426 &state->is_genregs[IOAT_CBVER]);
427 state->is_intrdelay = (uint_t)ddi_get16(state->is_reg_handle,
428 (uint16_t *)&state->is_genregs[IOAT_INTRDELAY]);
429 state->is_status = (uint_t)ddi_get16(state->is_reg_handle,
430 (uint16_t *)&state->is_genregs[IOAT_CSSTATUS]);
431 state->is_capabilities = (uint_t)ddi_get32(state->is_reg_handle,
432 (uint32_t *)&state->is_genregs[IOAT_DMACAPABILITY]);
433
434 if (state->is_cbver & 0x10) {
435 state->is_ver = IOAT_CBv1;
436 } else if (state->is_cbver & 0x20) {
437 state->is_ver = IOAT_CBv2;
438 } else {
439 goto chipinitfail_version;
440 }
441
442 return (DDI_SUCCESS);
443
444 chipinitfail_version:
445 chipinitfail_numchan:
446 ddi_regs_map_free(&state->is_reg_handle);
447 chipinitfail_regsmap:
448 return (DDI_FAILURE);
449 }
450
451
452 /*
453 * ioat_chip_fini()
454 */
455 static void
456 ioat_chip_fini(ioat_state_t *state)
457 {
458 ddi_regs_map_free(&state->is_reg_handle);
459 }
460
461
462 /*
463 * ioat_drv_init()
464 */
465 static int
466 ioat_drv_init(ioat_state_t *state)
467 {
468 ddi_acc_handle_t handle;
469 int e;
470
471
472 mutex_init(&state->is_mutex, NULL, MUTEX_DRIVER, NULL);
473
474 state->is_deviceinfo.di_dip = state->is_dip;
475 state->is_deviceinfo.di_num_dma = state->is_num_channels;
476 state->is_deviceinfo.di_maxxfer = state->is_maxxfer;
477 state->is_deviceinfo.di_capabilities = state->is_capabilities;
478 state->is_deviceinfo.di_cb = &ioat_cb;
479
480 e = pci_config_setup(state->is_dip, &handle);
481 if (e != DDI_SUCCESS) {
482 goto drvinitfail_config_setup;
483 }
484
485 /* read in Vendor ID */
486 state->is_deviceinfo.di_id = (uint64_t)pci_config_get16(handle, 0);
487 state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 16;
488
489 /* read in Device ID */
490 state->is_deviceinfo.di_id |= (uint64_t)pci_config_get16(handle, 2);
491 state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 32;
492
493 /* Add in chipset version */
494 state->is_deviceinfo.di_id |= (uint64_t)state->is_cbver;
495 pci_config_teardown(&handle);
496
497 e = ddi_intr_hilevel(state->is_dip, 0);
498 if (e != 0) {
499 cmn_err(CE_WARN, "hilevel interrupt not supported\n");
500 goto drvinitfail_hilevel;
501 }
502
503 /* we don't support MSIs for v2 yet */
504 e = ddi_add_intr(state->is_dip, 0, NULL, NULL, ioat_isr,
505 (caddr_t)state);
506 if (e != DDI_SUCCESS) {
507 goto drvinitfail_add_intr;
508 }
509
510 e = ddi_get_iblock_cookie(state->is_dip, 0, &state->is_iblock_cookie);
511 if (e != DDI_SUCCESS) {
512 goto drvinitfail_iblock_cookie;
513 }
514
515 e = ioat_channel_init(state);
516 if (e != DDI_SUCCESS) {
517 goto drvinitfail_channel_init;
518 }
519
520 return (DDI_SUCCESS);
521
522 drvinitfail_channel_init:
523 drvinitfail_iblock_cookie:
524 ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
525 drvinitfail_add_intr:
526 drvinitfail_hilevel:
527 drvinitfail_config_setup:
528 mutex_destroy(&state->is_mutex);
529
530 return (DDI_FAILURE);
531 }
532
533
534 /*
535 * ioat_drv_fini()
536 */
537 static void
538 ioat_drv_fini(ioat_state_t *state)
539 {
540 ioat_channel_fini(state);
541 ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
542 mutex_destroy(&state->is_mutex);
543 }
544
545
546 /*
547 * ioat_unregister_complete()
548 */
549 void
550 ioat_unregister_complete(void *device_private, int status)
551 {
552 ioat_state_t *state;
553
554
555 state = device_private;
556
557 if (status != DCOPY_SUCCESS) {
558 cmn_err(CE_WARN, "asynchronous detach aborted\n");
559 return;
560 }
561
562 cmn_err(CE_CONT, "detach completing\n");
563 ioat_detach_finish(state);
564 }
565
566
567 /*
568 * ioat_detach_finish()
569 */
570 void
571 ioat_detach_finish(ioat_state_t *state)
572 {
573 ioat_intr_disable(state);
574 ddi_remove_minor_node(state->is_dip, NULL);
575 ioat_drv_fini(state);
576 ioat_chip_fini(state);
577 (void) ddi_soft_state_free(ioat_statep, state->is_instance);
578 }
579
580
581 /*
582 * ioat_intr_enable()
583 */
584 static void
585 ioat_intr_enable(ioat_state_t *state)
586 {
587 uint32_t intr_status;
588
589
590 /* Clear any pending interrupts */
591 intr_status = ddi_get32(state->is_reg_handle,
592 (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
593 if (intr_status != 0) {
594 ddi_put32(state->is_reg_handle,
595 (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS],
596 intr_status);
597 }
598
599 /* Enable interrupts on the device */
600 ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
601 IOAT_INTRCTL_MASTER_EN);
602 }
603
604
605 /*
606 * ioat_intr_disable()
607 */
608 static void
609 ioat_intr_disable(ioat_state_t *state)
610 {
611 /*
612 * disable interrupts on the device. A read of the interrupt control
613 * register clears the enable bit.
614 */
615 (void) ddi_get8(state->is_reg_handle,
616 &state->is_genregs[IOAT_INTRCTL]);
617 }
618
619
620 /*
621 * ioat_isr()
622 */
623 static uint_t
624 ioat_isr(caddr_t parm)
625 {
626 uint32_t intr_status;
627 ioat_state_t *state;
628 uint8_t intrctrl;
629 uint32_t chan;
630 uint_t r;
631 int i;
632
633 state = (ioat_state_t *)parm;
634
635 intrctrl = ddi_get8(state->is_reg_handle,
636 &state->is_genregs[IOAT_INTRCTL]);
637 /* master interrupt enable should always be set */
638 ASSERT(intrctrl & IOAT_INTRCTL_MASTER_EN);
639
640 /* If the interrupt status bit isn't set, it's not ours */
641 if (!(intrctrl & IOAT_INTRCTL_INTR_STAT)) {
642 /* re-set master interrupt enable (since it clears on read) */
643 ddi_put8(state->is_reg_handle,
644 &state->is_genregs[IOAT_INTRCTL], intrctrl);
645 return (DDI_INTR_UNCLAIMED);
646 }
647
648 /* see which channels generated the interrupt */
649 intr_status = ddi_get32(state->is_reg_handle,
650 (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
651
652 /* call the intr handler for the channels */
653 r = DDI_INTR_UNCLAIMED;
654 chan = 1;
655 for (i = 0; i < state->is_num_channels; i++) {
656 if (intr_status & chan) {
657 ioat_channel_intr(&state->is_channel[i]);
658 r = DDI_INTR_CLAIMED;
659 }
660 chan = chan << 1;
661 }
662
663 /*
664 * if interrupt status bit was set, there should have been an
665 * attention status bit set too.
666 */
667 ASSERT(r == DDI_INTR_CLAIMED);
668
669 /* re-set master interrupt enable (since it clears on read) */
670 ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
671 intrctrl);
672
673 return (r);
674 }
675
676 static int
677 ioat_quiesce(dev_info_t *dip)
678 {
679 ioat_state_t *state;
680 int instance;
681
682 instance = ddi_get_instance(dip);
683 state = ddi_get_soft_state(ioat_statep, instance);
684 if (state == NULL) {
685 return (DDI_FAILURE);
686 }
687
688 ioat_intr_disable(state);
689 ioat_channel_quiesce(state);
690
691 return (DDI_SUCCESS);
692 }