1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * tavor.c
28 * Tavor (InfiniBand) HCA Driver attach/detach Routines
29 *
30 * Implements all the routines necessary for the attach, setup,
31 * initialization (and subsequent possible teardown and detach) of the
32 * Tavor InfiniBand HCA driver.
33 */
34
35 #include <sys/types.h>
36 #include <sys/file.h>
37 #include <sys/open.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/stat.h>
43 #include <sys/pci.h>
44 #include <sys/pci_cap.h>
45 #include <sys/bitmap.h>
46 #include <sys/policy.h>
47
48 #include <sys/ib/adapters/tavor/tavor.h>
49 #include <sys/pci.h>
50
51 /* Tavor HCA State Pointer */
52 void *tavor_statep;
53
54 /*
55 * The Tavor "userland resource database" is common to instances of the
56 * Tavor HCA driver. This structure "tavor_userland_rsrc_db" contains all
57 * the necessary information to maintain it.
58 */
59 tavor_umap_db_t tavor_userland_rsrc_db;
60
61 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t);
62 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t);
63 static int tavor_open(dev_t *, int, int, cred_t *);
64 static int tavor_close(dev_t, int, int, cred_t *);
65 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
66 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance);
67 static void tavor_drv_fini(tavor_state_t *state);
68 static void tavor_drv_fini2(tavor_state_t *state);
69 static int tavor_isr_init(tavor_state_t *state);
70 static void tavor_isr_fini(tavor_state_t *state);
71 static int tavor_hw_init(tavor_state_t *state);
72 static void tavor_hw_fini(tavor_state_t *state,
73 tavor_drv_cleanup_level_t cleanup);
74 static int tavor_soft_state_init(tavor_state_t *state);
75 static void tavor_soft_state_fini(tavor_state_t *state);
76 static int tavor_hca_port_init(tavor_state_t *state);
77 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init);
78 static void tavor_hca_config_setup(tavor_state_t *state,
79 tavor_hw_initqueryhca_t *inithca);
80 static int tavor_internal_uarpgs_init(tavor_state_t *state);
81 static void tavor_internal_uarpgs_fini(tavor_state_t *state);
82 static int tavor_special_qp_contexts_reserve(tavor_state_t *state);
83 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state);
84 static int tavor_sw_reset(tavor_state_t *state);
85 static int tavor_mcg_init(tavor_state_t *state);
86 static void tavor_mcg_fini(tavor_state_t *state);
87 static int tavor_fw_version_check(tavor_state_t *state);
88 static void tavor_device_info_report(tavor_state_t *state);
89 static void tavor_pci_capability_list(tavor_state_t *state,
90 ddi_acc_handle_t hdl);
91 static void tavor_pci_capability_vpd(tavor_state_t *state,
92 ddi_acc_handle_t hdl, uint_t offset);
93 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset,
94 uint32_t addr, uint32_t *data);
95 static void tavor_pci_capability_pcix(tavor_state_t *state,
96 ddi_acc_handle_t hdl, uint_t offset);
97 static int tavor_intr_or_msi_init(tavor_state_t *state);
98 static int tavor_add_intrs(tavor_state_t *state, int intr_type);
99 static int tavor_intr_or_msi_fini(tavor_state_t *state);
100
101 /* X86 fastreboot support */
102 static int tavor_intr_disable(tavor_state_t *);
103 static int tavor_quiesce(dev_info_t *);
104
105 /* Character/Block Operations */
106 static struct cb_ops tavor_cb_ops = {
107 tavor_open, /* open */
108 tavor_close, /* close */
109 nodev, /* strategy (block) */
110 nodev, /* print (block) */
111 nodev, /* dump (block) */
112 nodev, /* read */
113 nodev, /* write */
114 tavor_ioctl, /* ioctl */
115 tavor_devmap, /* devmap */
116 NULL, /* mmap */
117 nodev, /* segmap */
118 nochpoll, /* chpoll */
119 ddi_prop_op, /* prop_op */
120 NULL, /* streams */
121 D_NEW | D_MP |
122 D_64BIT | D_HOTPLUG |
123 D_DEVMAP, /* flags */
124 CB_REV /* rev */
125 };
126
127 /* Driver Operations */
128 static struct dev_ops tavor_ops = {
129 DEVO_REV, /* struct rev */
130 0, /* refcnt */
131 tavor_getinfo, /* getinfo */
132 nulldev, /* identify */
133 nulldev, /* probe */
134 tavor_attach, /* attach */
135 tavor_detach, /* detach */
136 nodev, /* reset */
137 &tavor_cb_ops, /* cb_ops */
138 NULL, /* bus_ops */
139 nodev, /* power */
140 tavor_quiesce, /* devo_quiesce */
141 };
142
143 /* Module Driver Info */
144 static struct modldrv tavor_modldrv = {
145 &mod_driverops,
146 "Tavor InfiniBand HCA Driver",
147 &tavor_ops
148 };
149
150 /* Module Linkage */
151 static struct modlinkage tavor_modlinkage = {
152 MODREV_1,
153 &tavor_modldrv,
154 NULL
155 };
156
157 /*
158 * This extern refers to the ibc_operations_t function vector that is defined
159 * in the tavor_ci.c file.
160 */
161 extern ibc_operations_t tavor_ibc_ops;
162
163 #ifndef NPROBE
164 extern int tnf_mod_load(void);
165 extern int tnf_mod_unload(struct modlinkage *mlp);
166 #endif
167
168
169 /*
170 * _init()
171 */
172 int
173 _init()
174 {
175 int status;
176
177 #ifndef NPROBE
178 (void) tnf_mod_load();
179 #endif
180 TAVOR_TNF_ENTER(tavor_init);
181
182 status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t),
183 (size_t)TAVOR_INITIAL_STATES);
184 if (status != 0) {
185 TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, "");
186 TAVOR_TNF_EXIT(tavor_init);
187 #ifndef NPROBE
188 (void) tnf_mod_unload(&tavor_modlinkage);
189 #endif
190 return (status);
191 }
192
193 status = ibc_init(&tavor_modlinkage);
194 if (status != 0) {
195 TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, "");
196 ddi_soft_state_fini(&tavor_statep);
197 TAVOR_TNF_EXIT(tavor_init);
198 #ifndef NPROBE
199 (void) tnf_mod_unload(&tavor_modlinkage);
200 #endif
201 return (status);
202 }
203 status = mod_install(&tavor_modlinkage);
204 if (status != 0) {
205 TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, "");
206 ibc_fini(&tavor_modlinkage);
207 ddi_soft_state_fini(&tavor_statep);
208 TAVOR_TNF_EXIT(tavor_init);
209 #ifndef NPROBE
210 (void) tnf_mod_unload(&tavor_modlinkage);
211 #endif
212 return (status);
213 }
214
215 /* Initialize the Tavor "userland resources database" */
216 tavor_umap_db_init();
217
218 TAVOR_TNF_EXIT(tavor_init);
219 return (status);
220 }
221
222
223 /*
224 * _info()
225 */
226 int
227 _info(struct modinfo *modinfop)
228 {
229 int status;
230
231 TAVOR_TNF_ENTER(tavor_info);
232 status = mod_info(&tavor_modlinkage, modinfop);
233 TAVOR_TNF_EXIT(tavor_info);
234 return (status);
235 }
236
237
238 /*
239 * _fini()
240 */
241 int
242 _fini()
243 {
244 int status;
245
246 TAVOR_TNF_ENTER(tavor_fini);
247
248 status = mod_remove(&tavor_modlinkage);
249 if (status != 0) {
250 TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, "");
251 TAVOR_TNF_EXIT(tavor_fini);
252 return (status);
253 }
254
255 /* Destroy the Tavor "userland resources database" */
256 tavor_umap_db_fini();
257
258 ibc_fini(&tavor_modlinkage);
259 ddi_soft_state_fini(&tavor_statep);
260 #ifndef NPROBE
261 (void) tnf_mod_unload(&tavor_modlinkage);
262 #endif
263 TAVOR_TNF_EXIT(tavor_fini);
264 return (status);
265 }
266
267
268 /*
269 * tavor_getinfo()
270 */
271 /* ARGSUSED */
272 static int
273 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
274 {
275 dev_t dev;
276 tavor_state_t *state;
277 minor_t instance;
278
279 TAVOR_TNF_ENTER(tavor_getinfo);
280
281 switch (cmd) {
282 case DDI_INFO_DEVT2DEVINFO:
283 dev = (dev_t)arg;
284 instance = TAVOR_DEV_INSTANCE(dev);
285 state = ddi_get_soft_state(tavor_statep, instance);
286 if (state == NULL) {
287 TNF_PROBE_0(tavor_getinfo_gss_fail,
288 TAVOR_TNF_ERROR, "");
289 TAVOR_TNF_EXIT(tavor_getinfo);
290 return (DDI_FAILURE);
291 }
292 *result = (void *)state->ts_dip;
293 return (DDI_SUCCESS);
294
295 case DDI_INFO_DEVT2INSTANCE:
296 dev = (dev_t)arg;
297 instance = TAVOR_DEV_INSTANCE(dev);
298 *result = (void *)(uintptr_t)instance;
299 return (DDI_SUCCESS);
300
301 default:
302 TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, "");
303 break;
304 }
305
306 TAVOR_TNF_EXIT(tavor_getinfo);
307 return (DDI_FAILURE);
308 }
309
310
311 /*
312 * tavor_open()
313 */
314 /* ARGSUSED */
315 static int
316 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp)
317 {
318 tavor_state_t *state;
319 tavor_rsrc_t *rsrcp;
320 tavor_umap_db_entry_t *umapdb, *umapdb2;
321 minor_t instance;
322 uint64_t key, value;
323 uint_t tr_indx;
324 dev_t dev;
325 int status;
326
327 TAVOR_TNF_ENTER(tavor_open);
328
329 instance = TAVOR_DEV_INSTANCE(*devp);
330 state = ddi_get_soft_state(tavor_statep, instance);
331 if (state == NULL) {
332 TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, "");
333 TAVOR_TNF_EXIT(tavor_open);
334 return (ENXIO);
335 }
336
337 /*
338 * Only allow driver to be opened for character access, and verify
339 * whether exclusive access is allowed.
340 */
341 if ((otyp != OTYP_CHR) || ((flag & FEXCL) &&
342 secpolicy_excl_open(credp) != 0)) {
343 TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, "");
344 TAVOR_TNF_EXIT(tavor_open);
345 return (EINVAL);
346 }
347
348 /*
349 * Search for the current process PID in the "userland resources
350 * database". If it is not found, then attempt to allocate a UAR
351 * page and add the ("key", "value") pair to the database.
352 * Note: As a last step we always return a devp appropriate for
353 * the open. Either we return a new minor number (based on the
354 * instance and the UAR page index) or we return the current minor
355 * number for the given client process.
356 *
357 * We also add an entry to the database to allow for lookup from
358 * "dev_t" to the current process PID. This is necessary because,
359 * under certain circumstance, the process PID that calls the Tavor
360 * close() entry point may not be the same as the one who called
361 * open(). Specifically, this can happen if a child process calls
362 * the Tavor's open() entry point, gets a UAR page, maps it out (using
363 * mmap()), and then exits without calling munmap(). Because mmap()
364 * adds a reference to the file descriptor, at the exit of the child
365 * process the file descriptor is "inherited" by the parent (and will
366 * be close()'d by the parent's PID only when it exits).
367 *
368 * Note: We use the tavor_umap_db_find_nolock() and
369 * tavor_umap_db_add_nolock() database access routines below (with
370 * an explicit mutex_enter of the database lock - "tdl_umapdb_lock")
371 * to ensure that the multiple accesses (in this case searching for,
372 * and then adding _two_ database entries) can be done atomically.
373 */
374 key = ddi_get_pid();
375 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
376 status = tavor_umap_db_find_nolock(instance, key,
377 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
378 if (status != DDI_SUCCESS) {
379 /*
380 * If we are in 'maintenance mode', we cannot alloc a UAR page.
381 * But we still need some rsrcp value, and a mostly unique
382 * tr_indx value. So we set rsrcp to NULL for maintenance
383 * mode, and use a rolling count for tr_indx. The field
384 * 'ts_open_tr_indx' is used only in this maintenance mode
385 * condition.
386 *
387 * Otherwise, if we are in operational mode then we allocate
388 * the UAR page as normal, and use the rsrcp value and tr_indx
389 * value from that allocation.
390 */
391 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
392 rsrcp = NULL;
393 tr_indx = state->ts_open_tr_indx++;
394 } else {
395 /* Allocate a new UAR page for this process */
396 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1,
397 TAVOR_NOSLEEP, &rsrcp);
398 if (status != DDI_SUCCESS) {
399 mutex_exit(
400 &tavor_userland_rsrc_db.tdl_umapdb_lock);
401 TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail,
402 TAVOR_TNF_ERROR, "");
403 TAVOR_TNF_EXIT(tavor_open);
404 return (EAGAIN);
405 }
406
407 tr_indx = rsrcp->tr_indx;
408 }
409
410 /*
411 * Allocate an entry to track the UAR page resource in the
412 * "userland resources database".
413 */
414 umapdb = tavor_umap_db_alloc(instance, key,
415 MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp);
416 if (umapdb == NULL) {
417 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
418 /* If in "maintenance mode", don't free the rsrc */
419 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
420 tavor_rsrc_free(state, &rsrcp);
421 }
422 TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
423 TAVOR_TNF_ERROR, "");
424 TAVOR_TNF_EXIT(tavor_open);
425 return (EAGAIN);
426 }
427
428 /*
429 * Create a new device number. Minor number is a function of
430 * the UAR page index (15 bits) and the device instance number
431 * (3 bits).
432 */
433 dev = makedevice(getmajor(*devp), (tr_indx <<
434 TAVOR_MINORNUM_SHIFT) | instance);
435
436 /*
437 * Allocate another entry in the "userland resources database"
438 * to track the association of the device number (above) to
439 * the current process ID (in "key").
440 */
441 umapdb2 = tavor_umap_db_alloc(instance, dev,
442 MLNX_UMAP_PID_RSRC, (uint64_t)key);
443 if (umapdb2 == NULL) {
444 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
445 tavor_umap_db_free(umapdb);
446 /* If in "maintenance mode", don't free the rsrc */
447 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
448 tavor_rsrc_free(state, &rsrcp);
449 }
450 TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
451 TAVOR_TNF_ERROR, "");
452 TAVOR_TNF_EXIT(tavor_open);
453 return (EAGAIN);
454 }
455
456 /* Add the entries to the database */
457 tavor_umap_db_add_nolock(umapdb);
458 tavor_umap_db_add_nolock(umapdb2);
459
460 } else {
461 /*
462 * Return the same device number as on the original open()
463 * call. This was calculated as a function of the UAR page
464 * index (top 16 bits) and the device instance number
465 */
466 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
467 dev = makedevice(getmajor(*devp), (rsrcp->tr_indx <<
468 TAVOR_MINORNUM_SHIFT) | instance);
469 }
470 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
471
472 *devp = dev;
473
474 TAVOR_TNF_EXIT(tavor_open);
475 return (0);
476 }
477
478
479 /*
480 * tavor_close()
481 */
482 /* ARGSUSED */
483 static int
484 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp)
485 {
486 tavor_state_t *state;
487 tavor_rsrc_t *rsrcp;
488 tavor_umap_db_entry_t *umapdb;
489 tavor_umap_db_priv_t *priv;
490 minor_t instance;
491 uint64_t key, value;
492 int status;
493
494 TAVOR_TNF_ENTER(tavor_close);
495
496 instance = TAVOR_DEV_INSTANCE(dev);
497 state = ddi_get_soft_state(tavor_statep, instance);
498 if (state == NULL) {
499 TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, "");
500 TAVOR_TNF_EXIT(tavor_close);
501 return (ENXIO);
502 }
503
504 /*
505 * Search for "dev_t" in the "userland resources database". As
506 * explained above in tavor_open(), we can't depend on using the
507 * current process ID here to do the lookup because the process
508 * that ultimately closes may not be the same one who opened
509 * (because of inheritance).
510 * So we lookup the "dev_t" (which points to the PID of the process
511 * that opened), and we remove the entry from the database (and free
512 * it up). Then we do another query based on the PID value. And when
513 * we find that database entry, we free it up too and then free the
514 * Tavor UAR page resource.
515 *
516 * Note: We use the tavor_umap_db_find_nolock() database access
517 * routine below (with an explicit mutex_enter of the database lock)
518 * to ensure that the multiple accesses (which attempt to remove the
519 * two database entries) can be done atomically.
520 *
521 * This works the same in both maintenance mode and HCA mode, except
522 * for the call to tavor_rsrc_free(). In the case of maintenance mode,
523 * this call is not needed, as it was not allocated in tavor_open()
524 * above.
525 */
526 key = dev;
527 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
528 status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC,
529 &value, TAVOR_UMAP_DB_REMOVE, &umapdb);
530 if (status == DDI_SUCCESS) {
531 /*
532 * If the "tdb_priv" field is non-NULL, it indicates that
533 * some "on close" handling is still necessary. Call
534 * tavor_umap_db_handle_onclose_cb() to do the handling (i.e.
535 * to invoke all the registered callbacks). Then free up
536 * the resources associated with "tdb_priv" and continue
537 * closing.
538 */
539 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv;
540 if (priv != NULL) {
541 tavor_umap_db_handle_onclose_cb(priv);
542 kmem_free(priv, sizeof (tavor_umap_db_priv_t));
543 umapdb->tdbe_common.tdb_priv = (void *)NULL;
544 }
545
546 tavor_umap_db_free(umapdb);
547
548 /*
549 * Now do another lookup using PID as the key (copy it from
550 * "value"). When this lookup is complete, the "value" field
551 * will contain the tavor_rsrc_t pointer for the UAR page
552 * resource.
553 */
554 key = value;
555 status = tavor_umap_db_find_nolock(instance, key,
556 MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
557 &umapdb);
558 if (status == DDI_SUCCESS) {
559 tavor_umap_db_free(umapdb);
560 /* If in "maintenance mode", don't free the rsrc */
561 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
562 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
563 tavor_rsrc_free(state, &rsrcp);
564 }
565 }
566 }
567 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
568
569 TAVOR_TNF_EXIT(tavor_close);
570 return (0);
571 }
572
573
574 /*
575 * tavor_attach()
576 * Context: Only called from attach() path context
577 */
578 static int
579 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
580 {
581 tavor_state_t *state;
582 ibc_clnt_hdl_t tmp_ibtfpriv;
583 ibc_status_t ibc_status;
584 int instance;
585 int status;
586
587 TAVOR_TNF_ENTER(tavor_attach);
588
589 #ifdef __lock_lint
590 (void) tavor_quiesce(dip);
591 #endif
592
593 switch (cmd) {
594 case DDI_ATTACH:
595 instance = ddi_get_instance(dip);
596 status = ddi_soft_state_zalloc(tavor_statep, instance);
597 if (status != DDI_SUCCESS) {
598 TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, "");
599 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
600 "attach_ssz_fail", instance);
601 goto fail_attach_nomsg;
602
603 }
604 state = ddi_get_soft_state(tavor_statep, instance);
605 if (state == NULL) {
606 ddi_soft_state_free(tavor_statep, instance);
607 TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, "");
608 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
609 "attach_gss_fail", instance);
610 goto fail_attach_nomsg;
611 }
612
613 /* clear the attach error buffer */
614 TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf);
615
616 /*
617 * Initialize Tavor driver and hardware.
618 *
619 * Note: If this initialization fails we may still wish to
620 * create a device node and remain operational so that Tavor
621 * firmware can be updated/flashed (i.e. "maintenance mode").
622 * If this is the case, then "ts_operational_mode" will be
623 * equal to TAVOR_MAINTENANCE_MODE. We will not attempt to
624 * attach to the IBTF or register with the IBMF (i.e. no
625 * InfiniBand interfaces will be enabled).
626 */
627 status = tavor_drv_init(state, dip, instance);
628 if ((status != DDI_SUCCESS) &&
629 (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
630 TNF_PROBE_0(tavor_attach_drvinit_fail,
631 TAVOR_TNF_ERROR, "");
632 goto fail_attach;
633 }
634
635 /* Create the minor node for device */
636 status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance,
637 DDI_PSEUDO, 0);
638 if (status != DDI_SUCCESS) {
639 tavor_drv_fini(state);
640 TAVOR_ATTACH_MSG(state->ts_attach_buf,
641 "attach_create_mn_fail");
642 TNF_PROBE_0(tavor_attach_create_mn_fail,
643 TAVOR_TNF_ERROR, "");
644 goto fail_attach;
645 }
646
647 /*
648 * If we are in "maintenance mode", then we don't want to
649 * register with the IBTF. All InfiniBand interfaces are
650 * uninitialized, and the device is only capable of handling
651 * requests to update/flash firmware (or test/debug requests).
652 */
653 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
654
655 /* Attach to InfiniBand Transport Framework (IBTF) */
656 ibc_status = ibc_attach(&tmp_ibtfpriv,
657 &state->ts_ibtfinfo);
658 if (ibc_status != IBC_SUCCESS) {
659 ddi_remove_minor_node(dip, "devctl");
660 tavor_drv_fini(state);
661 TNF_PROBE_0(tavor_attach_ibcattach_fail,
662 TAVOR_TNF_ERROR, "");
663 TAVOR_ATTACH_MSG(state->ts_attach_buf,
664 "attach_ibcattach_fail");
665 goto fail_attach;
666 }
667
668 /*
669 * Now that we've successfully attached to the IBTF,
670 * we enable all appropriate asynch and CQ events to
671 * be forwarded to the IBTF.
672 */
673 TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv);
674
675 ibc_post_attach(state->ts_ibtfpriv);
676
677 /* Register agents with IB Mgmt Framework (IBMF) */
678 status = tavor_agent_handlers_init(state);
679 if (status != DDI_SUCCESS) {
680 (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH);
681 TAVOR_QUIESCE_IBTF_CALLB(state);
682 if (state->ts_in_evcallb != 0) {
683 TAVOR_WARNING(state, "unable to "
684 "quiesce Tavor IBTF callbacks");
685 }
686 ibc_detach(tmp_ibtfpriv);
687 ddi_remove_minor_node(dip, "devctl");
688 tavor_drv_fini(state);
689 TNF_PROBE_0(tavor_attach_agentinit_fail,
690 TAVOR_TNF_ERROR, "");
691 TAVOR_ATTACH_MSG(state->ts_attach_buf,
692 "attach_agentinit_fail");
693 goto fail_attach;
694 }
695 }
696
697 /* Report that driver was loaded */
698 ddi_report_dev(dip);
699
700 /* Send device information to log file */
701 tavor_device_info_report(state);
702
703 /* Report attach in maintenance mode, if appropriate */
704 if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
705 cmn_err(CE_NOTE, "tavor%d: driver attached "
706 "(for maintenance mode only)", state->ts_instance);
707 }
708
709 TAVOR_TNF_EXIT(tavor_attach);
710 return (DDI_SUCCESS);
711
712 case DDI_RESUME:
713 /* Add code here for DDI_RESUME XXX */
714 TAVOR_TNF_EXIT(tavor_attach);
715 return (DDI_FAILURE);
716
717 default:
718 TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, "");
719 break;
720 }
721
722 fail_attach:
723 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance,
724 state->ts_attach_buf);
725 tavor_drv_fini2(state);
726 ddi_soft_state_free(tavor_statep, instance);
727 fail_attach_nomsg:
728 TAVOR_TNF_EXIT(tavor_attach);
729 return (DDI_FAILURE);
730 }
731
732
733 /*
734 * tavor_detach()
735 * Context: Only called from detach() path context
736 */
737 static int
738 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
739 {
740 tavor_state_t *state;
741 ibc_clnt_hdl_t tmp_ibtfpriv;
742 ibc_status_t ibc_status;
743 int instance, status;
744
745 TAVOR_TNF_ENTER(tavor_detach);
746
747 instance = ddi_get_instance(dip);
748 state = ddi_get_soft_state(tavor_statep, instance);
749 if (state == NULL) {
750 TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, "");
751 TAVOR_TNF_EXIT(tavor_detach);
752 return (DDI_FAILURE);
753 }
754
755 switch (cmd) {
756 case DDI_DETACH:
757 /*
758 * If we are in "maintenance mode", then we do not want to
759 * do teardown for any of the InfiniBand interfaces.
760 * Specifically, this means not detaching from IBTF (we never
761 * attached to begin with) and not deregistering from IBMF.
762 */
763 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
764 /* Unregister agents from IB Mgmt Framework (IBMF) */
765 status = tavor_agent_handlers_fini(state);
766 if (status != DDI_SUCCESS) {
767 TNF_PROBE_0(tavor_detach_agentfini_fail,
768 TAVOR_TNF_ERROR, "");
769 TAVOR_TNF_EXIT(tavor_detach);
770 return (DDI_FAILURE);
771 }
772
773 /*
774 * Attempt the "pre-detach" from InfiniBand Transport
775 * Framework (IBTF). At this point the IBTF is still
776 * capable of handling incoming asynch and completion
777 * events. This "pre-detach" is primarily a mechanism
778 * to notify the appropriate IBTF clients that the
779 * HCA is being removed/offlined.
780 */
781 ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd);
782 if (ibc_status != IBC_SUCCESS) {
783 status = tavor_agent_handlers_init(state);
784 if (status != DDI_SUCCESS) {
785 TAVOR_WARNING(state, "failed to "
786 "restart Tavor agents");
787 }
788 TNF_PROBE_0(tavor_detach_ibcpredetach_fail,
789 TAVOR_TNF_ERROR, "");
790 TAVOR_TNF_EXIT(tavor_detach);
791 return (DDI_FAILURE);
792 }
793
794 /*
795 * Before we can fully detach from the IBTF we need to
796 * ensure that we have handled all outstanding event
797 * callbacks. This is accomplished by quiescing the
798 * event callback mechanism. Note: if we are unable
799 * to successfully quiesce the callbacks, then this is
800 * an indication that something has probably gone
801 * seriously wrong. We print out a warning, but
802 * continue.
803 */
804 tmp_ibtfpriv = state->ts_ibtfpriv;
805 TAVOR_QUIESCE_IBTF_CALLB(state);
806 if (state->ts_in_evcallb != 0) {
807 TAVOR_WARNING(state, "unable to quiesce Tavor "
808 "IBTF callbacks");
809 }
810
811 /* Complete the detach from the IBTF */
812 ibc_detach(tmp_ibtfpriv);
813 }
814
815 /* Remove the minor node for device */
816 ddi_remove_minor_node(dip, "devctl");
817
818 /*
819 * Only call tavor_drv_fini() if we are in Tavor HCA mode.
820 * (Because if we are in "maintenance mode", then we never
821 * successfully finished init.) Only report successful
822 * detach for normal HCA mode.
823 */
824 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
825 /* Cleanup driver resources and shutdown hardware */
826 tavor_drv_fini(state);
827 cmn_err(CE_CONT, "Tavor driver successfully "
828 "detached\n");
829 }
830
831 tavor_drv_fini2(state);
832 ddi_soft_state_free(tavor_statep, instance);
833
834 TAVOR_TNF_EXIT(tavor_detach);
835 return (DDI_SUCCESS);
836
837 case DDI_SUSPEND:
838 /* Add code here for DDI_SUSPEND XXX */
839 TAVOR_TNF_EXIT(tavor_detach);
840 return (DDI_FAILURE);
841
842 default:
843 TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, "");
844 break;
845 }
846
847 TAVOR_TNF_EXIT(tavor_detach);
848 return (DDI_FAILURE);
849 }
850
851
852 /*
853 * tavor_drv_init()
854 * Context: Only called from attach() path context
855 */
856 static int
857 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance)
858 {
859 int status;
860
861 TAVOR_TNF_ENTER(tavor_drv_init);
862
863 /* Save away devinfo and instance */
864 state->ts_dip = dip;
865 state->ts_instance = instance;
866
867 /*
868 * Check and set the operational mode of the device. If the driver is
869 * bound to the Tavor device in "maintenance mode", then this generally
870 * means that either the device has been specifically jumpered to
871 * start in this mode or the firmware boot process has failed to
872 * successfully load either the primary or the secondary firmware
873 * image.
874 */
875 if (TAVOR_IS_HCA_MODE(state->ts_dip)) {
876 state->ts_operational_mode = TAVOR_HCA_MODE;
877
878 } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) {
879 state->ts_operational_mode = TAVOR_COMPAT_MODE;
880
881 } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) {
882 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
883 return (DDI_FAILURE);
884
885 } else {
886 state->ts_operational_mode = 0; /* invalid operational mode */
887 TAVOR_WARNING(state, "unexpected device type detected");
888 TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail,
889 TAVOR_TNF_ERROR, "");
890 TAVOR_TNF_EXIT(tavor_hw_init);
891 return (DDI_FAILURE);
892 }
893
894 /*
895 * Initialize the Tavor hardware.
896 * Note: If this routine returns an error, it is often an reasonably
897 * good indication that something Tavor firmware-related has caused
898 * the failure. In order to give the user an opportunity (if desired)
899 * to update or reflash the Tavor firmware image, we set
900 * "ts_operational_mode" flag (described above) to indicate that we
901 * wish to enter maintenance mode.
902 */
903 status = tavor_hw_init(state);
904 if (status != DDI_SUCCESS) {
905 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
906 cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance,
907 state->ts_attach_buf);
908 TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, "");
909 TAVOR_TNF_EXIT(tavor_drv_init);
910 return (DDI_FAILURE);
911 }
912
913 /* Setup Tavor interrupt handler */
914 status = tavor_isr_init(state);
915 if (status != DDI_SUCCESS) {
916 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
917 TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, "");
918 TAVOR_TNF_EXIT(tavor_drv_init);
919 return (DDI_FAILURE);
920 }
921
922 /* Initialize Tavor softstate */
923 status = tavor_soft_state_init(state);
924 if (status != DDI_SUCCESS) {
925 tavor_isr_fini(state);
926 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
927 TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, "");
928 TAVOR_TNF_EXIT(tavor_drv_init);
929 return (DDI_FAILURE);
930 }
931
932 TAVOR_TNF_EXIT(tavor_drv_init);
933 return (DDI_SUCCESS);
934 }
935
936
937 /*
938 * tavor_drv_fini()
939 * Context: Only called from attach() and/or detach() path contexts
940 */
941 static void
942 tavor_drv_fini(tavor_state_t *state)
943 {
944 TAVOR_TNF_ENTER(tavor_drv_fini);
945
946 /* Cleanup Tavor softstate */
947 tavor_soft_state_fini(state);
948
949 /* Teardown Tavor interrupts */
950 tavor_isr_fini(state);
951
952 /* Cleanup Tavor resources and shutdown hardware */
953 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
954
955 TAVOR_TNF_EXIT(tavor_drv_fini);
956 }
957
958 /*
959 * tavor_drv_fini2()
960 * Context: Only called from attach() and/or detach() path contexts
961 */
962 static void
963 tavor_drv_fini2(tavor_state_t *state)
964 {
965 TAVOR_TNF_ENTER(tavor_drv_fini2);
966
967 /* TAVOR_DRV_CLEANUP_LEVEL1 */
968 if (state->ts_reg_cmdhdl) {
969 ddi_regs_map_free(&state->ts_reg_cmdhdl);
970 state->ts_reg_cmdhdl = NULL;
971 }
972
973 /* TAVOR_DRV_CLEANUP_LEVEL0 */
974 if (state->ts_pci_cfghdl) {
975 pci_config_teardown(&state->ts_pci_cfghdl);
976 state->ts_pci_cfghdl = NULL;
977 }
978
979 TAVOR_TNF_EXIT(tavor_drv_fini2);
980 }
981
982 /*
983 * tavor_isr_init()
984 * Context: Only called from attach() path context
985 */
986 static int
987 tavor_isr_init(tavor_state_t *state)
988 {
989 int status;
990
991 TAVOR_TNF_ENTER(tavor_isr_init);
992
993 /*
994 * Add a handler for the interrupt or MSI
995 */
996 status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr,
997 (caddr_t)state, NULL);
998 if (status != DDI_SUCCESS) {
999 TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, "");
1000 TAVOR_TNF_EXIT(tavor_isr_init);
1001 return (DDI_FAILURE);
1002 }
1003
1004 /*
1005 * Enable the software interrupt. Note: Even though we are only
1006 * using one (1) interrupt/MSI, depending on the value returned in
1007 * the capability flag, we have to call either ddi_intr_block_enable()
1008 * or ddi_intr_enable().
1009 */
1010 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1011 status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1);
1012 if (status != DDI_SUCCESS) {
1013 TNF_PROBE_0(tavor_isr_init_blockenable_fail,
1014 TAVOR_TNF_ERROR, "");
1015 TAVOR_TNF_EXIT(tavor_isr_init);
1016 return (DDI_FAILURE);
1017 }
1018 } else {
1019 status = ddi_intr_enable(state->ts_intrmsi_hdl);
1020 if (status != DDI_SUCCESS) {
1021 TNF_PROBE_0(tavor_isr_init_intrenable_fail,
1022 TAVOR_TNF_ERROR, "");
1023 TAVOR_TNF_EXIT(tavor_isr_init);
1024 return (DDI_FAILURE);
1025 }
1026 }
1027
1028 /*
1029 * Now that the ISR has been setup, arm all the EQs for event
1030 * generation.
1031 */
1032 tavor_eq_arm_all(state);
1033
1034 TAVOR_TNF_EXIT(tavor_isr_init);
1035 return (DDI_SUCCESS);
1036 }
1037
1038
1039 /*
1040 * tavor_isr_fini()
1041 * Context: Only called from attach() and/or detach() path contexts
1042 */
1043 static void
1044 tavor_isr_fini(tavor_state_t *state)
1045 {
1046 TAVOR_TNF_ENTER(tavor_isr_fini);
1047
1048 /* Disable the software interrupt */
1049 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1050 (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1);
1051 } else {
1052 (void) ddi_intr_disable(state->ts_intrmsi_hdl);
1053 }
1054
1055 /*
1056 * Remove the software handler for the interrupt or MSI
1057 */
1058 (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl);
1059
1060 TAVOR_TNF_EXIT(tavor_isr_fini);
1061 }
1062
1063
1064 /*
1065 * tavor_fix_error_buf()
1066 * Context: Only called from attach().
1067 *
1068 * The error_buf_addr returned from QUERY_FW is a PCI address.
1069 * We need to convert it to an offset from the base address,
1070 * which is stored in the assigned-addresses property.
1071 */
1072 static int
1073 tavor_fix_error_buf(tavor_state_t *state)
1074 {
1075 int assigned_addr_len;
1076 pci_regspec_t *assigned_addr;
1077
1078 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip,
1079 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr,
1080 (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS)
1081 return (DDI_FAILURE);
1082
1083 state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low +
1084 ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32);
1085 ddi_prop_free(assigned_addr);
1086 return (DDI_SUCCESS);
1087 }
1088
1089 /*
1090 * tavor_hw_init()
1091 * Context: Only called from attach() path context
1092 */
1093 static int
1094 tavor_hw_init(tavor_state_t *state)
1095 {
1096 tavor_drv_cleanup_level_t cleanup;
1097 sm_nodeinfo_t nodeinfo;
1098 uint64_t errorcode;
1099 off_t ddr_size;
1100 int status;
1101 int retries;
1102
1103 TAVOR_TNF_ENTER(tavor_hw_init);
1104
1105 /* This is where driver initialization begins */
1106 cleanup = TAVOR_DRV_CLEANUP_LEVEL0;
1107
1108 /* Setup device access attributes */
1109 state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
1110 state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
1111 state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1112
1113 /* Setup for PCI config read/write of HCA device */
1114 status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl);
1115 if (status != DDI_SUCCESS) {
1116 tavor_hw_fini(state, cleanup);
1117 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1118 "hw_init_PCI_config_space_regmap_fail");
1119 /* This case is not the degraded one */
1120 return (DDI_FAILURE);
1121 }
1122
1123 /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */
1124 status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR,
1125 &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr,
1126 &state->ts_reg_cmdhdl);
1127 if (status != DDI_SUCCESS) {
1128 tavor_hw_fini(state, cleanup);
1129 TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, "");
1130 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1131 "hw_init_CMD_ddirms_fail");
1132 TAVOR_TNF_EXIT(tavor_hw_init);
1133 return (DDI_FAILURE);
1134 }
1135 cleanup = TAVOR_DRV_CLEANUP_LEVEL1;
1136
1137 status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR,
1138 &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr,
1139 &state->ts_reg_uarhdl);
1140 if (status != DDI_SUCCESS) {
1141 tavor_hw_fini(state, cleanup);
1142 TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, "");
1143 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1144 "hw_init_UAR_ddirms_fail");
1145 TAVOR_TNF_EXIT(tavor_hw_init);
1146 return (DDI_FAILURE);
1147 }
1148 cleanup = TAVOR_DRV_CLEANUP_LEVEL2;
1149
1150 status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size);
1151 if (status != DDI_SUCCESS) {
1152 cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed "
1153 "(check HCA-attached DIMM memory?)\n");
1154 tavor_hw_fini(state, cleanup);
1155 TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail,
1156 TAVOR_TNF_ERROR, "");
1157 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1158 "hw_init_DDR_ddi_regsize_fail");
1159 TAVOR_TNF_EXIT(tavor_hw_init);
1160 return (DDI_FAILURE);
1161 }
1162
1163 #if !defined(_ELF64) && !defined(__sparc)
1164 /*
1165 * For 32 bit x86/x64 kernels, where there is limited kernel virtual
1166 * memory available, define a minimal memory footprint. This is
1167 * specified in order to not take up too much resources, thus starving
1168 * out others. Only specified if the HCA DIMM is equal to or greater
1169 * than 256MB.
1170 *
1171 * Note: x86/x64 install and safemode boot are both 32bit.
1172 */
1173 ddr_size = TAVOR_DDR_SIZE_MIN;
1174 #endif /* !(_ELF64) && !(__sparc) */
1175
1176 state->ts_cfg_profile_setting = ddr_size;
1177
1178 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1179 &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr,
1180 &state->ts_reg_ddrhdl);
1181
1182 /*
1183 * On 32-bit platform testing (primarily x86), it was seen that the
1184 * ddi_regs_map_setup() call would fail because there wasn't enough
1185 * kernel virtual address space available to map in the entire 256MB
1186 * DDR. So we add this check in here, so that if the 256 (or other
1187 * larger value of DDR) map in fails, that we fallback to try the lower
1188 * size of 128MB.
1189 *
1190 * Note: If we only have 128MB of DDR in the system in the first place,
1191 * we don't try another ddi_regs_map_setup(), and just skip over this
1192 * check and return failures.
1193 */
1194 if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) {
1195 /* Try falling back to 128MB DDR mapping */
1196 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1197 &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128,
1198 &state->ts_reg_accattr, &state->ts_reg_ddrhdl);
1199
1200 /*
1201 * 128MB DDR mapping worked.
1202 * Set the updated config profile setting here.
1203 */
1204 if (status == DDI_SUCCESS) {
1205 TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success,
1206 TAVOR_TNF_TRACE, "");
1207 state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128;
1208 }
1209 }
1210
1211 if (status != DDI_SUCCESS) {
1212 if (status == DDI_ME_RNUMBER_RANGE) {
1213 cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed "
1214 "(check HCA-attached DIMM memory?)\n");
1215 }
1216 tavor_hw_fini(state, cleanup);
1217 TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, "");
1218 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1219 "hw_init_DDR_ddirms_fail");
1220 TAVOR_TNF_EXIT(tavor_hw_init);
1221 return (DDI_FAILURE);
1222 }
1223 cleanup = TAVOR_DRV_CLEANUP_LEVEL3;
1224
1225 /* Setup Tavor Host Command Register (HCR) */
1226 state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *)
1227 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET);
1228
1229 /* Setup Tavor Event Cause Register (ecr and clr_ecr) */
1230 state->ts_cmd_regs.ecr = (uint64_t *)
1231 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET);
1232 state->ts_cmd_regs.clr_ecr = (uint64_t *)
1233 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET);
1234
1235 /* Setup Tavor Software Reset register (sw_reset) */
1236 state->ts_cmd_regs.sw_reset = (uint32_t *)
1237 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET);
1238
1239 /* Setup Tavor Clear Interrupt register (clr_int) */
1240 state->ts_cmd_regs.clr_int = (uint64_t *)
1241 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET);
1242
1243 /* Initialize the Phase1 Tavor configuration profile */
1244 status = tavor_cfg_profile_init_phase1(state);
1245 if (status != DDI_SUCCESS) {
1246 tavor_hw_fini(state, cleanup);
1247 TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, "");
1248 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail");
1249 TAVOR_TNF_EXIT(tavor_hw_init);
1250 return (DDI_FAILURE);
1251 }
1252 cleanup = TAVOR_DRV_CLEANUP_LEVEL4;
1253
1254 /* Do a software reset of the Tavor HW to ensure proper state */
1255 status = tavor_sw_reset(state);
1256 if (status != TAVOR_CMD_SUCCESS) {
1257 tavor_hw_fini(state, cleanup);
1258 TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, "");
1259 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail");
1260 TAVOR_TNF_EXIT(tavor_hw_init);
1261 return (DDI_FAILURE);
1262 }
1263
1264 /* Post the SYS_EN command to start the hardware */
1265 status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL,
1266 &errorcode, TAVOR_CMD_NOSLEEP_SPIN);
1267 if (status != TAVOR_CMD_SUCCESS) {
1268 if ((status == TAVOR_CMD_BAD_NVMEM) ||
1269 (status == TAVOR_CMD_DDR_MEM_ERR)) {
1270 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1271 "0x%" PRIx64 " (invalid firmware image?)\n",
1272 status, errorcode);
1273 } else {
1274 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1275 "0x%" PRIx64 "\n", status, errorcode);
1276 }
1277 tavor_hw_fini(state, cleanup);
1278 TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail,
1279 TAVOR_TNF_ERROR, "");
1280 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1281 "hw_init_sys_en_cmd_fail");
1282 TAVOR_TNF_EXIT(tavor_hw_init);
1283 return (DDI_FAILURE);
1284 }
1285 cleanup = TAVOR_DRV_CLEANUP_LEVEL5;
1286
1287 /* First phase of init for Tavor configuration/resources */
1288 status = tavor_rsrc_init_phase1(state);
1289 if (status != DDI_SUCCESS) {
1290 tavor_hw_fini(state, cleanup);
1291 TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, "");
1292 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1293 "hw_init_rsrcinit1_fail");
1294 TAVOR_TNF_EXIT(tavor_hw_init);
1295 return (DDI_FAILURE);
1296 }
1297 cleanup = TAVOR_DRV_CLEANUP_LEVEL6;
1298
1299 /* Query the DDR properties (e.g. total DDR size) */
1300 status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0,
1301 &state->ts_ddr, sizeof (tavor_hw_queryddr_t),
1302 TAVOR_CMD_NOSLEEP_SPIN);
1303 if (status != TAVOR_CMD_SUCCESS) {
1304 cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n",
1305 status);
1306 tavor_hw_fini(state, cleanup);
1307 TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail,
1308 TAVOR_TNF_ERROR, "");
1309 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1310 "hw_init_query_ddr_cmd_fail");
1311 TAVOR_TNF_EXIT(tavor_hw_init);
1312 return (DDI_FAILURE);
1313 }
1314
1315 /* Figure out how big the firmware image (in DDR) is */
1316 status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw,
1317 sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN);
1318 if (status != TAVOR_CMD_SUCCESS) {
1319 cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n",
1320 status);
1321 tavor_hw_fini(state, cleanup);
1322 TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail,
1323 TAVOR_TNF_ERROR, "");
1324 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1325 "hw_init_query_fw_cmd_fail");
1326 TAVOR_TNF_EXIT(tavor_hw_init);
1327 return (DDI_FAILURE);
1328 }
1329
1330 if (tavor_fix_error_buf(state) != DDI_SUCCESS) {
1331 tavor_hw_fini(state, cleanup);
1332 TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail,
1333 TAVOR_TNF_ERROR, "");
1334 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1335 "hw_init_fixerrorbuf_fail");
1336 TAVOR_TNF_EXIT(tavor_hw_init);
1337 return (DDI_FAILURE);
1338 }
1339
1340 /* Validate that the FW version is appropriate */
1341 status = tavor_fw_version_check(state);
1342 if (status != DDI_SUCCESS) {
1343 if (state->ts_operational_mode == TAVOR_HCA_MODE) {
1344 cmn_err(CE_CONT, "Unsupported Tavor FW version: "
1345 "expected: %04d.%04d.%04d, "
1346 "actual: %04d.%04d.%04d\n",
1347 TAVOR_FW_VER_MAJOR,
1348 TAVOR_FW_VER_MINOR,
1349 TAVOR_FW_VER_SUBMINOR,
1350 state->ts_fw.fw_rev_major,
1351 state->ts_fw.fw_rev_minor,
1352 state->ts_fw.fw_rev_subminor);
1353 } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) {
1354 cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: "
1355 "expected: %04d.%04d.%04d, "
1356 "actual: %04d.%04d.%04d\n",
1357 TAVOR_COMPAT_FW_VER_MAJOR,
1358 TAVOR_COMPAT_FW_VER_MINOR,
1359 TAVOR_COMPAT_FW_VER_SUBMINOR,
1360 state->ts_fw.fw_rev_major,
1361 state->ts_fw.fw_rev_minor,
1362 state->ts_fw.fw_rev_subminor);
1363 } else {
1364 cmn_err(CE_CONT, "Unsupported FW version: "
1365 "%04d.%04d.%04d\n",
1366 state->ts_fw.fw_rev_major,
1367 state->ts_fw.fw_rev_minor,
1368 state->ts_fw.fw_rev_subminor);
1369 }
1370 tavor_hw_fini(state, cleanup);
1371 TNF_PROBE_0(tavor_hw_init_checkfwver_fail,
1372 TAVOR_TNF_ERROR, "");
1373 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1374 "hw_init_checkfwver_fail");
1375 TAVOR_TNF_EXIT(tavor_hw_init);
1376 return (DDI_FAILURE);
1377 }
1378
1379 drv_usecwait(10);
1380 retries = 1000; /* retry up to 1 second before giving up */
1381 retry:
1382 /* Call MOD_STAT_CFG to setup SRQ support (or disable) */
1383 status = tavor_mod_stat_cfg_cmd_post(state);
1384 if (status != DDI_SUCCESS) {
1385 if (retries > 0) {
1386 drv_usecwait(1000);
1387 retries--;
1388 goto retry;
1389 }
1390 cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n",
1391 status);
1392 tavor_hw_fini(state, cleanup);
1393 TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail,
1394 TAVOR_TNF_ERROR, "");
1395 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1396 "hw_init_mod_stat_cfg_cmd_fail");
1397 TAVOR_TNF_EXIT(tavor_hw_init);
1398 return (DDI_FAILURE);
1399 }
1400
1401 /* Figure out Tavor device limits */
1402 status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0,
1403 &state->ts_devlim, sizeof (tavor_hw_querydevlim_t),
1404 TAVOR_CMD_NOSLEEP_SPIN);
1405 if (status != TAVOR_CMD_SUCCESS) {
1406 cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n",
1407 status);
1408 tavor_hw_fini(state, cleanup);
1409 TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail,
1410 TAVOR_TNF_ERROR, "");
1411 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1412 "hw_init_query_devlim_cmd_fail");
1413 TAVOR_TNF_EXIT(tavor_hw_init);
1414 return (DDI_FAILURE);
1415 }
1416
1417 /* Initialize the Phase2 Tavor configuration profile */
1418 status = tavor_cfg_profile_init_phase2(state);
1419 if (status != DDI_SUCCESS) {
1420 tavor_hw_fini(state, cleanup);
1421 TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, "");
1422 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail");
1423 TAVOR_TNF_EXIT(tavor_hw_init);
1424 return (DDI_FAILURE);
1425 }
1426
1427 /* Second phase of init for Tavor configuration/resources */
1428 status = tavor_rsrc_init_phase2(state);
1429 if (status != DDI_SUCCESS) {
1430 tavor_hw_fini(state, cleanup);
1431 TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, "");
1432 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1433 "hw_init_rsrcinit2_fail");
1434 TAVOR_TNF_EXIT(tavor_hw_init);
1435 return (DDI_FAILURE);
1436 }
1437 cleanup = TAVOR_DRV_CLEANUP_LEVEL7;
1438
1439 /* Miscellaneous query information */
1440 status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0,
1441 &state->ts_adapter, sizeof (tavor_hw_queryadapter_t),
1442 TAVOR_CMD_NOSLEEP_SPIN);
1443 if (status != TAVOR_CMD_SUCCESS) {
1444 cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n",
1445 status);
1446 tavor_hw_fini(state, cleanup);
1447 TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail,
1448 TAVOR_TNF_ERROR, "");
1449 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1450 "hw_init_query_adapter_cmd_fail");
1451 TAVOR_TNF_EXIT(tavor_hw_init);
1452 return (DDI_FAILURE);
1453 }
1454
1455 /* Prepare configuration for Tavor INIT_HCA command */
1456 tavor_hca_config_setup(state, &state->ts_hcaparams);
1457
1458 /* Post command to init Tavor HCA */
1459 status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams,
1460 TAVOR_CMD_NOSLEEP_SPIN);
1461 if (status != TAVOR_CMD_SUCCESS) {
1462 cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n",
1463 status);
1464 tavor_hw_fini(state, cleanup);
1465 TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail,
1466 TAVOR_TNF_ERROR, "");
1467 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1468 "hw_init_init_hca_cmd_fail");
1469 TAVOR_TNF_EXIT(tavor_hw_init);
1470 return (DDI_FAILURE);
1471 }
1472 cleanup = TAVOR_DRV_CLEANUP_LEVEL8;
1473
1474 /* Allocate protection domain (PD) for Tavor internal use */
1475 status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP);
1476 if (status != DDI_SUCCESS) {
1477 tavor_hw_fini(state, cleanup);
1478 TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail,
1479 TAVOR_TNF_ERROR, "");
1480 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1481 "hw_init_internal_pd_alloc_fail");
1482 TAVOR_TNF_EXIT(tavor_hw_init);
1483 return (DDI_FAILURE);
1484 }
1485 cleanup = TAVOR_DRV_CLEANUP_LEVEL9;
1486
1487 /* Setup Tavor internal UAR pages (0 and 1) */
1488 status = tavor_internal_uarpgs_init(state);
1489 if (status != DDI_SUCCESS) {
1490 tavor_hw_fini(state, cleanup);
1491 TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail,
1492 TAVOR_TNF_ERROR, "");
1493 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1494 "hw_init_internal_uarpgs_alloc_fail");
1495 TAVOR_TNF_EXIT(tavor_hw_init);
1496 return (DDI_FAILURE);
1497 }
1498 cleanup = TAVOR_DRV_CLEANUP_LEVEL10;
1499
1500 /* Query and initialize the Tavor interrupt/MSI information */
1501 status = tavor_intr_or_msi_init(state);
1502 if (status != DDI_SUCCESS) {
1503 tavor_hw_fini(state, cleanup);
1504 TNF_PROBE_0(tavor_intr_or_msi_init_fail,
1505 TAVOR_TNF_ERROR, "");
1506 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1507 "intr_or_msi_init_fail");
1508 TAVOR_TNF_EXIT(tavor_hw_init);
1509 return (DDI_FAILURE);
1510 }
1511 cleanup = TAVOR_DRV_CLEANUP_LEVEL11;
1512
1513 /* Setup all of the Tavor EQs */
1514 status = tavor_eq_init_all(state);
1515 if (status != DDI_SUCCESS) {
1516 tavor_hw_fini(state, cleanup);
1517 TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, "");
1518 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1519 "hw_init_eqinitall_fail");
1520 TAVOR_TNF_EXIT(tavor_hw_init);
1521 return (DDI_FAILURE);
1522 }
1523 cleanup = TAVOR_DRV_CLEANUP_LEVEL12;
1524
1525 /* Set aside contexts for QP0 and QP1 */
1526 status = tavor_special_qp_contexts_reserve(state);
1527 if (status != DDI_SUCCESS) {
1528 tavor_hw_fini(state, cleanup);
1529 TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail,
1530 TAVOR_TNF_ERROR, "");
1531 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1532 "hw_init_reserve_special_qp_fail");
1533 TAVOR_TNF_EXIT(tavor_hw_init);
1534 return (DDI_FAILURE);
1535 }
1536 cleanup = TAVOR_DRV_CLEANUP_LEVEL13;
1537
1538 /* Initialize for multicast group handling */
1539 status = tavor_mcg_init(state);
1540 if (status != DDI_SUCCESS) {
1541 tavor_hw_fini(state, cleanup);
1542 TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, "");
1543 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail");
1544 TAVOR_TNF_EXIT(tavor_hw_init);
1545 return (DDI_FAILURE);
1546 }
1547 cleanup = TAVOR_DRV_CLEANUP_LEVEL14;
1548
1549 /* Initialize the Tavor IB port(s) */
1550 status = tavor_hca_port_init(state);
1551 if (status != DDI_SUCCESS) {
1552 tavor_hw_fini(state, cleanup);
1553 TNF_PROBE_0(tavor_hw_init_hca_port_init_fail,
1554 TAVOR_TNF_ERROR, "");
1555 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1556 "hw_init_hca_port_init_fail");
1557 TAVOR_TNF_EXIT(tavor_hw_init);
1558 return (DDI_FAILURE);
1559 }
1560 cleanup = TAVOR_DRV_CLEANUP_ALL;
1561
1562 /* Determine NodeGUID and SystemImageGUID */
1563 status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1564 &nodeinfo);
1565 if (status != TAVOR_CMD_SUCCESS) {
1566 cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n",
1567 status);
1568 tavor_hw_fini(state, cleanup);
1569 TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail,
1570 TAVOR_TNF_ERROR, "");
1571 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1572 "hw_init_getnodeinfo_cmd_fail");
1573 TAVOR_TNF_EXIT(tavor_hw_init);
1574 return (DDI_FAILURE);
1575 }
1576
1577 /*
1578 * If the NodeGUID value was set in OBP properties, then we use that
1579 * value. But we still print a message if the value we queried from
1580 * firmware does not match this value.
1581 *
1582 * Otherwise if OBP value is not set then we use the value from
1583 * firmware unconditionally.
1584 */
1585 if (state->ts_cfg_profile->cp_nodeguid) {
1586 state->ts_nodeguid = state->ts_cfg_profile->cp_nodeguid;
1587 } else {
1588 state->ts_nodeguid = nodeinfo.NodeGUID;
1589 }
1590
1591 if (state->ts_nodeguid != nodeinfo.NodeGUID) {
1592 cmn_err(CE_NOTE, "!NodeGUID value queried from firmware "
1593 "does not match value set by device property");
1594 }
1595
1596 /*
1597 * If the SystemImageGUID value was set in OBP properties, then we use
1598 * that value. But we still print a message if the value we queried
1599 * from firmware does not match this value.
1600 *
1601 * Otherwise if OBP value is not set then we use the value from
1602 * firmware unconditionally.
1603 */
1604 if (state->ts_cfg_profile->cp_sysimgguid) {
1605 state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid;
1606 } else {
1607 state->ts_sysimgguid = nodeinfo.SystemImageGUID;
1608 }
1609
1610 if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) {
1611 cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware "
1612 "does not match value set by device property");
1613 }
1614
1615 /* Get NodeDescription */
1616 status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1617 (sm_nodedesc_t *)&state->ts_nodedesc);
1618 if (status != TAVOR_CMD_SUCCESS) {
1619 cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n",
1620 status);
1621 tavor_hw_fini(state, cleanup);
1622 TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail,
1623 TAVOR_TNF_ERROR, "");
1624 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1625 "hw_init_getnodedesc_cmd_fail");
1626 TAVOR_TNF_EXIT(tavor_hw_init);
1627 return (DDI_FAILURE);
1628 }
1629
1630 TAVOR_TNF_EXIT(tavor_hw_init);
1631 return (DDI_SUCCESS);
1632 }
1633
1634
1635 /*
1636 * tavor_hw_fini()
1637 * Context: Only called from attach() and/or detach() path contexts
1638 */
1639 static void
1640 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup)
1641 {
1642 uint_t num_ports;
1643 int status;
1644
1645 TAVOR_TNF_ENTER(tavor_hw_fini);
1646
1647 switch (cleanup) {
1648 /*
1649 * If we add more driver initialization steps that should be cleaned
1650 * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the
1651 * first entry (i.e. corresponds to the last init step).
1652 */
1653 case TAVOR_DRV_CLEANUP_ALL:
1654 /* Shutdown the Tavor IB port(s) */
1655 num_ports = state->ts_cfg_profile->cp_num_ports;
1656 (void) tavor_hca_ports_shutdown(state, num_ports);
1657 /* FALLTHROUGH */
1658
1659 case TAVOR_DRV_CLEANUP_LEVEL14:
1660 /* Teardown resources used for multicast group handling */
1661 tavor_mcg_fini(state);
1662 /* FALLTHROUGH */
1663
1664 case TAVOR_DRV_CLEANUP_LEVEL13:
1665 /* Unreserve the special QP contexts */
1666 tavor_special_qp_contexts_unreserve(state);
1667 /* FALLTHROUGH */
1668
1669 case TAVOR_DRV_CLEANUP_LEVEL12:
1670 /*
1671 * Attempt to teardown all event queues (EQ). If we fail
1672 * here then print a warning message and return. Something
1673 * (either in HW or SW) has gone seriously wrong.
1674 */
1675 status = tavor_eq_fini_all(state);
1676 if (status != DDI_SUCCESS) {
1677 TAVOR_WARNING(state, "failed to teardown EQs");
1678 TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail,
1679 TAVOR_TNF_ERROR, "");
1680 TAVOR_TNF_EXIT(tavor_hw_fini);
1681 return;
1682 }
1683 /* FALLTHROUGH */
1684
1685 case TAVOR_DRV_CLEANUP_LEVEL11:
1686 status = tavor_intr_or_msi_fini(state);
1687 if (status != DDI_SUCCESS) {
1688 TAVOR_WARNING(state, "failed to free intr/MSI");
1689 TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail,
1690 TAVOR_TNF_ERROR, "");
1691 TAVOR_TNF_EXIT(tavor_hw_fini);
1692 return;
1693 }
1694 /* FALLTHROUGH */
1695
1696 case TAVOR_DRV_CLEANUP_LEVEL10:
1697 /* Free the resources for the Tavor internal UAR pages */
1698 tavor_internal_uarpgs_fini(state);
1699 /* FALLTHROUGH */
1700
1701 case TAVOR_DRV_CLEANUP_LEVEL9:
1702 /*
1703 * Free the PD that was used internally by Tavor software. If
1704 * we fail here then print a warning and return. Something
1705 * (probably software-related, but perhaps HW) has gone wrong.
1706 */
1707 status = tavor_pd_free(state, &state->ts_pdhdl_internal);
1708 if (status != DDI_SUCCESS) {
1709 TAVOR_WARNING(state, "failed to free internal PD");
1710 TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail,
1711 TAVOR_TNF_ERROR, "");
1712 TAVOR_TNF_EXIT(tavor_hw_fini);
1713 return;
1714 }
1715 /* FALLTHROUGH */
1716
1717 case TAVOR_DRV_CLEANUP_LEVEL8:
1718 /*
1719 * Post the CLOSE_HCA command to Tavor firmware. If we fail
1720 * here then print a warning and return. Something (either in
1721 * HW or SW) has gone seriously wrong.
1722 */
1723 status = tavor_close_hca_cmd_post(state,
1724 TAVOR_CMD_NOSLEEP_SPIN);
1725 if (status != TAVOR_CMD_SUCCESS) {
1726 TAVOR_WARNING(state, "failed to shutdown HCA");
1727 TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail,
1728 TAVOR_TNF_ERROR, "");
1729 TAVOR_TNF_EXIT(tavor_hw_fini);
1730 return;
1731 }
1732 /* FALLTHROUGH */
1733
1734 case TAVOR_DRV_CLEANUP_LEVEL7:
1735 /* Cleanup all the phase2 resources first */
1736 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL);
1737 /* FALLTHROUGH */
1738
1739 case TAVOR_DRV_CLEANUP_LEVEL6:
1740 /* Then cleanup the phase1 resources */
1741 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE);
1742 /* FALLTHROUGH */
1743
1744 case TAVOR_DRV_CLEANUP_LEVEL5:
1745 /*
1746 * Post the SYS_DIS command to Tavor firmware to shut
1747 * everything down again. If we fail here then print a
1748 * warning and return. Something (probably in HW, but maybe
1749 * in SW) has gone seriously wrong.
1750 */
1751 status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN);
1752 if (status != TAVOR_CMD_SUCCESS) {
1753 TAVOR_WARNING(state, "failed to shutdown hardware");
1754 TNF_PROBE_0(tavor_hw_fini_sys_dis_fail,
1755 TAVOR_TNF_ERROR, "");
1756 TAVOR_TNF_EXIT(tavor_hw_fini);
1757 return;
1758 }
1759 /* FALLTHROUGH */
1760
1761 case TAVOR_DRV_CLEANUP_LEVEL4:
1762 /* Teardown any resources allocated for the config profile */
1763 tavor_cfg_profile_fini(state);
1764 /* FALLTHROUGH */
1765
1766 case TAVOR_DRV_CLEANUP_LEVEL3:
1767 ddi_regs_map_free(&state->ts_reg_ddrhdl);
1768 /* FALLTHROUGH */
1769
1770 case TAVOR_DRV_CLEANUP_LEVEL2:
1771 ddi_regs_map_free(&state->ts_reg_uarhdl);
1772 /* FALLTHROUGH */
1773
1774 case TAVOR_DRV_CLEANUP_LEVEL1:
1775 case TAVOR_DRV_CLEANUP_LEVEL0:
1776 /*
1777 * LEVEL1 and LEVEL0 resources are freed in
1778 * tavor_drv_fini2().
1779 */
1780 break;
1781
1782 default:
1783 TAVOR_WARNING(state, "unexpected driver cleanup level");
1784 TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, "");
1785 TAVOR_TNF_EXIT(tavor_hw_fini);
1786 return;
1787 }
1788
1789 TAVOR_TNF_EXIT(tavor_hw_fini);
1790 }
1791
1792
1793 /*
1794 * tavor_soft_state_init()
1795 * Context: Only called from attach() path context
1796 */
1797 static int
1798 tavor_soft_state_init(tavor_state_t *state)
1799 {
1800 ibt_hca_attr_t *hca_attr;
1801 uint64_t maxval, val;
1802 ibt_hca_flags_t caps = IBT_HCA_NO_FLAGS;
1803 int status;
1804
1805 TAVOR_TNF_ENTER(tavor_soft_state_init);
1806
1807 /*
1808 * The ibc_hca_info_t struct is passed to the IBTF. This is the
1809 * routine where we initialize it. Many of the init values come from
1810 * either configuration variables or successful queries of the Tavor
1811 * hardware abilities
1812 */
1813 state->ts_ibtfinfo.hca_ci_vers = IBCI_V4;
1814 state->ts_ibtfinfo.hca_handle = (ibc_hca_hdl_t)state;
1815 state->ts_ibtfinfo.hca_ops = &tavor_ibc_ops;
1816
1817 hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
1818 state->ts_ibtfinfo.hca_attr = hca_attr;
1819
1820 hca_attr->hca_dip = state->ts_dip;
1821 hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major;
1822 hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor;
1823 hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor;
1824
1825 /*
1826 * Determine HCA capabilities:
1827 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST,
1828 * IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE,
1829 * or IBT_HCA_SHUTDOWN_PORT
1830 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID,
1831 * IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always
1832 * supported
1833 * All other features are conditionally supported, depending on the
1834 * status return by the Tavor HCA (in QUERY_DEV_LIM)
1835 */
1836 if (state->ts_devlim.ud_multi) {
1837 caps |= IBT_HCA_UD_MULTICAST;
1838 }
1839 if (state->ts_devlim.atomic) {
1840 caps |= IBT_HCA_ATOMICS_HCA;
1841 }
1842 if (state->ts_devlim.apm) {
1843 caps |= IBT_HCA_AUTO_PATH_MIG;
1844 }
1845 if (state->ts_devlim.pkey_v) {
1846 caps |= IBT_HCA_PKEY_CNTR;
1847 }
1848 if (state->ts_devlim.qkey_v) {
1849 caps |= IBT_HCA_QKEY_CNTR;
1850 }
1851 if (state->ts_cfg_profile->cp_srq_enable) {
1852 caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ;
1853 }
1854 caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT |
1855 IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE |
1856 IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE);
1857 hca_attr->hca_flags = caps;
1858 hca_attr->hca_flags2 = IBT_HCA2_DMA_MR;
1859
1860 /* Determine VendorID, DeviceID, and revision ID */
1861 hca_attr->hca_vendor_id = state->ts_adapter.vendor_id;
1862 hca_attr->hca_device_id = state->ts_adapter.device_id;
1863 hca_attr->hca_version_id = state->ts_adapter.rev_id;
1864
1865 /*
1866 * Determine number of available QPs and max QP size. Number of
1867 * available QPs is determined by subtracting the number of
1868 * "reserved QPs" (i.e. reserved for firmware use) from the
1869 * total number configured.
1870 */
1871 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
1872 hca_attr->hca_max_qp = val - ((uint64_t)1 <<
1873 state->ts_devlim.log_rsvd_qp);
1874 maxval = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz);
1875 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz);
1876 if (val > maxval) {
1877 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1878 TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail,
1879 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size "
1880 "exceeds device maximum", tnf_uint, maxsz, maxval);
1881 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1882 "soft_state_init_maxqpsz_toobig_fail");
1883 TAVOR_TNF_EXIT(tavor_soft_state_init);
1884 return (DDI_FAILURE);
1885 }
1886 hca_attr->hca_max_qp_sz = val;
1887
1888 /* Determine max scatter-gather size in WQEs */
1889 maxval = state->ts_devlim.max_sg;
1890 val = state->ts_cfg_profile->cp_wqe_max_sgl;
1891 if (val > maxval) {
1892 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1893 TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail,
1894 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl "
1895 "exceeds device maximum", tnf_uint, maxsgl, maxval);
1896 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1897 "soft_state_init_toomanysgl_fail");
1898 TAVOR_TNF_EXIT(tavor_soft_state_init);
1899 return (DDI_FAILURE);
1900 }
1901 /* If the rounded value for max SGL is too large, cap it */
1902 if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) {
1903 state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval;
1904 val = maxval;
1905 } else {
1906 val = state->ts_cfg_profile->cp_wqe_real_max_sgl;
1907 }
1908
1909 hca_attr->hca_max_sgl = val;
1910 hca_attr->hca_max_rd_sgl = 0; /* zero because RD is unsupported */
1911
1912 /*
1913 * Determine number of available CQs and max CQ size. Number of
1914 * available CQs is determined by subtracting the number of
1915 * "reserved CQs" (i.e. reserved for firmware use) from the
1916 * total number configured.
1917 */
1918 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq);
1919 hca_attr->hca_max_cq = val - ((uint64_t)1 <<
1920 state->ts_devlim.log_rsvd_cq);
1921 maxval = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz);
1922 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1;
1923 if (val > maxval) {
1924 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1925 TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail,
1926 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size "
1927 "exceeds device maximum", tnf_uint, maxsz, maxval);
1928 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1929 "soft_state_init_maxcqsz_toobig_fail");
1930 TAVOR_TNF_EXIT(tavor_soft_state_init);
1931 return (DDI_FAILURE);
1932 }
1933 hca_attr->hca_max_cq_sz = val;
1934
1935 /*
1936 * Determine number of available SRQs and max SRQ size. Number of
1937 * available SRQs is determined by subtracting the number of
1938 * "reserved SRQs" (i.e. reserved for firmware use) from the
1939 * total number configured.
1940 */
1941 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq);
1942 hca_attr->hca_max_srqs = val - ((uint64_t)1 <<
1943 state->ts_devlim.log_rsvd_srq);
1944 maxval = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz);
1945 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz);
1946
1947 if (val > maxval) {
1948 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1949 TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail,
1950 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size "
1951 "exceeds device maximum", tnf_uint, maxsz, maxval);
1952 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1953 "soft_state_init_maxsrqsz_toobig_fail");
1954 TAVOR_TNF_EXIT(tavor_soft_state_init);
1955 return (DDI_FAILURE);
1956 }
1957 hca_attr->hca_max_srqs_sz = val;
1958
1959 val = state->ts_cfg_profile->cp_srq_max_sgl;
1960 maxval = state->ts_devlim.max_sg;
1961 if (val > maxval) {
1962 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1963 TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail,
1964 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq "
1965 "sgl exceeds device maximum", tnf_uint, maxsgl, maxval);
1966 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1967 "soft_state_init_toomanysrqsgl_fail");
1968 TAVOR_TNF_EXIT(tavor_soft_state_init);
1969 return (DDI_FAILURE);
1970 }
1971 hca_attr->hca_max_srq_sgl = val;
1972
1973 /*
1974 * Determine supported HCA page sizes
1975 * XXX
1976 * For now we simply return the system pagesize as the only supported
1977 * pagesize
1978 */
1979 hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K :
1980 IBT_PAGE_4K);
1981
1982 /*
1983 * Determine number of available MemReg, MemWin, and their max size.
1984 * Number of available MRs and MWs is determined by subtracting
1985 * the number of "reserved MPTs" (i.e. reserved for firmware use)
1986 * from the total number configured for each.
1987 */
1988 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt);
1989 hca_attr->hca_max_memr = val - ((uint64_t)1 <<
1990 state->ts_devlim.log_rsvd_mpt);
1991 hca_attr->hca_max_mem_win = val - ((uint64_t)1 <<
1992 state->ts_devlim.log_rsvd_mpt);
1993 maxval = state->ts_devlim.log_max_mrw_sz;
1994 val = state->ts_cfg_profile->cp_log_max_mrw_sz;
1995 if (val > maxval) {
1996 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1997 TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail,
1998 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size "
1999 "exceeds device maximum", tnf_uint, maxsz, maxval);
2000 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2001 "soft_state_init_maxmrwsz_toobig_fail");
2002 TAVOR_TNF_EXIT(tavor_soft_state_init);
2003 return (DDI_FAILURE);
2004 }
2005 hca_attr->hca_max_memr_len = ((uint64_t)1 << val);
2006
2007 /* Determine RDMA/Atomic properties */
2008 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb);
2009 hca_attr->hca_max_rsc = val;
2010 val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp;
2011 hca_attr->hca_max_rdma_in_qp = val;
2012 val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp;
2013 hca_attr->hca_max_rdma_out_qp = val;
2014 hca_attr->hca_max_rdma_in_ee = 0;
2015 hca_attr->hca_max_rdma_out_ee = 0;
2016
2017 /*
2018 * Determine maximum number of raw IPv6 and Ether QPs. Set to 0
2019 * because neither type of raw QP is supported
2020 */
2021 hca_attr->hca_max_ipv6_qp = 0;
2022 hca_attr->hca_max_ether_qp = 0;
2023
2024 /* Determine max number of MCGs and max QP-per-MCG */
2025 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
2026 hca_attr->hca_max_mcg_qps = val;
2027 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg);
2028 hca_attr->hca_max_mcg = val;
2029 val = state->ts_cfg_profile->cp_num_qp_per_mcg;
2030 hca_attr->hca_max_qp_per_mcg = val;
2031
2032 /* Determine max number partitions (i.e. PKeys) */
2033 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2034 val = ((uint64_t)state->ts_cfg_profile->cp_num_ports <<
2035 state->ts_cfg_profile->cp_log_max_pkeytbl);
2036
2037 if (val > maxval) {
2038 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2039 TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail,
2040 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys "
2041 "exceeds device maximum", tnf_uint, maxpkey, maxval);
2042 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2043 "soft_state_init_toomanypkey_fail");
2044 TAVOR_TNF_EXIT(tavor_soft_state_init);
2045 return (DDI_FAILURE);
2046 }
2047 hca_attr->hca_max_partitions = val;
2048
2049 /* Determine number of ports */
2050 maxval = state->ts_devlim.num_ports;
2051 val = state->ts_cfg_profile->cp_num_ports;
2052 if ((val > maxval) || (val == 0)) {
2053 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2054 TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail,
2055 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports "
2056 "exceeds device maximum", tnf_uint, maxports, maxval);
2057 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2058 "soft_state_init_toomanyports_fail");
2059 TAVOR_TNF_EXIT(tavor_soft_state_init);
2060 return (DDI_FAILURE);
2061 }
2062 hca_attr->hca_nports = val;
2063
2064 /* Copy NodeGUID and SystemImageGUID from softstate */
2065 hca_attr->hca_node_guid = state->ts_nodeguid;
2066 hca_attr->hca_si_guid = state->ts_sysimgguid;
2067
2068 /*
2069 * Determine local ACK delay. Use the value suggested by the Tavor
2070 * hardware (from the QUERY_DEV_LIM command)
2071 */
2072 hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay;
2073
2074 /* Determine max SGID table and PKey table sizes */
2075 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl);
2076 hca_attr->hca_max_port_sgid_tbl_sz = val;
2077 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
2078 hca_attr->hca_max_port_pkey_tbl_sz = val;
2079
2080 /* Determine max number of PDs */
2081 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pd);
2082 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd);
2083 if (val > maxval) {
2084 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2085 TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail,
2086 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD "
2087 "exceeds device maximum", tnf_uint, maxpd, maxval);
2088 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2089 "soft_state_init_toomanypd_fail");
2090 TAVOR_TNF_EXIT(tavor_soft_state_init);
2091 return (DDI_FAILURE);
2092 }
2093 hca_attr->hca_max_pd = val;
2094
2095 /* Determine max number of Address Handles */
2096 maxval = ((uint64_t)1 << state->ts_devlim.log_max_av);
2097 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah);
2098 if (val > maxval) {
2099 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2100 TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail,
2101 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH "
2102 "exceeds device maximum", tnf_uint, maxah, maxval);
2103 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2104 "soft_state_init_toomanyah_fail");
2105 TAVOR_TNF_EXIT(tavor_soft_state_init);
2106 return (DDI_FAILURE);
2107 }
2108 hca_attr->hca_max_ah = val;
2109
2110 /* No RDDs or EECs (since Reliable Datagram is not supported) */
2111 hca_attr->hca_max_rdd = 0;
2112 hca_attr->hca_max_eec = 0;
2113
2114 /* Initialize lock for reserved UAR page access */
2115 mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER,
2116 DDI_INTR_PRI(state->ts_intrmsi_pri));
2117
2118 /* Initialize the flash fields */
2119 state->ts_fw_flashstarted = 0;
2120 mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER,
2121 DDI_INTR_PRI(state->ts_intrmsi_pri));
2122
2123 /* Initialize the lock for the info ioctl */
2124 mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER,
2125 DDI_INTR_PRI(state->ts_intrmsi_pri));
2126
2127 /* Initialize the AVL tree for QP number support */
2128 tavor_qpn_avl_init(state);
2129
2130 /* Initialize the kstat info structure */
2131 status = tavor_kstat_init(state);
2132 if (status != DDI_SUCCESS) {
2133 tavor_qpn_avl_fini(state);
2134 mutex_destroy(&state->ts_info_lock);
2135 mutex_destroy(&state->ts_fw_flashlock);
2136 mutex_destroy(&state->ts_uar_lock);
2137 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2138 TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail,
2139 TAVOR_TNF_ERROR, "");
2140 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2141 "soft_state_init_kstatinit_fail");
2142 TAVOR_TNF_EXIT(tavor_soft_state_init);
2143 return (DDI_FAILURE);
2144 }
2145
2146 TAVOR_TNF_EXIT(tavor_soft_state_init);
2147 return (DDI_SUCCESS);
2148 }
2149
2150
2151 /*
2152 * tavor_soft_state_fini()
2153 * Context: Called only from detach() path context
2154 */
2155 static void
2156 tavor_soft_state_fini(tavor_state_t *state)
2157 {
2158 TAVOR_TNF_ENTER(tavor_soft_state_fini);
2159
2160 /* Teardown the kstat info */
2161 tavor_kstat_fini(state);
2162
2163 /* Teardown the AVL tree for QP number support */
2164 tavor_qpn_avl_fini(state);
2165
2166 /* Free up info ioctl mutex */
2167 mutex_destroy(&state->ts_info_lock);
2168
2169 /* Free up flash mutex */
2170 mutex_destroy(&state->ts_fw_flashlock);
2171
2172 /* Free up the UAR page access mutex */
2173 mutex_destroy(&state->ts_uar_lock);
2174
2175 /* Free up the hca_attr struct */
2176 kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t));
2177
2178 TAVOR_TNF_EXIT(tavor_soft_state_fini);
2179 }
2180
2181
2182 /*
2183 * tavor_hca_config_setup()
2184 * Context: Only called from attach() path context
2185 */
2186 static void
2187 tavor_hca_config_setup(tavor_state_t *state,
2188 tavor_hw_initqueryhca_t *inithca)
2189 {
2190 tavor_rsrc_pool_info_t *rsrc_pool;
2191 uint64_t ddr_baseaddr, ddr_base_map_addr;
2192 uint64_t offset, addr;
2193 uint_t mcg_size;
2194
2195 TAVOR_TNF_ENTER(tavor_hca_config_setup);
2196
2197 /* Set "host endianness". Default is big endian */
2198 #ifdef _LITTLE_ENDIAN
2199 inithca->big_endian = 0;
2200 #else
2201 inithca->big_endian = 1;
2202 #endif
2203 /* No Address Vector Protection, but Port Checking on by default */
2204 inithca->udav_chk = TAVOR_UDAV_PROTECT_DISABLED;
2205 inithca->udav_port_chk = TAVOR_UDAV_PORTCHK_ENABLED;
2206
2207 ddr_baseaddr = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr;
2208 ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr;
2209
2210 /* Setup QPC table */
2211 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
2212 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2213 addr = ddr_base_map_addr + offset;
2214 inithca->context.qpc_baseaddr_h = (addr >> 32);
2215 inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7;
2216 inithca->context.log_num_qp = state->ts_cfg_profile->cp_log_num_qp;
2217
2218 /* Setup EEC table (initialize to zero - RD unsupported) */
2219 inithca->context.eec_baseaddr_h = 0;
2220 inithca->context.eec_baseaddr_l = 0;
2221 inithca->context.log_num_ee = 0;
2222
2223 /* Setup CQC table */
2224 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
2225 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2226 addr = ddr_base_map_addr + offset;
2227 inithca->context.cqc_baseaddr_h = (addr >> 32);
2228 inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2229 inithca->context.log_num_cq = state->ts_cfg_profile->cp_log_num_cq;
2230
2231 /* Setup SRQC table */
2232 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
2233 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2234 addr = ddr_base_map_addr + offset;
2235 inithca->context.srqc_baseaddr_h = (addr >> 32);
2236 inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2237 inithca->context.log_num_srq =
2238 state->ts_cfg_profile->cp_log_num_srq;
2239
2240 /* Setup EQPC table */
2241 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
2242 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2243 addr = ddr_base_map_addr + offset;
2244 inithca->context.eqpc_baseaddr = addr;
2245
2246 /* Setup EEEC table (initialize to zero - RD unsupported) */
2247 inithca->context.eeec_baseaddr = 0;
2248
2249 /* Setup EQC table */
2250 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
2251 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2252 addr = ddr_base_map_addr + offset;
2253 inithca->context.eqc_baseaddr_h = (addr >> 32);
2254 inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2255 inithca->context.log_num_eq = TAVOR_NUM_EQ_SHIFT;
2256
2257 /* Setup RDB table */
2258 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
2259 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2260 addr = ddr_base_map_addr + offset;
2261 inithca->context.rdb_baseaddr_h = (addr >> 32);
2262 inithca->context.rdb_baseaddr_l = 0;
2263
2264 /* Setup Multicast */
2265 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
2266 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2267 addr = ddr_base_map_addr + offset;
2268 inithca->multi.mc_baseaddr = addr;
2269 mcg_size = TAVOR_MCGMEM_SZ(state);
2270 inithca->multi.log_mc_tbl_ent = highbit(mcg_size) - 1;
2271 inithca->multi.mc_tbl_hash_sz =
2272 (1 << state->ts_cfg_profile->cp_log_num_mcg_hash);
2273 inithca->multi.mc_hash_fn = TAVOR_MCG_DEFAULT_HASH_FN;
2274 inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg;
2275
2276
2277 /* Setup TPT */
2278 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
2279 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2280 addr = ddr_base_map_addr + offset;
2281 inithca->tpt.mpt_baseaddr = addr;
2282 inithca->tpt.mttseg_sz = TAVOR_MTTSEG_SIZE_SHIFT;
2283 inithca->tpt.log_mpt_sz = state->ts_cfg_profile->cp_log_num_mpt;
2284 inithca->tpt.mtt_version = TAVOR_MTT_PG_WALK_VER;
2285
2286 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
2287 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2288 addr = ddr_base_map_addr + offset;
2289 inithca->tpt.mtt_baseaddr = addr;
2290
2291 /* Setup UAR */
2292 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
2293 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2294 addr = ddr_base_map_addr + offset;
2295 inithca->uar.uarscr_baseaddr = addr;
2296
2297 inithca->uar.uar_pg_sz = PAGESHIFT - 0xC;
2298
2299 TAVOR_TNF_EXIT(tavor_hca_config_setup);
2300 }
2301
2302
2303 /*
2304 * tavor_hca_port_init()
2305 * Context: Only called from attach() path context
2306 */
2307 static int
2308 tavor_hca_port_init(tavor_state_t *state)
2309 {
2310 tavor_hw_initib_t *portinits, *initib;
2311 tavor_cfg_profile_t *cfgprof;
2312 uint_t num_ports;
2313 int i, status;
2314 uint64_t maxval, val;
2315 uint64_t sysimgguid, nodeguid, portguid;
2316
2317 TAVOR_TNF_ENTER(tavor_hca_port_init);
2318
2319 cfgprof = state->ts_cfg_profile;
2320
2321 /* Get number of HCA ports */
2322 num_ports = cfgprof->cp_num_ports;
2323
2324 /* Allocate space for Tavor port init struct(s) */
2325 portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports *
2326 sizeof (tavor_hw_initib_t), KM_SLEEP);
2327
2328 /* Post command to initialize Tavor HCA port */
2329 for (i = 0; i < num_ports; i++) {
2330 initib = &portinits[i];
2331
2332 /*
2333 * Determine whether we need to override the firmware's
2334 * default SystemImageGUID setting.
2335 */
2336 sysimgguid = cfgprof->cp_sysimgguid;
2337 if (sysimgguid != 0) {
2338 initib->set_sysimg_guid = 1;
2339 initib->sysimg_guid = sysimgguid;
2340 }
2341
2342 /*
2343 * Determine whether we need to override the firmware's
2344 * default NodeGUID setting.
2345 */
2346 nodeguid = cfgprof->cp_nodeguid;
2347 if (nodeguid != 0) {
2348 initib->set_node_guid = 1;
2349 initib->node_guid = nodeguid;
2350 }
2351
2352 /*
2353 * Determine whether we need to override the firmware's
2354 * default PortGUID setting.
2355 */
2356 portguid = cfgprof->cp_portguid[i];
2357 if (portguid != 0) {
2358 initib->set_port_guid0 = 1;
2359 initib->guid0 = portguid;
2360 }
2361
2362 /* Validate max MTU size */
2363 maxval = state->ts_devlim.max_mtu;
2364 val = cfgprof->cp_max_mtu;
2365 if (val > maxval) {
2366 TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail,
2367 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2368 "MTU size exceeds device maximum", tnf_uint,
2369 maxmtu, maxval);
2370 TAVOR_TNF_EXIT(tavor_hca_port_init);
2371 goto init_ports_fail;
2372 }
2373 initib->mtu_cap = val;
2374
2375 /* Validate the max port width */
2376 maxval = state->ts_devlim.max_port_width;
2377 val = cfgprof->cp_max_port_width;
2378 if (val > maxval) {
2379 TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail,
2380 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2381 "port width exceeds device maximum", tnf_uint,
2382 maxportwidth, maxval);
2383 TAVOR_TNF_EXIT(tavor_hca_port_init);
2384 goto init_ports_fail;
2385 }
2386 initib->port_width_cap = val;
2387
2388 /* Validate max VL cap size */
2389 maxval = state->ts_devlim.max_vl;
2390 val = cfgprof->cp_max_vlcap;
2391 if (val > maxval) {
2392 TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail,
2393 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2394 "VLcap size exceeds device maximum", tnf_uint,
2395 maxvlcap, maxval);
2396 TAVOR_TNF_EXIT(tavor_hca_port_init);
2397 goto init_ports_fail;
2398 }
2399 initib->vl_cap = val;
2400
2401 /* Validate max GID table size */
2402 maxval = ((uint64_t)1 << state->ts_devlim.log_max_gid);
2403 val = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl);
2404 if (val > maxval) {
2405 TNF_PROBE_2(tavor_hca_port_init_gidtable_fail,
2406 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2407 "GID table size exceeds device maximum", tnf_uint,
2408 maxgidtbl, maxval);
2409 TAVOR_TNF_EXIT(tavor_hca_port_init);
2410 goto init_ports_fail;
2411 }
2412 initib->max_gid = val;
2413
2414 /* Validate max PKey table size */
2415 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2416 val = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl);
2417 if (val > maxval) {
2418 TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail,
2419 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2420 "PKey table size exceeds device maximum", tnf_uint,
2421 maxpkeytbl, maxval);
2422 TAVOR_TNF_EXIT(tavor_hca_port_init);
2423 goto init_ports_fail;
2424 }
2425 initib->max_pkey = val;
2426
2427 /*
2428 * Post the INIT_IB command to Tavor firmware. When this
2429 * command completes, the corresponding Tavor port will be
2430 * physically "Up" and initialized.
2431 */
2432 status = tavor_init_ib_cmd_post(state, initib, i + 1,
2433 TAVOR_CMD_NOSLEEP_SPIN);
2434 if (status != TAVOR_CMD_SUCCESS) {
2435 cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command "
2436 "failed: %08x\n", i + 1, status);
2437 TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail,
2438 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2439 tnf_uint, port, i + 1);
2440 TAVOR_TNF_EXIT(tavor_hca_port_init);
2441 goto init_ports_fail;
2442 }
2443 }
2444
2445 /* Free up the memory for Tavor port init struct(s), return success */
2446 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2447 TAVOR_TNF_EXIT(tavor_hca_port_init);
2448 return (DDI_SUCCESS);
2449
2450 init_ports_fail:
2451 /*
2452 * Free up the memory for Tavor port init struct(s), shutdown any
2453 * successfully initialized ports, and return failure
2454 */
2455 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2456 (void) tavor_hca_ports_shutdown(state, i);
2457
2458 TAVOR_TNF_EXIT(tavor_hca_port_init);
2459 return (DDI_FAILURE);
2460 }
2461
2462
2463 /*
2464 * tavor_hca_ports_shutdown()
2465 * Context: Only called from attach() and/or detach() path contexts
2466 */
2467 static int
2468 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init)
2469 {
2470 int i, status;
2471
2472 TAVOR_TNF_ENTER(tavor_hca_ports_shutdown);
2473
2474 /*
2475 * Post commands to shutdown all init'd Tavor HCA ports. Note: if
2476 * any of these commands fail for any reason, it would be entirely
2477 * unexpected and probably indicative a serious problem (HW or SW).
2478 * Although we do return void from this function, this type of failure
2479 * should not go unreported. That is why we have the warning message
2480 * and the detailed TNF information.
2481 */
2482 for (i = 0; i < num_init; i++) {
2483 status = tavor_close_ib_cmd_post(state, i + 1,
2484 TAVOR_CMD_NOSLEEP_SPIN);
2485 if (status != TAVOR_CMD_SUCCESS) {
2486 TAVOR_WARNING(state, "failed to shutdown HCA port");
2487 TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail,
2488 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2489 tnf_uint, port, i + 1);
2490 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2491 return (status);
2492 }
2493 }
2494
2495 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2496
2497 return (TAVOR_CMD_SUCCESS);
2498 }
2499
2500
2501 /*
2502 * tavor_internal_uarpgs_init
2503 * Context: Only called from attach() path context
2504 */
2505 static int
2506 tavor_internal_uarpgs_init(tavor_state_t *state)
2507 {
2508 int status;
2509
2510 TAVOR_TNF_ENTER(tavor_internal_uarpgs_init);
2511
2512 /*
2513 * Save away reserved Tavor UAR page #0. This UAR page is not to
2514 * be used by software.
2515 */
2516 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2517 &state->ts_uarpg0_rsrc_rsrvd);
2518 if (status != DDI_SUCCESS) {
2519 TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2520 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2521 return (DDI_FAILURE);
2522 }
2523
2524 /*
2525 * Save away Tavor UAR page #1 (for internal use). This UAR page is
2526 * the privileged UAR page through which all kernel generated
2527 * doorbells will be rung.
2528 */
2529 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2530 &state->ts_uarpg1_rsrc);
2531 if (status != DDI_SUCCESS) {
2532 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2533 TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2534 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2535 return (DDI_FAILURE);
2536 }
2537
2538 /* Setup pointer to UAR page #1 doorbells */
2539 state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr;
2540
2541 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2542 return (DDI_SUCCESS);
2543 }
2544
2545
2546 /*
2547 * tavor_internal_uarpgs_fini
2548 * Context: Only called from attach() and/or detach() path contexts
2549 */
2550 static void
2551 tavor_internal_uarpgs_fini(tavor_state_t *state)
2552 {
2553 TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini);
2554
2555 /* Free up Tavor UAR page #1 (kernel driver doorbells) */
2556 tavor_rsrc_free(state, &state->ts_uarpg1_rsrc);
2557
2558 /* Free up Tavor UAR page #0 (reserved) */
2559 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2560
2561 TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini);
2562 }
2563
2564
2565 /*
2566 * tavor_special_qp_contexts_reserve()
2567 * Context: Only called from attach() path context
2568 */
2569 static int
2570 tavor_special_qp_contexts_reserve(tavor_state_t *state)
2571 {
2572 tavor_rsrc_t *qp0_rsrc, *qp1_rsrc;
2573 int status;
2574
2575 TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve);
2576
2577 /* Initialize the lock used for special QP rsrc management */
2578 mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER,
2579 DDI_INTR_PRI(state->ts_intrmsi_pri));
2580
2581 /*
2582 * Reserve contexts for QP0. These QP contexts will be setup to
2583 * act as aliases for the real QP0. Note: We are required to grab
2584 * two QPs (one per port) even if we are operating in single-port
2585 * mode.
2586 */
2587 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc);
2588 if (status != DDI_SUCCESS) {
2589 mutex_destroy(&state->ts_spec_qplock);
2590 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail,
2591 TAVOR_TNF_ERROR, "");
2592 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2593 return (DDI_FAILURE);
2594 }
2595 state->ts_spec_qp0 = qp0_rsrc;
2596
2597 /*
2598 * Reserve contexts for QP1. These QP contexts will be setup to
2599 * act as aliases for the real QP1. Note: We are required to grab
2600 * two QPs (one per port) even if we are operating in single-port
2601 * mode.
2602 */
2603 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc);
2604 if (status != DDI_SUCCESS) {
2605 tavor_rsrc_free(state, &qp0_rsrc);
2606 mutex_destroy(&state->ts_spec_qplock);
2607 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail,
2608 TAVOR_TNF_ERROR, "");
2609 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2610 return (DDI_FAILURE);
2611 }
2612 state->ts_spec_qp1 = qp1_rsrc;
2613
2614 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2615 return (DDI_SUCCESS);
2616 }
2617
2618
2619 /*
2620 * tavor_special_qp_contexts_unreserve()
2621 * Context: Only called from attach() and/or detach() path contexts
2622 */
2623 static void
2624 tavor_special_qp_contexts_unreserve(tavor_state_t *state)
2625 {
2626 TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve);
2627
2628 /* Unreserve contexts for QP1 */
2629 tavor_rsrc_free(state, &state->ts_spec_qp1);
2630
2631 /* Unreserve contexts for QP0 */
2632 tavor_rsrc_free(state, &state->ts_spec_qp0);
2633
2634 /* Destroy the lock used for special QP rsrc management */
2635 mutex_destroy(&state->ts_spec_qplock);
2636
2637 TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve);
2638 }
2639
2640
2641 /*
2642 * tavor_sw_reset()
2643 * Context: Currently called only from attach() path context
2644 */
2645 static int
2646 tavor_sw_reset(tavor_state_t *state)
2647 {
2648 dev_info_t *dip, *pdip;
2649 ddi_acc_handle_t hdl = state->ts_pci_cfghdl, phdl;
2650 uint32_t reset_delay;
2651 int status, i;
2652
2653 TAVOR_TNF_ENTER(tavor_sw_reset);
2654
2655 /*
2656 * If the configured software reset delay is set to zero, then we
2657 * will not attempt a software reset of the Tavor device.
2658 */
2659 reset_delay = state->ts_cfg_profile->cp_sw_reset_delay;
2660 if (reset_delay == 0) {
2661 TAVOR_TNF_EXIT(tavor_sw_reset);
2662 return (DDI_SUCCESS);
2663 }
2664
2665 /*
2666 * Get dip for HCA device _and_ parent device as well. Parent access
2667 * is necessary here because software reset of the Tavor hardware
2668 * will reinitialize both the config registers of the PCI bridge
2669 * (parent, if it exists) and the IB HCA (self)
2670 */
2671 dip = state->ts_dip;
2672 pdip = ddi_get_parent(dip);
2673
2674 /* Query the PCI capabilities of the HCA device */
2675 tavor_pci_capability_list(state, hdl);
2676
2677 /*
2678 * Read all PCI config info (reg0...reg63). Note: According to the
2679 * Tavor software reset application note, we should not read or
2680 * restore the values in reg22 and reg23.
2681 */
2682 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2683 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2684 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2685 state->ts_cfg_data[i] = pci_config_get32(hdl, i << 2);
2686 }
2687 }
2688
2689 if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2690 /*
2691 * Setup for PCI config read/write of bridge device
2692 */
2693 status = pci_config_setup(pdip, &phdl);
2694 if (status != DDI_SUCCESS) {
2695 TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail,
2696 TAVOR_TNF_ERROR, "");
2697 TAVOR_TNF_EXIT(tavor_sw_reset);
2698 return (DDI_FAILURE);
2699 }
2700
2701 /*
2702 * Read all PCI config info (reg0...reg63). Note: According to
2703 * the Tavor software reset application note, we should not
2704 * read or restore the values in reg22 and reg23.
2705 */
2706 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2707 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2708 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2709 state->ts_cfg_pdata[i] =
2710 pci_config_get32(phdl, i << 2);
2711 }
2712 }
2713 }
2714
2715 /*
2716 * Perform the software reset (by writing 1 at offset 0xF0010)
2717 */
2718 ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset,
2719 TAVOR_SW_RESET_START);
2720
2721 drv_usecwait(reset_delay);
2722
2723 if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2724 /*
2725 * Bridge exists, so wait for the bridge to become ready.
2726 *
2727 * The above delay is necessary to avoid system panic from
2728 * Master Abort. If the device is accessed before this delay,
2729 * device will not respond to config cycles and they will be
2730 * terminate with a Master Abort which will panic the system.
2731 * Below is the loop we use to poll status from the device to
2732 * determine if it is OK to proceed.
2733 */
2734 i = 0;
2735 while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2736 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2737 }
2738
2739 /*
2740 * Write all the PCI config registers back into each device
2741 * (except for reg22 and reg23 - see above)
2742 */
2743 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2744 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2745 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2746 pci_config_put32(phdl, i << 2,
2747 state->ts_cfg_pdata[i]);
2748 }
2749 }
2750
2751 /*
2752 * Tear down the config setup (for bridge device)
2753 */
2754 pci_config_teardown(&phdl);
2755
2756 /* No Bridge Device */
2757 } else {
2758 /*
2759 * Bridge does not exist, so instead wait for the device itself
2760 * to become ready.
2761 *
2762 * The above delay is necessary to avoid system panic from
2763 * Master Abort. If the device is accessed before this delay,
2764 * device will not respond to config cycles and they will be
2765 * terminate with a Master Abort which will panic the system.
2766 * Below is the loop we use to poll status from the device to
2767 * determine if it is OK to proceed.
2768 */
2769 i = 0;
2770 while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2771 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2772 }
2773 }
2774
2775 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2776 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2777 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2778 pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]);
2779 }
2780 }
2781
2782 TAVOR_TNF_EXIT(tavor_sw_reset);
2783 return (DDI_SUCCESS);
2784 }
2785
2786
2787 /*
2788 * tavor_mcg_init()
2789 * Context: Only called from attach() path context
2790 */
2791 static int
2792 tavor_mcg_init(tavor_state_t *state)
2793 {
2794 uint_t mcg_tmp_sz;
2795
2796 TAVOR_TNF_ENTER(tavor_mcg_init);
2797
2798 /*
2799 * Allocate space for the MCG temporary copy buffer. This is
2800 * used by the Attach/Detach Multicast Group code
2801 */
2802 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2803 state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP);
2804
2805 /*
2806 * Initialize the multicast group mutex. This ensures atomic
2807 * access to add, modify, and remove entries in the multicast
2808 * group hash lists.
2809 */
2810 mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER,
2811 DDI_INTR_PRI(state->ts_intrmsi_pri));
2812
2813 TAVOR_TNF_EXIT(tavor_mcg_init);
2814 return (DDI_SUCCESS);
2815 }
2816
2817
2818 /*
2819 * tavor_mcg_fini()
2820 * Context: Only called from attach() and/or detach() path contexts
2821 */
2822 static void
2823 tavor_mcg_fini(tavor_state_t *state)
2824 {
2825 uint_t mcg_tmp_sz;
2826
2827 TAVOR_TNF_ENTER(tavor_mcg_fini);
2828
2829 /* Free up the space used for the MCG temporary copy buffer */
2830 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2831 kmem_free(state->ts_mcgtmp, mcg_tmp_sz);
2832
2833 /* Destroy the multicast group mutex */
2834 mutex_destroy(&state->ts_mcglock);
2835
2836 TAVOR_TNF_EXIT(tavor_mcg_fini);
2837 }
2838
2839
2840 /*
2841 * tavor_fw_version_check()
2842 * Context: Only called from attach() path context
2843 */
2844 static int
2845 tavor_fw_version_check(tavor_state_t *state)
2846 {
2847 uint_t tavor_fw_ver_major;
2848 uint_t tavor_fw_ver_minor;
2849 uint_t tavor_fw_ver_subminor;
2850
2851 /*
2852 * Depending on which version of driver we have attached, the firmware
2853 * version checks will be different. We set up the comparison values
2854 * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware
2855 * running in tavor mode).
2856 */
2857 switch (state->ts_operational_mode) {
2858 case TAVOR_HCA_MODE:
2859 tavor_fw_ver_major = TAVOR_FW_VER_MAJOR;
2860 tavor_fw_ver_minor = TAVOR_FW_VER_MINOR;
2861 tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR;
2862 break;
2863
2864 case TAVOR_COMPAT_MODE:
2865 tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR;
2866 tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR;
2867 tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR;
2868 break;
2869
2870 default:
2871 return (DDI_FAILURE);
2872 }
2873
2874 /*
2875 * If FW revision major number is less than acceptable,
2876 * return failure, else if greater return success. If
2877 * the major numbers are equal than check the minor number
2878 */
2879 if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) {
2880 return (DDI_FAILURE);
2881 } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) {
2882 return (DDI_SUCCESS);
2883 }
2884 /*
2885 * Do the same check as above, except for minor revision numbers
2886 * If the minor numbers are equal than check the subminor number
2887 */
2888 if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) {
2889 return (DDI_FAILURE);
2890 } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) {
2891 return (DDI_SUCCESS);
2892 }
2893
2894 /*
2895 * Once again we do the same check as above, except for the subminor
2896 * revision number. If the subminor numbers are equal here, then
2897 * these are the same firmware version, return success
2898 */
2899 if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) {
2900 return (DDI_FAILURE);
2901 } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) {
2902 return (DDI_SUCCESS);
2903 }
2904
2905 return (DDI_SUCCESS);
2906 }
2907
2908
2909 /*
2910 * tavor_device_info_report()
2911 * Context: Only called from attach() path context
2912 */
2913 static void
2914 tavor_device_info_report(tavor_state_t *state)
2915 {
2916 cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, "
2917 "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major,
2918 state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor,
2919 state->ts_adapter.rev_id);
2920 cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n",
2921 state->ts_instance, state->ts_nodedesc, state->ts_nodeguid);
2922 }
2923
2924
2925 /*
2926 * tavor_pci_capability_list()
2927 * Context: Only called from attach() path context
2928 */
2929 static void
2930 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl)
2931 {
2932 uint_t offset, data;
2933
2934 TAVOR_TNF_ENTER(tavor_pci_capability_list);
2935
2936 /*
2937 * Check for the "PCI Capabilities" bit in the "Status Register".
2938 * Bit 4 in this register indicates the presence of a "PCI
2939 * Capabilities" list.
2940 */
2941 data = pci_config_get16(hdl, 0x6);
2942 if ((data & 0x10) == 0) {
2943 TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, "");
2944 TAVOR_TNF_EXIT(tavor_pci_capability_list);
2945 return;
2946 }
2947
2948 /*
2949 * Starting from offset 0x34 in PCI config space, find the
2950 * head of "PCI capabilities" list, and walk the list. If
2951 * capabilities of a known type are encountered (e.g.
2952 * "PCI-X Capability"), then call the appropriate handler
2953 * function.
2954 */
2955 offset = pci_config_get8(hdl, 0x34);
2956 while (offset != 0x0) {
2957 data = pci_config_get8(hdl, offset);
2958
2959 /*
2960 * Check for known capability types. Tavor has the
2961 * following:
2962 * o VPD Capability (0x03)
2963 * o PCI-X Capability (0x07)
2964 * o MSI Capability (0x05)
2965 * o MSIX Capability (0x11)
2966 */
2967 switch (data) {
2968 case 0x03:
2969 tavor_pci_capability_vpd(state, hdl, offset);
2970 break;
2971 case 0x07:
2972 tavor_pci_capability_pcix(state, hdl, offset);
2973 break;
2974 case 0x05:
2975 break;
2976 default:
2977 break;
2978 }
2979
2980 /* Get offset of next entry in list */
2981 offset = pci_config_get8(hdl, offset + 1);
2982 }
2983
2984 TAVOR_TNF_EXIT(tavor_pci_capability_list);
2985 }
2986
2987 /*
2988 * tavor_pci_read_vpd()
2989 * Context: Only called from attach() path context
2990 * utility routine for tavor_pci_capability_vpd()
2991 */
2992 static int
2993 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr,
2994 uint32_t *data)
2995 {
2996 int retry = 4; /* retry counter for EEPROM poll */
2997 uint32_t val;
2998 int vpd_addr = offset + 2;
2999 int vpd_data = offset + 4;
3000
3001 TAVOR_TNF_ENTER(tavor_pci_read_vpd);
3002
3003 /*
3004 * In order to read a 32-bit value from VPD, we are to write down
3005 * the address (offset in the VPD itself) to the address register.
3006 * To signal the read, we also clear bit 31. We then poll on bit 31
3007 * and when it is set, we can then read our 4 bytes from the data
3008 * register.
3009 */
3010 (void) pci_config_put32(hdl, offset, addr << 16);
3011 do {
3012 drv_usecwait(1000);
3013 val = pci_config_get16(hdl, vpd_addr);
3014 if ((val >> 15) & 0x01) {
3015 *data = pci_config_get32(hdl, vpd_data);
3016 TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3017 return (DDI_SUCCESS);
3018 }
3019 } while (--retry);
3020
3021 TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, "");
3022 TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3023 return (DDI_FAILURE);
3024 }
3025
3026
3027 /*
3028 * tavor_pci_capability_vpd()
3029 * Context: Only called from attach() path context
3030 */
3031 static void
3032 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl,
3033 uint_t offset)
3034 {
3035 uint8_t name_length;
3036 uint8_t pn_length;
3037 int i, err = 0;
3038 int vpd_str_id = 0;
3039 int vpd_ro_desc;
3040 int vpd_ro_pn_desc;
3041 #ifndef _LITTLE_ENDIAN
3042 uint32_t data32;
3043 #endif /* _LITTLE_ENDIAN */
3044 union {
3045 uint32_t vpd_int[TAVOR_VPD_HDR_DWSIZE];
3046 uchar_t vpd_char[TAVOR_VPD_HDR_BSIZE];
3047 } vpd;
3048
3049 TAVOR_TNF_ENTER(tavor_pci_capability_vpd);
3050
3051 /*
3052 * Read Vital Product Data (VPD) from PCI-X capability.
3053 */
3054 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3055 err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]);
3056 if (err != DDI_SUCCESS) {
3057 cmn_err(CE_NOTE, "!VPD read failed\n");
3058 goto out;
3059 }
3060 }
3061
3062 #ifndef _LITTLE_ENDIAN
3063 /*
3064 * Need to swap bytes for big endian.
3065 */
3066 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3067 data32 = vpd.vpd_int[i];
3068 vpd.vpd_char[(i << 2) + 3] =
3069 (uchar_t)((data32 & 0xFF000000) >> 24);
3070 vpd.vpd_char[(i << 2) + 2] =
3071 (uchar_t)((data32 & 0x00FF0000) >> 16);
3072 vpd.vpd_char[(i << 2) + 1] =
3073 (uchar_t)((data32 & 0x0000FF00) >> 8);
3074 vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF);
3075 }
3076 #endif /* _LITTLE_ENDIAN */
3077
3078 /* Check for VPD String ID Tag */
3079 if (vpd.vpd_char[vpd_str_id] == 0x82) {
3080 /* get the product name */
3081 name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1];
3082 if (name_length > sizeof (state->ts_hca_name)) {
3083 cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n",
3084 name_length);
3085 goto out;
3086 }
3087 (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3],
3088 name_length);
3089 state->ts_hca_name[name_length] = 0;
3090
3091 /* get the part number */
3092 vpd_ro_desc = name_length + 3; /* read-only tag location */
3093 vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */
3094 /*
3095 * Verify read-only tag and Part Number keyword.
3096 */
3097 if (vpd.vpd_char[vpd_ro_desc] != 0x90 ||
3098 (vpd.vpd_char[vpd_ro_pn_desc] != 'P' &&
3099 vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) {
3100 cmn_err(CE_NOTE, "!VPD Part Number not found\n");
3101 goto out;
3102 }
3103
3104 pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2];
3105 if (pn_length > sizeof (state->ts_hca_pn)) {
3106 cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n",
3107 name_length);
3108 goto out;
3109 }
3110 (void) memcpy(state->ts_hca_pn,
3111 &vpd.vpd_char[vpd_ro_pn_desc + 3],
3112 pn_length);
3113 state->ts_hca_pn[pn_length] = 0;
3114 state->ts_hca_pn_len = pn_length;
3115 } else {
3116 /* Wrong VPD String ID Tag */
3117 cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n",
3118 vpd.vpd_char[0]);
3119 goto out;
3120 }
3121 TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3122 return;
3123 out:
3124 state->ts_hca_pn_len = 0;
3125 TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, "");
3126 TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3127 }
3128
3129 /*
3130 * tavor_pci_capability_pcix()
3131 * Context: Only called from attach() path context
3132 */
3133 static void
3134 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl,
3135 uint_t offset)
3136 {
3137 uint_t command, status;
3138 int max_out_splt_trans, max_mem_rd_byte_cnt;
3139 int designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt;
3140
3141 TAVOR_TNF_ENTER(tavor_pci_capability_pcix);
3142
3143 /*
3144 * Query the current values for the PCI-X Command Register and
3145 * the PCI-X Status Register.
3146 */
3147 command = pci_config_get16(hdl, offset + 2);
3148 status = pci_config_get32(hdl, offset + 4);
3149
3150 /*
3151 * Check for config property specifying "maximum outstanding
3152 * split transactions". If the property is defined and valid
3153 * (i.e. no larger than the so-called "designed maximum"),
3154 * then use the specified value to update the PCI-X Command Register.
3155 * Otherwise, extract the value from the Tavor config profile.
3156 */
3157 designed_max_out_splt_trans = ((status >> 23) & 7);
3158 max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3159 DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1);
3160 if ((max_out_splt_trans != -1) &&
3161 ((max_out_splt_trans < 0) ||
3162 (max_out_splt_trans > designed_max_out_splt_trans))) {
3163 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-"
3164 "split-trans\" (%d) invalid or exceeds device maximum"
3165 " (%d), using default value (%d)\n", state->ts_instance,
3166 max_out_splt_trans, designed_max_out_splt_trans,
3167 state->ts_cfg_profile->cp_max_out_splt_trans);
3168 max_out_splt_trans =
3169 state->ts_cfg_profile->cp_max_out_splt_trans;
3170 } else if (max_out_splt_trans == -1) {
3171 max_out_splt_trans =
3172 state->ts_cfg_profile->cp_max_out_splt_trans;
3173 }
3174
3175 /*
3176 * The config profile setting for max_out_splt_trans is determined
3177 * based on arch. Check tavor_cfg.c for more information. A value of
3178 * '-1' in the patchable variable means "do not change". A value of
3179 * '0' means 1 outstanding splt trans and other values as defined by
3180 * PCI. So we do one more check here, that if 'max_out_splt_trans' is
3181 * -1 (ie: < 0) we do not set the PCI command and leave it at the
3182 * default.
3183 */
3184 if (max_out_splt_trans >= 0) {
3185 command = ((command & 0xFF8F) | max_out_splt_trans << 4);
3186 }
3187
3188 /*
3189 * Check for config property specifying "maximum memory read
3190 * byte count. If the property is defined and valid
3191 * (i.e. no larger than the so-called "designed maximum"),
3192 * then use the specified value to update the PCI-X Command Register.
3193 * Otherwise, extract the value from the Tavor config profile.
3194 */
3195 designed_max_mem_rd_byte_cnt = ((status >> 21) & 3);
3196 max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3197 DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1);
3198 if ((max_mem_rd_byte_cnt != -1) &&
3199 ((max_mem_rd_byte_cnt < 0) ||
3200 (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) {
3201 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-"
3202 "count\" (%d) invalid or exceeds device maximum"
3203 " (%d), using default value (%d)\n", state->ts_instance,
3204 max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt,
3205 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt);
3206 max_mem_rd_byte_cnt =
3207 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3208 } else if (max_mem_rd_byte_cnt == -1) {
3209 max_mem_rd_byte_cnt =
3210 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3211 }
3212
3213 /*
3214 * The config profile setting for max_mem_rd_byte_cnt is determined
3215 * based on arch. Check tavor_cfg.c for more information. A value of
3216 * '-1' in the patchable variable means "do not change". A value of
3217 * '0' means minimum (512B) read, and other values as defined by
3218 * PCI. So we do one more check here, that if 'max_mem_rd_byte_cnt' is
3219 * -1 (ie: < 0) we do not set the PCI command and leave it at the
3220 * default.
3221 */
3222 if (max_mem_rd_byte_cnt >= 0) {
3223 command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2);
3224 }
3225
3226 /*
3227 * Update the PCI-X Command Register with the newly configured
3228 * values.
3229 */
3230 pci_config_put16(hdl, offset + 2, command);
3231
3232 TAVOR_TNF_EXIT(tavor_pci_capability_pcix);
3233 }
3234
3235
3236 /*
3237 * tavor_intr_or_msi_init()
3238 * Context: Only called from attach() path context
3239 */
3240 static int
3241 tavor_intr_or_msi_init(tavor_state_t *state)
3242 {
3243 int status;
3244
3245 TAVOR_TNF_ENTER(tavor_intr_or_msi_init);
3246
3247 /* Query for the list of supported interrupt event types */
3248 status = ddi_intr_get_supported_types(state->ts_dip,
3249 &state->ts_intr_types_avail);
3250 if (status != DDI_SUCCESS) {
3251 TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail,
3252 TAVOR_TNF_ERROR, "");
3253 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3254 return (DDI_FAILURE);
3255 }
3256
3257 /*
3258 * If Tavor/Arbel supports MSI in this system (and, if it
3259 * hasn't been overridden by a configuration variable), then
3260 * the default behavior is to use a single MSI. Otherwise,
3261 * fallback to using legacy interrupts. Also, if MSI allocatis chosen,
3262 * but fails for whatever reasons, then fallback to using legacy
3263 * interrupts.
3264 */
3265 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3266 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3267 status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI);
3268 if (status == DDI_SUCCESS) {
3269 state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI;
3270 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3271 return (DDI_SUCCESS);
3272 }
3273 }
3274
3275 /*
3276 * MSI interrupt allocation failed, or was not available. Fallback to
3277 * legacy interrupt support.
3278 */
3279 if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) {
3280 status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED);
3281 if (status == DDI_SUCCESS) {
3282 state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED;
3283 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3284 return (DDI_SUCCESS);
3285 }
3286 }
3287
3288 /*
3289 * Neither MSI or legacy interrupts were successful. return failure.
3290 */
3291 TAVOR_TNF_EXIT(tavor_intr_or_msi_setup);
3292 return (DDI_FAILURE);
3293 }
3294
3295 /*
3296 * tavor_add_intrs()
3297 * Context: Only called from attach() patch context
3298 */
3299 static int
3300 tavor_add_intrs(tavor_state_t *state, int intr_type)
3301 {
3302 int status;
3303
3304 TAVOR_TNF_ENTER(tavor_add_intrs);
3305
3306 /* Get number of interrupts/MSI supported */
3307 status = ddi_intr_get_nintrs(state->ts_dip, intr_type,
3308 &state->ts_intrmsi_count);
3309 if (status != DDI_SUCCESS) {
3310 TNF_PROBE_0(tavor_add_intrs_getnintrs_fail,
3311 TAVOR_TNF_ERROR, "");
3312 TAVOR_TNF_EXIT(tavor_add_intrs);
3313 return (DDI_FAILURE);
3314 }
3315
3316 /* Get number of available interrupts/MSI */
3317 status = ddi_intr_get_navail(state->ts_dip, intr_type,
3318 &state->ts_intrmsi_avail);
3319 if (status != DDI_SUCCESS) {
3320 TNF_PROBE_0(tavor_add_intrs_getnavail_fail,
3321 TAVOR_TNF_ERROR, "");
3322 TAVOR_TNF_EXIT(tavor_add_intrs);
3323 return (DDI_FAILURE);
3324 }
3325
3326 /* Ensure that we have at least one (1) usable MSI or interrupt */
3327 if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) {
3328 TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail,
3329 TAVOR_TNF_ERROR, "");
3330 TAVOR_TNF_EXIT(tavor_add_intrs);
3331 return (DDI_FAILURE);
3332 }
3333
3334 /* Attempt to allocate a single interrupt/MSI handle */
3335 status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl,
3336 intr_type, 0, 1, &state->ts_intrmsi_allocd,
3337 DDI_INTR_ALLOC_STRICT);
3338 if (status != DDI_SUCCESS) {
3339 TNF_PROBE_0(tavor_add_intrs_intralloc_fail,
3340 TAVOR_TNF_ERROR, "");
3341 TAVOR_TNF_EXIT(tavor_add_intrs);
3342 return (DDI_FAILURE);
3343 }
3344
3345 /* Ensure that we have allocated at least one (1) MSI or interrupt */
3346 if (state->ts_intrmsi_allocd < 1) {
3347 TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail,
3348 TAVOR_TNF_ERROR, "");
3349 TAVOR_TNF_EXIT(tavor_add_intrs);
3350 return (DDI_FAILURE);
3351 }
3352
3353 /*
3354 * Extract the priority for the allocated interrupt/MSI. This
3355 * will be used later when initializing certain mutexes.
3356 */
3357 status = ddi_intr_get_pri(state->ts_intrmsi_hdl,
3358 &state->ts_intrmsi_pri);
3359 if (status != DDI_SUCCESS) {
3360 /* Free the allocated interrupt/MSI handle */
3361 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3362
3363 TNF_PROBE_0(tavor_add_intrs_getpri_fail,
3364 TAVOR_TNF_ERROR, "");
3365 TAVOR_TNF_EXIT(tavor_add_intrs);
3366 return (DDI_FAILURE);
3367 }
3368
3369 /* Make sure the interrupt/MSI priority is below 'high level' */
3370 if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) {
3371 /* Free the allocated interrupt/MSI handle */
3372 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3373
3374 TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail,
3375 TAVOR_TNF_ERROR, "");
3376 TAVOR_TNF_EXIT(tavor_add_intrs);
3377 return (DDI_FAILURE);
3378 }
3379
3380 /* Get add'l capability information regarding interrupt/MSI */
3381 status = ddi_intr_get_cap(state->ts_intrmsi_hdl,
3382 &state->ts_intrmsi_cap);
3383 if (status != DDI_SUCCESS) {
3384 /* Free the allocated interrupt/MSI handle */
3385 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3386
3387 TNF_PROBE_0(tavor_add_intrs_getcap_fail,
3388 TAVOR_TNF_ERROR, "");
3389 TAVOR_TNF_EXIT(tavor_add_intrs);
3390 return (DDI_FAILURE);
3391 }
3392
3393 TAVOR_TNF_EXIT(tavor_add_intrs);
3394 return (DDI_SUCCESS);
3395 }
3396
3397
3398 /*
3399 * tavor_intr_or_msi_fini()
3400 * Context: Only called from attach() and/or detach() path contexts
3401 */
3402 static int
3403 tavor_intr_or_msi_fini(tavor_state_t *state)
3404 {
3405 int status;
3406
3407 TAVOR_TNF_ENTER(tavor_intr_or_msi_fini);
3408
3409 /* Free the allocated interrupt/MSI handle */
3410 status = ddi_intr_free(state->ts_intrmsi_hdl);
3411 if (status != DDI_SUCCESS) {
3412 TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail,
3413 TAVOR_TNF_ERROR, "");
3414 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3415 return (DDI_FAILURE);
3416 }
3417
3418 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3419 return (DDI_SUCCESS);
3420 }
3421
3422
3423 /* Disable Tavor interrupts */
3424 static int
3425 tavor_intr_disable(tavor_state_t *state)
3426 {
3427 ushort_t msi_ctrl = 0, caps_ctrl = 0;
3428 ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl;
3429 ASSERT(pci_cfg_hdl != NULL);
3430 ASSERT(state->ts_intr_types_avail &
3431 (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI));
3432
3433 /*
3434 * Check if MSI interrupts are used. If so, disable MSI interupts.
3435 * If not, since Tavor doesn't support MSI-X interrupts, assuming the
3436 * legacy interrupt is used instead, disable the legacy interrupt.
3437 */
3438 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3439 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3440
3441 if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI,
3442 &caps_ctrl) == DDI_SUCCESS)) {
3443 if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL,
3444 caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16)
3445 return (DDI_FAILURE);
3446 }
3447 ASSERT(msi_ctrl != 0);
3448
3449 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
3450 return (DDI_SUCCESS);
3451
3452 if (msi_ctrl & PCI_MSI_PVM_MASK) {
3453 int offset = (msi_ctrl & PCI_MSI_64BIT_MASK) ?
3454 PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK;
3455
3456 /* Clear all inums in MSI */
3457 PCI_CAP_PUT32(pci_cfg_hdl, NULL, caps_ctrl,
3458 offset, 0x0);
3459 }
3460
3461 /* Disable MSI interrupts */
3462 msi_ctrl &= ~PCI_MSI_ENABLE_BIT;
3463 PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSI_CTRL,
3464 msi_ctrl);
3465
3466 } else {
3467 uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM);
3468 ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED);
3469
3470 /* Disable the legacy interrupts */
3471 cmdreg |= PCI_COMM_INTX_DISABLE;
3472 pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg);
3473 }
3474
3475 return (DDI_SUCCESS);
3476 }
3477
3478 /* Tavor quiesce(9F) entry */
3479 static int
3480 tavor_quiesce(dev_info_t *dip)
3481 {
3482 tavor_state_t *state = ddi_get_soft_state(tavor_statep,
3483 DEVI(dip)->devi_instance);
3484 ASSERT(state != NULL);
3485
3486 /* start fastreboot */
3487 state->ts_quiescing = B_TRUE;
3488
3489 /* If it's in maintenance mode, do nothing but return with SUCCESS */
3490 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
3491 return (DDI_SUCCESS);
3492 }
3493
3494 /* Shutdown HCA ports */
3495 if (tavor_hca_ports_shutdown(state,
3496 state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) {
3497 state->ts_quiescing = B_FALSE;
3498 return (DDI_FAILURE);
3499 }
3500
3501 /* Close HCA */
3502 if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3503 TAVOR_CMD_SUCCESS) {
3504 state->ts_quiescing = B_FALSE;
3505 return (DDI_FAILURE);
3506 }
3507
3508 /* Shutdown FW */
3509 if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3510 TAVOR_CMD_SUCCESS) {
3511 state->ts_quiescing = B_FALSE;
3512 return (DDI_FAILURE);
3513 }
3514
3515 /* Disable interrupts */
3516 if (tavor_intr_disable(state) != DDI_SUCCESS) {
3517 state->ts_quiescing = B_FALSE;
3518 return (DDI_FAILURE);
3519 }
3520
3521 /* SW-reset */
3522 if (tavor_sw_reset(state) != DDI_SUCCESS) {
3523 state->ts_quiescing = B_FALSE;
3524 return (DDI_FAILURE);
3525 }
3526
3527 return (DDI_SUCCESS);
3528 }