1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * tavor.c
28 * Tavor (InfiniBand) HCA Driver attach/detach Routines
29 *
30 * Implements all the routines necessary for the attach, setup,
31 * initialization (and subsequent possible teardown and detach) of the
32 * Tavor InfiniBand HCA driver.
33 */
34
35 #include <sys/types.h>
36 #include <sys/file.h>
37 #include <sys/open.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/stat.h>
43 #include <sys/pci.h>
44 #include <sys/pci_cap.h>
45 #include <sys/bitmap.h>
46 #include <sys/policy.h>
47
48 #include <sys/ib/adapters/tavor/tavor.h>
49 #include <sys/pci.h>
50
51 /* Tavor HCA State Pointer */
52 void *tavor_statep;
53
54 /*
55 * The Tavor "userland resource database" is common to instances of the
56 * Tavor HCA driver. This structure "tavor_userland_rsrc_db" contains all
57 * the necessary information to maintain it.
58 */
59 tavor_umap_db_t tavor_userland_rsrc_db;
60
61 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t);
62 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t);
63 static int tavor_open(dev_t *, int, int, cred_t *);
64 static int tavor_close(dev_t, int, int, cred_t *);
65 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
66 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance);
67 static void tavor_drv_fini(tavor_state_t *state);
68 static void tavor_drv_fini2(tavor_state_t *state);
69 static int tavor_isr_init(tavor_state_t *state);
70 static void tavor_isr_fini(tavor_state_t *state);
71 static int tavor_hw_init(tavor_state_t *state);
72 static void tavor_hw_fini(tavor_state_t *state,
73 tavor_drv_cleanup_level_t cleanup);
74 static int tavor_soft_state_init(tavor_state_t *state);
75 static void tavor_soft_state_fini(tavor_state_t *state);
76 static int tavor_hca_port_init(tavor_state_t *state);
77 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init);
78 static void tavor_hca_config_setup(tavor_state_t *state,
79 tavor_hw_initqueryhca_t *inithca);
80 static int tavor_internal_uarpgs_init(tavor_state_t *state);
81 static void tavor_internal_uarpgs_fini(tavor_state_t *state);
82 static int tavor_special_qp_contexts_reserve(tavor_state_t *state);
83 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state);
84 static int tavor_sw_reset(tavor_state_t *state);
85 static int tavor_mcg_init(tavor_state_t *state);
86 static void tavor_mcg_fini(tavor_state_t *state);
87 static int tavor_fw_version_check(tavor_state_t *state);
88 static void tavor_device_info_report(tavor_state_t *state);
89 static void tavor_pci_capability_list(tavor_state_t *state,
90 ddi_acc_handle_t hdl);
91 static void tavor_pci_capability_vpd(tavor_state_t *state,
92 ddi_acc_handle_t hdl, uint_t offset);
93 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset,
94 uint32_t addr, uint32_t *data);
95 static void tavor_pci_capability_pcix(tavor_state_t *state,
96 ddi_acc_handle_t hdl, uint_t offset);
97 static int tavor_intr_or_msi_init(tavor_state_t *state);
98 static int tavor_add_intrs(tavor_state_t *state, int intr_type);
99 static int tavor_intr_or_msi_fini(tavor_state_t *state);
100
101 /* X86 fastreboot support */
102 static int tavor_intr_disable(tavor_state_t *);
103 static int tavor_quiesce(dev_info_t *);
104
105 /* Character/Block Operations */
106 static struct cb_ops tavor_cb_ops = {
107 tavor_open, /* open */
108 tavor_close, /* close */
109 nodev, /* strategy (block) */
110 nodev, /* print (block) */
111 nodev, /* dump (block) */
112 nodev, /* read */
113 nodev, /* write */
114 tavor_ioctl, /* ioctl */
115 tavor_devmap, /* devmap */
116 NULL, /* mmap */
117 nodev, /* segmap */
118 nochpoll, /* chpoll */
119 ddi_prop_op, /* prop_op */
120 NULL, /* streams */
121 D_NEW | D_MP |
122 D_64BIT | D_HOTPLUG |
123 D_DEVMAP, /* flags */
124 CB_REV /* rev */
125 };
126
127 /* Driver Operations */
128 static struct dev_ops tavor_ops = {
129 DEVO_REV, /* struct rev */
130 0, /* refcnt */
131 tavor_getinfo, /* getinfo */
132 nulldev, /* identify */
133 nulldev, /* probe */
134 tavor_attach, /* attach */
135 tavor_detach, /* detach */
136 nodev, /* reset */
137 &tavor_cb_ops, /* cb_ops */
138 NULL, /* bus_ops */
139 nodev, /* power */
140 tavor_quiesce, /* devo_quiesce */
141 };
142
143 /* Module Driver Info */
144 static struct modldrv tavor_modldrv = {
145 &mod_driverops,
146 "Tavor InfiniBand HCA Driver",
147 &tavor_ops
148 };
149
150 /* Module Linkage */
151 static struct modlinkage tavor_modlinkage = {
152 MODREV_1,
153 &tavor_modldrv,
154 NULL
155 };
156
157 /*
158 * This extern refers to the ibc_operations_t function vector that is defined
159 * in the tavor_ci.c file.
160 */
161 extern ibc_operations_t tavor_ibc_ops;
162
163 #ifndef NPROBE
164 extern int tnf_mod_load(void);
165 extern int tnf_mod_unload(struct modlinkage *mlp);
166 #endif
167
168
169 /*
170 * _init()
171 */
172 int
173 _init()
174 {
175 int status;
176
177 #ifndef NPROBE
178 (void) tnf_mod_load();
179 #endif
180 TAVOR_TNF_ENTER(tavor_init);
181
182 status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t),
183 (size_t)TAVOR_INITIAL_STATES);
184 if (status != 0) {
185 TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, "");
186 TAVOR_TNF_EXIT(tavor_init);
187 #ifndef NPROBE
188 (void) tnf_mod_unload(&tavor_modlinkage);
189 #endif
190 return (status);
191 }
192
193 status = ibc_init(&tavor_modlinkage);
194 if (status != 0) {
195 TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, "");
196 ddi_soft_state_fini(&tavor_statep);
197 TAVOR_TNF_EXIT(tavor_init);
198 #ifndef NPROBE
199 (void) tnf_mod_unload(&tavor_modlinkage);
200 #endif
201 return (status);
202 }
203 status = mod_install(&tavor_modlinkage);
204 if (status != 0) {
205 TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, "");
206 ibc_fini(&tavor_modlinkage);
207 ddi_soft_state_fini(&tavor_statep);
208 TAVOR_TNF_EXIT(tavor_init);
209 #ifndef NPROBE
210 (void) tnf_mod_unload(&tavor_modlinkage);
211 #endif
212 return (status);
213 }
214
215 /* Initialize the Tavor "userland resources database" */
216 tavor_umap_db_init();
217
218 TAVOR_TNF_EXIT(tavor_init);
219 return (status);
220 }
221
222
223 /*
224 * _info()
225 */
226 int
227 _info(struct modinfo *modinfop)
228 {
229 int status;
230
231 TAVOR_TNF_ENTER(tavor_info);
232 status = mod_info(&tavor_modlinkage, modinfop);
233 TAVOR_TNF_EXIT(tavor_info);
234 return (status);
235 }
236
237
238 /*
239 * _fini()
240 */
241 int
242 _fini()
243 {
244 int status;
245
246 TAVOR_TNF_ENTER(tavor_fini);
247
248 status = mod_remove(&tavor_modlinkage);
249 if (status != 0) {
250 TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, "");
251 TAVOR_TNF_EXIT(tavor_fini);
252 return (status);
253 }
254
255 /* Destroy the Tavor "userland resources database" */
256 tavor_umap_db_fini();
257
258 ibc_fini(&tavor_modlinkage);
259 ddi_soft_state_fini(&tavor_statep);
260 #ifndef NPROBE
261 (void) tnf_mod_unload(&tavor_modlinkage);
262 #endif
263 TAVOR_TNF_EXIT(tavor_fini);
264 return (status);
265 }
266
267
268 /*
269 * tavor_getinfo()
270 */
271 /* ARGSUSED */
272 static int
273 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
274 {
275 dev_t dev;
276 tavor_state_t *state;
277 minor_t instance;
278
279 TAVOR_TNF_ENTER(tavor_getinfo);
280
281 switch (cmd) {
282 case DDI_INFO_DEVT2DEVINFO:
283 dev = (dev_t)arg;
284 instance = TAVOR_DEV_INSTANCE(dev);
285 state = ddi_get_soft_state(tavor_statep, instance);
286 if (state == NULL) {
287 TNF_PROBE_0(tavor_getinfo_gss_fail,
288 TAVOR_TNF_ERROR, "");
289 TAVOR_TNF_EXIT(tavor_getinfo);
290 return (DDI_FAILURE);
291 }
292 *result = (void *)state->ts_dip;
293 return (DDI_SUCCESS);
294
295 case DDI_INFO_DEVT2INSTANCE:
296 dev = (dev_t)arg;
297 instance = TAVOR_DEV_INSTANCE(dev);
298 *result = (void *)(uintptr_t)instance;
299 return (DDI_SUCCESS);
300
301 default:
302 TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, "");
303 break;
304 }
305
306 TAVOR_TNF_EXIT(tavor_getinfo);
307 return (DDI_FAILURE);
308 }
309
310
311 /*
312 * tavor_open()
313 */
314 /* ARGSUSED */
315 static int
316 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp)
317 {
318 tavor_state_t *state;
319 tavor_rsrc_t *rsrcp;
320 tavor_umap_db_entry_t *umapdb, *umapdb2;
321 minor_t instance;
322 uint64_t key, value;
323 uint_t tr_indx;
324 dev_t dev;
325 int status;
326
327 TAVOR_TNF_ENTER(tavor_open);
328
329 instance = TAVOR_DEV_INSTANCE(*devp);
330 state = ddi_get_soft_state(tavor_statep, instance);
331 if (state == NULL) {
332 TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, "");
333 TAVOR_TNF_EXIT(tavor_open);
334 return (ENXIO);
335 }
336
337 /*
338 * Only allow driver to be opened for character access, and verify
339 * whether exclusive access is allowed.
340 */
341 if ((otyp != OTYP_CHR) || ((flag & FEXCL) &&
342 secpolicy_excl_open(credp) != 0)) {
343 TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, "");
344 TAVOR_TNF_EXIT(tavor_open);
345 return (EINVAL);
346 }
347
348 /*
349 * Search for the current process PID in the "userland resources
350 * database". If it is not found, then attempt to allocate a UAR
351 * page and add the ("key", "value") pair to the database.
352 * Note: As a last step we always return a devp appropriate for
353 * the open. Either we return a new minor number (based on the
354 * instance and the UAR page index) or we return the current minor
355 * number for the given client process.
356 *
357 * We also add an entry to the database to allow for lookup from
358 * "dev_t" to the current process PID. This is necessary because,
359 * under certain circumstance, the process PID that calls the Tavor
360 * close() entry point may not be the same as the one who called
361 * open(). Specifically, this can happen if a child process calls
362 * the Tavor's open() entry point, gets a UAR page, maps it out (using
363 * mmap()), and then exits without calling munmap(). Because mmap()
364 * adds a reference to the file descriptor, at the exit of the child
365 * process the file descriptor is "inherited" by the parent (and will
366 * be close()'d by the parent's PID only when it exits).
367 *
368 * Note: We use the tavor_umap_db_find_nolock() and
369 * tavor_umap_db_add_nolock() database access routines below (with
370 * an explicit mutex_enter of the database lock - "tdl_umapdb_lock")
371 * to ensure that the multiple accesses (in this case searching for,
372 * and then adding _two_ database entries) can be done atomically.
373 */
374 key = ddi_get_pid();
375 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
376 status = tavor_umap_db_find_nolock(instance, key,
377 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
378 if (status != DDI_SUCCESS) {
379 /*
380 * If we are in 'maintenance mode', we cannot alloc a UAR page.
381 * But we still need some rsrcp value, and a mostly unique
382 * tr_indx value. So we set rsrcp to NULL for maintenance
383 * mode, and use a rolling count for tr_indx. The field
384 * 'ts_open_tr_indx' is used only in this maintenance mode
385 * condition.
386 *
387 * Otherwise, if we are in operational mode then we allocate
388 * the UAR page as normal, and use the rsrcp value and tr_indx
389 * value from that allocation.
390 */
391 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
392 rsrcp = NULL;
393 tr_indx = state->ts_open_tr_indx++;
394 } else {
395 /* Allocate a new UAR page for this process */
396 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1,
397 TAVOR_NOSLEEP, &rsrcp);
398 if (status != DDI_SUCCESS) {
399 mutex_exit(
400 &tavor_userland_rsrc_db.tdl_umapdb_lock);
401 TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail,
402 TAVOR_TNF_ERROR, "");
403 TAVOR_TNF_EXIT(tavor_open);
404 return (EAGAIN);
405 }
406
407 tr_indx = rsrcp->tr_indx;
408 }
409
410 /*
411 * Allocate an entry to track the UAR page resource in the
412 * "userland resources database".
413 */
414 umapdb = tavor_umap_db_alloc(instance, key,
415 MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp);
416 if (umapdb == NULL) {
417 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
418 /* If in "maintenance mode", don't free the rsrc */
419 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
420 tavor_rsrc_free(state, &rsrcp);
421 }
422 TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
423 TAVOR_TNF_ERROR, "");
424 TAVOR_TNF_EXIT(tavor_open);
425 return (EAGAIN);
426 }
427
428 /*
429 * Create a new device number. Minor number is a function of
430 * the UAR page index (15 bits) and the device instance number
431 * (3 bits).
432 */
433 dev = makedevice(getmajor(*devp), (tr_indx <<
434 TAVOR_MINORNUM_SHIFT) | instance);
435
436 /*
437 * Allocate another entry in the "userland resources database"
438 * to track the association of the device number (above) to
439 * the current process ID (in "key").
440 */
441 umapdb2 = tavor_umap_db_alloc(instance, dev,
442 MLNX_UMAP_PID_RSRC, (uint64_t)key);
443 if (umapdb2 == NULL) {
444 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
445 tavor_umap_db_free(umapdb);
446 /* If in "maintenance mode", don't free the rsrc */
447 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
448 tavor_rsrc_free(state, &rsrcp);
449 }
450 TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
451 TAVOR_TNF_ERROR, "");
452 TAVOR_TNF_EXIT(tavor_open);
453 return (EAGAIN);
454 }
455
456 /* Add the entries to the database */
457 tavor_umap_db_add_nolock(umapdb);
458 tavor_umap_db_add_nolock(umapdb2);
459
460 } else {
461 /*
462 * Return the same device number as on the original open()
463 * call. This was calculated as a function of the UAR page
464 * index (top 16 bits) and the device instance number
465 */
466 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
467 dev = makedevice(getmajor(*devp), (rsrcp->tr_indx <<
468 TAVOR_MINORNUM_SHIFT) | instance);
469 }
470 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
471
472 *devp = dev;
473
474 TAVOR_TNF_EXIT(tavor_open);
475 return (0);
476 }
477
478
479 /*
480 * tavor_close()
481 */
482 /* ARGSUSED */
483 static int
484 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp)
485 {
486 tavor_state_t *state;
487 tavor_rsrc_t *rsrcp;
488 tavor_umap_db_entry_t *umapdb;
489 tavor_umap_db_priv_t *priv;
490 minor_t instance;
491 uint64_t key, value;
492 int status;
493
494 TAVOR_TNF_ENTER(tavor_close);
495
496 instance = TAVOR_DEV_INSTANCE(dev);
497 state = ddi_get_soft_state(tavor_statep, instance);
498 if (state == NULL) {
499 TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, "");
500 TAVOR_TNF_EXIT(tavor_close);
501 return (ENXIO);
502 }
503
504 /*
505 * Search for "dev_t" in the "userland resources database". As
506 * explained above in tavor_open(), we can't depend on using the
507 * current process ID here to do the lookup because the process
508 * that ultimately closes may not be the same one who opened
509 * (because of inheritance).
510 * So we lookup the "dev_t" (which points to the PID of the process
511 * that opened), and we remove the entry from the database (and free
512 * it up). Then we do another query based on the PID value. And when
513 * we find that database entry, we free it up too and then free the
514 * Tavor UAR page resource.
515 *
516 * Note: We use the tavor_umap_db_find_nolock() database access
517 * routine below (with an explicit mutex_enter of the database lock)
518 * to ensure that the multiple accesses (which attempt to remove the
519 * two database entries) can be done atomically.
520 *
521 * This works the same in both maintenance mode and HCA mode, except
522 * for the call to tavor_rsrc_free(). In the case of maintenance mode,
523 * this call is not needed, as it was not allocated in tavor_open()
524 * above.
525 */
526 key = dev;
527 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
528 status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC,
529 &value, TAVOR_UMAP_DB_REMOVE, &umapdb);
530 if (status == DDI_SUCCESS) {
531 /*
532 * If the "tdb_priv" field is non-NULL, it indicates that
533 * some "on close" handling is still necessary. Call
534 * tavor_umap_db_handle_onclose_cb() to do the handling (i.e.
535 * to invoke all the registered callbacks). Then free up
536 * the resources associated with "tdb_priv" and continue
537 * closing.
538 */
539 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv;
540 if (priv != NULL) {
541 tavor_umap_db_handle_onclose_cb(priv);
542 kmem_free(priv, sizeof (tavor_umap_db_priv_t));
543 umapdb->tdbe_common.tdb_priv = (void *)NULL;
544 }
545
546 tavor_umap_db_free(umapdb);
547
548 /*
549 * Now do another lookup using PID as the key (copy it from
550 * "value"). When this lookup is complete, the "value" field
551 * will contain the tavor_rsrc_t pointer for the UAR page
552 * resource.
553 */
554 key = value;
555 status = tavor_umap_db_find_nolock(instance, key,
556 MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
557 &umapdb);
558 if (status == DDI_SUCCESS) {
559 tavor_umap_db_free(umapdb);
560 /* If in "maintenance mode", don't free the rsrc */
561 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
562 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
563 tavor_rsrc_free(state, &rsrcp);
564 }
565 }
566 }
567 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
568
569 TAVOR_TNF_EXIT(tavor_close);
570 return (0);
571 }
572
573
574 /*
575 * tavor_attach()
576 * Context: Only called from attach() path context
577 */
578 static int
579 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
580 {
581 tavor_state_t *state;
582 ibc_clnt_hdl_t tmp_ibtfpriv;
583 ibc_status_t ibc_status;
584 int instance;
585 int status;
586
587 TAVOR_TNF_ENTER(tavor_attach);
588
589 switch (cmd) {
590 case DDI_ATTACH:
591 instance = ddi_get_instance(dip);
592 status = ddi_soft_state_zalloc(tavor_statep, instance);
593 if (status != DDI_SUCCESS) {
594 TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, "");
595 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
596 "attach_ssz_fail", instance);
597 goto fail_attach_nomsg;
598
599 }
600 state = ddi_get_soft_state(tavor_statep, instance);
601 if (state == NULL) {
602 ddi_soft_state_free(tavor_statep, instance);
603 TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, "");
604 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
605 "attach_gss_fail", instance);
606 goto fail_attach_nomsg;
607 }
608
609 /* clear the attach error buffer */
610 TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf);
611
612 /*
613 * Initialize Tavor driver and hardware.
614 *
615 * Note: If this initialization fails we may still wish to
616 * create a device node and remain operational so that Tavor
617 * firmware can be updated/flashed (i.e. "maintenance mode").
618 * If this is the case, then "ts_operational_mode" will be
619 * equal to TAVOR_MAINTENANCE_MODE. We will not attempt to
620 * attach to the IBTF or register with the IBMF (i.e. no
621 * InfiniBand interfaces will be enabled).
622 */
623 status = tavor_drv_init(state, dip, instance);
624 if ((status != DDI_SUCCESS) &&
625 (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
626 TNF_PROBE_0(tavor_attach_drvinit_fail,
627 TAVOR_TNF_ERROR, "");
628 goto fail_attach;
629 }
630
631 /* Create the minor node for device */
632 status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance,
633 DDI_PSEUDO, 0);
634 if (status != DDI_SUCCESS) {
635 tavor_drv_fini(state);
636 TAVOR_ATTACH_MSG(state->ts_attach_buf,
637 "attach_create_mn_fail");
638 TNF_PROBE_0(tavor_attach_create_mn_fail,
639 TAVOR_TNF_ERROR, "");
640 goto fail_attach;
641 }
642
643 /*
644 * If we are in "maintenance mode", then we don't want to
645 * register with the IBTF. All InfiniBand interfaces are
646 * uninitialized, and the device is only capable of handling
647 * requests to update/flash firmware (or test/debug requests).
648 */
649 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
650
651 /* Attach to InfiniBand Transport Framework (IBTF) */
652 ibc_status = ibc_attach(&tmp_ibtfpriv,
653 &state->ts_ibtfinfo);
654 if (ibc_status != IBC_SUCCESS) {
655 ddi_remove_minor_node(dip, "devctl");
656 tavor_drv_fini(state);
657 TNF_PROBE_0(tavor_attach_ibcattach_fail,
658 TAVOR_TNF_ERROR, "");
659 TAVOR_ATTACH_MSG(state->ts_attach_buf,
660 "attach_ibcattach_fail");
661 goto fail_attach;
662 }
663
664 /*
665 * Now that we've successfully attached to the IBTF,
666 * we enable all appropriate asynch and CQ events to
667 * be forwarded to the IBTF.
668 */
669 TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv);
670
671 ibc_post_attach(state->ts_ibtfpriv);
672
673 /* Register agents with IB Mgmt Framework (IBMF) */
674 status = tavor_agent_handlers_init(state);
675 if (status != DDI_SUCCESS) {
676 (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH);
677 TAVOR_QUIESCE_IBTF_CALLB(state);
678 if (state->ts_in_evcallb != 0) {
679 TAVOR_WARNING(state, "unable to "
680 "quiesce Tavor IBTF callbacks");
681 }
682 ibc_detach(tmp_ibtfpriv);
683 ddi_remove_minor_node(dip, "devctl");
684 tavor_drv_fini(state);
685 TNF_PROBE_0(tavor_attach_agentinit_fail,
686 TAVOR_TNF_ERROR, "");
687 TAVOR_ATTACH_MSG(state->ts_attach_buf,
688 "attach_agentinit_fail");
689 goto fail_attach;
690 }
691 }
692
693 /* Report that driver was loaded */
694 ddi_report_dev(dip);
695
696 /* Send device information to log file */
697 tavor_device_info_report(state);
698
699 /* Report attach in maintenance mode, if appropriate */
700 if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
701 cmn_err(CE_NOTE, "tavor%d: driver attached "
702 "(for maintenance mode only)", state->ts_instance);
703 }
704
705 TAVOR_TNF_EXIT(tavor_attach);
706 return (DDI_SUCCESS);
707
708 case DDI_RESUME:
709 /* Add code here for DDI_RESUME XXX */
710 TAVOR_TNF_EXIT(tavor_attach);
711 return (DDI_FAILURE);
712
713 default:
714 TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, "");
715 break;
716 }
717
718 fail_attach:
719 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance,
720 state->ts_attach_buf);
721 tavor_drv_fini2(state);
722 ddi_soft_state_free(tavor_statep, instance);
723 fail_attach_nomsg:
724 TAVOR_TNF_EXIT(tavor_attach);
725 return (DDI_FAILURE);
726 }
727
728
729 /*
730 * tavor_detach()
731 * Context: Only called from detach() path context
732 */
733 static int
734 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
735 {
736 tavor_state_t *state;
737 ibc_clnt_hdl_t tmp_ibtfpriv;
738 ibc_status_t ibc_status;
739 int instance, status;
740
741 TAVOR_TNF_ENTER(tavor_detach);
742
743 instance = ddi_get_instance(dip);
744 state = ddi_get_soft_state(tavor_statep, instance);
745 if (state == NULL) {
746 TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, "");
747 TAVOR_TNF_EXIT(tavor_detach);
748 return (DDI_FAILURE);
749 }
750
751 switch (cmd) {
752 case DDI_DETACH:
753 /*
754 * If we are in "maintenance mode", then we do not want to
755 * do teardown for any of the InfiniBand interfaces.
756 * Specifically, this means not detaching from IBTF (we never
757 * attached to begin with) and not deregistering from IBMF.
758 */
759 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
760 /* Unregister agents from IB Mgmt Framework (IBMF) */
761 status = tavor_agent_handlers_fini(state);
762 if (status != DDI_SUCCESS) {
763 TNF_PROBE_0(tavor_detach_agentfini_fail,
764 TAVOR_TNF_ERROR, "");
765 TAVOR_TNF_EXIT(tavor_detach);
766 return (DDI_FAILURE);
767 }
768
769 /*
770 * Attempt the "pre-detach" from InfiniBand Transport
771 * Framework (IBTF). At this point the IBTF is still
772 * capable of handling incoming asynch and completion
773 * events. This "pre-detach" is primarily a mechanism
774 * to notify the appropriate IBTF clients that the
775 * HCA is being removed/offlined.
776 */
777 ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd);
778 if (ibc_status != IBC_SUCCESS) {
779 status = tavor_agent_handlers_init(state);
780 if (status != DDI_SUCCESS) {
781 TAVOR_WARNING(state, "failed to "
782 "restart Tavor agents");
783 }
784 TNF_PROBE_0(tavor_detach_ibcpredetach_fail,
785 TAVOR_TNF_ERROR, "");
786 TAVOR_TNF_EXIT(tavor_detach);
787 return (DDI_FAILURE);
788 }
789
790 /*
791 * Before we can fully detach from the IBTF we need to
792 * ensure that we have handled all outstanding event
793 * callbacks. This is accomplished by quiescing the
794 * event callback mechanism. Note: if we are unable
795 * to successfully quiesce the callbacks, then this is
796 * an indication that something has probably gone
797 * seriously wrong. We print out a warning, but
798 * continue.
799 */
800 tmp_ibtfpriv = state->ts_ibtfpriv;
801 TAVOR_QUIESCE_IBTF_CALLB(state);
802 if (state->ts_in_evcallb != 0) {
803 TAVOR_WARNING(state, "unable to quiesce Tavor "
804 "IBTF callbacks");
805 }
806
807 /* Complete the detach from the IBTF */
808 ibc_detach(tmp_ibtfpriv);
809 }
810
811 /* Remove the minor node for device */
812 ddi_remove_minor_node(dip, "devctl");
813
814 /*
815 * Only call tavor_drv_fini() if we are in Tavor HCA mode.
816 * (Because if we are in "maintenance mode", then we never
817 * successfully finished init.) Only report successful
818 * detach for normal HCA mode.
819 */
820 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
821 /* Cleanup driver resources and shutdown hardware */
822 tavor_drv_fini(state);
823 cmn_err(CE_CONT, "Tavor driver successfully "
824 "detached\n");
825 }
826
827 tavor_drv_fini2(state);
828 ddi_soft_state_free(tavor_statep, instance);
829
830 TAVOR_TNF_EXIT(tavor_detach);
831 return (DDI_SUCCESS);
832
833 case DDI_SUSPEND:
834 /* Add code here for DDI_SUSPEND XXX */
835 TAVOR_TNF_EXIT(tavor_detach);
836 return (DDI_FAILURE);
837
838 default:
839 TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, "");
840 break;
841 }
842
843 TAVOR_TNF_EXIT(tavor_detach);
844 return (DDI_FAILURE);
845 }
846
847
848 /*
849 * tavor_drv_init()
850 * Context: Only called from attach() path context
851 */
852 static int
853 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance)
854 {
855 int status;
856
857 TAVOR_TNF_ENTER(tavor_drv_init);
858
859 /* Save away devinfo and instance */
860 state->ts_dip = dip;
861 state->ts_instance = instance;
862
863 /*
864 * Check and set the operational mode of the device. If the driver is
865 * bound to the Tavor device in "maintenance mode", then this generally
866 * means that either the device has been specifically jumpered to
867 * start in this mode or the firmware boot process has failed to
868 * successfully load either the primary or the secondary firmware
869 * image.
870 */
871 if (TAVOR_IS_HCA_MODE(state->ts_dip)) {
872 state->ts_operational_mode = TAVOR_HCA_MODE;
873
874 } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) {
875 state->ts_operational_mode = TAVOR_COMPAT_MODE;
876
877 } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) {
878 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
879 return (DDI_FAILURE);
880
881 } else {
882 state->ts_operational_mode = 0; /* invalid operational mode */
883 TAVOR_WARNING(state, "unexpected device type detected");
884 TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail,
885 TAVOR_TNF_ERROR, "");
886 TAVOR_TNF_EXIT(tavor_hw_init);
887 return (DDI_FAILURE);
888 }
889
890 /*
891 * Initialize the Tavor hardware.
892 * Note: If this routine returns an error, it is often an reasonably
893 * good indication that something Tavor firmware-related has caused
894 * the failure. In order to give the user an opportunity (if desired)
895 * to update or reflash the Tavor firmware image, we set
896 * "ts_operational_mode" flag (described above) to indicate that we
897 * wish to enter maintenance mode.
898 */
899 status = tavor_hw_init(state);
900 if (status != DDI_SUCCESS) {
901 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
902 cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance,
903 state->ts_attach_buf);
904 TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, "");
905 TAVOR_TNF_EXIT(tavor_drv_init);
906 return (DDI_FAILURE);
907 }
908
909 /* Setup Tavor interrupt handler */
910 status = tavor_isr_init(state);
911 if (status != DDI_SUCCESS) {
912 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
913 TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, "");
914 TAVOR_TNF_EXIT(tavor_drv_init);
915 return (DDI_FAILURE);
916 }
917
918 /* Initialize Tavor softstate */
919 status = tavor_soft_state_init(state);
920 if (status != DDI_SUCCESS) {
921 tavor_isr_fini(state);
922 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
923 TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, "");
924 TAVOR_TNF_EXIT(tavor_drv_init);
925 return (DDI_FAILURE);
926 }
927
928 TAVOR_TNF_EXIT(tavor_drv_init);
929 return (DDI_SUCCESS);
930 }
931
932
933 /*
934 * tavor_drv_fini()
935 * Context: Only called from attach() and/or detach() path contexts
936 */
937 static void
938 tavor_drv_fini(tavor_state_t *state)
939 {
940 TAVOR_TNF_ENTER(tavor_drv_fini);
941
942 /* Cleanup Tavor softstate */
943 tavor_soft_state_fini(state);
944
945 /* Teardown Tavor interrupts */
946 tavor_isr_fini(state);
947
948 /* Cleanup Tavor resources and shutdown hardware */
949 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
950
951 TAVOR_TNF_EXIT(tavor_drv_fini);
952 }
953
954 /*
955 * tavor_drv_fini2()
956 * Context: Only called from attach() and/or detach() path contexts
957 */
958 static void
959 tavor_drv_fini2(tavor_state_t *state)
960 {
961 TAVOR_TNF_ENTER(tavor_drv_fini2);
962
963 /* TAVOR_DRV_CLEANUP_LEVEL1 */
964 if (state->ts_reg_cmdhdl) {
965 ddi_regs_map_free(&state->ts_reg_cmdhdl);
966 state->ts_reg_cmdhdl = NULL;
967 }
968
969 /* TAVOR_DRV_CLEANUP_LEVEL0 */
970 if (state->ts_pci_cfghdl) {
971 pci_config_teardown(&state->ts_pci_cfghdl);
972 state->ts_pci_cfghdl = NULL;
973 }
974
975 TAVOR_TNF_EXIT(tavor_drv_fini2);
976 }
977
978 /*
979 * tavor_isr_init()
980 * Context: Only called from attach() path context
981 */
982 static int
983 tavor_isr_init(tavor_state_t *state)
984 {
985 int status;
986
987 TAVOR_TNF_ENTER(tavor_isr_init);
988
989 /*
990 * Add a handler for the interrupt or MSI
991 */
992 status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr,
993 (caddr_t)state, NULL);
994 if (status != DDI_SUCCESS) {
995 TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, "");
996 TAVOR_TNF_EXIT(tavor_isr_init);
997 return (DDI_FAILURE);
998 }
999
1000 /*
1001 * Enable the software interrupt. Note: Even though we are only
1002 * using one (1) interrupt/MSI, depending on the value returned in
1003 * the capability flag, we have to call either ddi_intr_block_enable()
1004 * or ddi_intr_enable().
1005 */
1006 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1007 status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1);
1008 if (status != DDI_SUCCESS) {
1009 TNF_PROBE_0(tavor_isr_init_blockenable_fail,
1010 TAVOR_TNF_ERROR, "");
1011 TAVOR_TNF_EXIT(tavor_isr_init);
1012 return (DDI_FAILURE);
1013 }
1014 } else {
1015 status = ddi_intr_enable(state->ts_intrmsi_hdl);
1016 if (status != DDI_SUCCESS) {
1017 TNF_PROBE_0(tavor_isr_init_intrenable_fail,
1018 TAVOR_TNF_ERROR, "");
1019 TAVOR_TNF_EXIT(tavor_isr_init);
1020 return (DDI_FAILURE);
1021 }
1022 }
1023
1024 /*
1025 * Now that the ISR has been setup, arm all the EQs for event
1026 * generation.
1027 */
1028 tavor_eq_arm_all(state);
1029
1030 TAVOR_TNF_EXIT(tavor_isr_init);
1031 return (DDI_SUCCESS);
1032 }
1033
1034
1035 /*
1036 * tavor_isr_fini()
1037 * Context: Only called from attach() and/or detach() path contexts
1038 */
1039 static void
1040 tavor_isr_fini(tavor_state_t *state)
1041 {
1042 TAVOR_TNF_ENTER(tavor_isr_fini);
1043
1044 /* Disable the software interrupt */
1045 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1046 (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1);
1047 } else {
1048 (void) ddi_intr_disable(state->ts_intrmsi_hdl);
1049 }
1050
1051 /*
1052 * Remove the software handler for the interrupt or MSI
1053 */
1054 (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl);
1055
1056 TAVOR_TNF_EXIT(tavor_isr_fini);
1057 }
1058
1059
1060 /*
1061 * tavor_fix_error_buf()
1062 * Context: Only called from attach().
1063 *
1064 * The error_buf_addr returned from QUERY_FW is a PCI address.
1065 * We need to convert it to an offset from the base address,
1066 * which is stored in the assigned-addresses property.
1067 */
1068 static int
1069 tavor_fix_error_buf(tavor_state_t *state)
1070 {
1071 int assigned_addr_len;
1072 pci_regspec_t *assigned_addr;
1073
1074 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip,
1075 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr,
1076 (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS)
1077 return (DDI_FAILURE);
1078
1079 state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low +
1080 ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32);
1081 ddi_prop_free(assigned_addr);
1082 return (DDI_SUCCESS);
1083 }
1084
1085 /*
1086 * tavor_hw_init()
1087 * Context: Only called from attach() path context
1088 */
1089 static int
1090 tavor_hw_init(tavor_state_t *state)
1091 {
1092 tavor_drv_cleanup_level_t cleanup;
1093 sm_nodeinfo_t nodeinfo;
1094 uint64_t errorcode;
1095 off_t ddr_size;
1096 int status;
1097 int retries;
1098
1099 TAVOR_TNF_ENTER(tavor_hw_init);
1100
1101 /* This is where driver initialization begins */
1102 cleanup = TAVOR_DRV_CLEANUP_LEVEL0;
1103
1104 /* Setup device access attributes */
1105 state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
1106 state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
1107 state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1108
1109 /* Setup for PCI config read/write of HCA device */
1110 status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl);
1111 if (status != DDI_SUCCESS) {
1112 tavor_hw_fini(state, cleanup);
1113 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1114 "hw_init_PCI_config_space_regmap_fail");
1115 /* This case is not the degraded one */
1116 return (DDI_FAILURE);
1117 }
1118
1119 /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */
1120 status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR,
1121 &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr,
1122 &state->ts_reg_cmdhdl);
1123 if (status != DDI_SUCCESS) {
1124 tavor_hw_fini(state, cleanup);
1125 TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, "");
1126 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1127 "hw_init_CMD_ddirms_fail");
1128 TAVOR_TNF_EXIT(tavor_hw_init);
1129 return (DDI_FAILURE);
1130 }
1131 cleanup = TAVOR_DRV_CLEANUP_LEVEL1;
1132
1133 status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR,
1134 &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr,
1135 &state->ts_reg_uarhdl);
1136 if (status != DDI_SUCCESS) {
1137 tavor_hw_fini(state, cleanup);
1138 TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, "");
1139 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1140 "hw_init_UAR_ddirms_fail");
1141 TAVOR_TNF_EXIT(tavor_hw_init);
1142 return (DDI_FAILURE);
1143 }
1144 cleanup = TAVOR_DRV_CLEANUP_LEVEL2;
1145
1146 status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size);
1147 if (status != DDI_SUCCESS) {
1148 cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed "
1149 "(check HCA-attached DIMM memory?)\n");
1150 tavor_hw_fini(state, cleanup);
1151 TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail,
1152 TAVOR_TNF_ERROR, "");
1153 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1154 "hw_init_DDR_ddi_regsize_fail");
1155 TAVOR_TNF_EXIT(tavor_hw_init);
1156 return (DDI_FAILURE);
1157 }
1158
1159 #if !defined(_ELF64) && !defined(__sparc)
1160 /*
1161 * For 32 bit x86/x64 kernels, where there is limited kernel virtual
1162 * memory available, define a minimal memory footprint. This is
1163 * specified in order to not take up too much resources, thus starving
1164 * out others. Only specified if the HCA DIMM is equal to or greater
1165 * than 256MB.
1166 *
1167 * Note: x86/x64 install and safemode boot are both 32bit.
1168 */
1169 ddr_size = TAVOR_DDR_SIZE_MIN;
1170 #endif /* !(_ELF64) && !(__sparc) */
1171
1172 state->ts_cfg_profile_setting = ddr_size;
1173
1174 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1175 &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr,
1176 &state->ts_reg_ddrhdl);
1177
1178 /*
1179 * On 32-bit platform testing (primarily x86), it was seen that the
1180 * ddi_regs_map_setup() call would fail because there wasn't enough
1181 * kernel virtual address space available to map in the entire 256MB
1182 * DDR. So we add this check in here, so that if the 256 (or other
1183 * larger value of DDR) map in fails, that we fallback to try the lower
1184 * size of 128MB.
1185 *
1186 * Note: If we only have 128MB of DDR in the system in the first place,
1187 * we don't try another ddi_regs_map_setup(), and just skip over this
1188 * check and return failures.
1189 */
1190 if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) {
1191 /* Try falling back to 128MB DDR mapping */
1192 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1193 &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128,
1194 &state->ts_reg_accattr, &state->ts_reg_ddrhdl);
1195
1196 /*
1197 * 128MB DDR mapping worked.
1198 * Set the updated config profile setting here.
1199 */
1200 if (status == DDI_SUCCESS) {
1201 TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success,
1202 TAVOR_TNF_TRACE, "");
1203 state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128;
1204 }
1205 }
1206
1207 if (status != DDI_SUCCESS) {
1208 if (status == DDI_ME_RNUMBER_RANGE) {
1209 cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed "
1210 "(check HCA-attached DIMM memory?)\n");
1211 }
1212 tavor_hw_fini(state, cleanup);
1213 TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, "");
1214 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1215 "hw_init_DDR_ddirms_fail");
1216 TAVOR_TNF_EXIT(tavor_hw_init);
1217 return (DDI_FAILURE);
1218 }
1219 cleanup = TAVOR_DRV_CLEANUP_LEVEL3;
1220
1221 /* Setup Tavor Host Command Register (HCR) */
1222 state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *)
1223 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET);
1224
1225 /* Setup Tavor Event Cause Register (ecr and clr_ecr) */
1226 state->ts_cmd_regs.ecr = (uint64_t *)
1227 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET);
1228 state->ts_cmd_regs.clr_ecr = (uint64_t *)
1229 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET);
1230
1231 /* Setup Tavor Software Reset register (sw_reset) */
1232 state->ts_cmd_regs.sw_reset = (uint32_t *)
1233 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET);
1234
1235 /* Setup Tavor Clear Interrupt register (clr_int) */
1236 state->ts_cmd_regs.clr_int = (uint64_t *)
1237 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET);
1238
1239 /* Initialize the Phase1 Tavor configuration profile */
1240 status = tavor_cfg_profile_init_phase1(state);
1241 if (status != DDI_SUCCESS) {
1242 tavor_hw_fini(state, cleanup);
1243 TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, "");
1244 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail");
1245 TAVOR_TNF_EXIT(tavor_hw_init);
1246 return (DDI_FAILURE);
1247 }
1248 cleanup = TAVOR_DRV_CLEANUP_LEVEL4;
1249
1250 /* Do a software reset of the Tavor HW to ensure proper state */
1251 status = tavor_sw_reset(state);
1252 if (status != TAVOR_CMD_SUCCESS) {
1253 tavor_hw_fini(state, cleanup);
1254 TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, "");
1255 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail");
1256 TAVOR_TNF_EXIT(tavor_hw_init);
1257 return (DDI_FAILURE);
1258 }
1259
1260 /* Post the SYS_EN command to start the hardware */
1261 status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL,
1262 &errorcode, TAVOR_CMD_NOSLEEP_SPIN);
1263 if (status != TAVOR_CMD_SUCCESS) {
1264 if ((status == TAVOR_CMD_BAD_NVMEM) ||
1265 (status == TAVOR_CMD_DDR_MEM_ERR)) {
1266 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1267 "0x%" PRIx64 " (invalid firmware image?)\n",
1268 status, errorcode);
1269 } else {
1270 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1271 "0x%" PRIx64 "\n", status, errorcode);
1272 }
1273 tavor_hw_fini(state, cleanup);
1274 TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail,
1275 TAVOR_TNF_ERROR, "");
1276 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1277 "hw_init_sys_en_cmd_fail");
1278 TAVOR_TNF_EXIT(tavor_hw_init);
1279 return (DDI_FAILURE);
1280 }
1281 cleanup = TAVOR_DRV_CLEANUP_LEVEL5;
1282
1283 /* First phase of init for Tavor configuration/resources */
1284 status = tavor_rsrc_init_phase1(state);
1285 if (status != DDI_SUCCESS) {
1286 tavor_hw_fini(state, cleanup);
1287 TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, "");
1288 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1289 "hw_init_rsrcinit1_fail");
1290 TAVOR_TNF_EXIT(tavor_hw_init);
1291 return (DDI_FAILURE);
1292 }
1293 cleanup = TAVOR_DRV_CLEANUP_LEVEL6;
1294
1295 /* Query the DDR properties (e.g. total DDR size) */
1296 status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0,
1297 &state->ts_ddr, sizeof (tavor_hw_queryddr_t),
1298 TAVOR_CMD_NOSLEEP_SPIN);
1299 if (status != TAVOR_CMD_SUCCESS) {
1300 cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n",
1301 status);
1302 tavor_hw_fini(state, cleanup);
1303 TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail,
1304 TAVOR_TNF_ERROR, "");
1305 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1306 "hw_init_query_ddr_cmd_fail");
1307 TAVOR_TNF_EXIT(tavor_hw_init);
1308 return (DDI_FAILURE);
1309 }
1310
1311 /* Figure out how big the firmware image (in DDR) is */
1312 status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw,
1313 sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN);
1314 if (status != TAVOR_CMD_SUCCESS) {
1315 cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n",
1316 status);
1317 tavor_hw_fini(state, cleanup);
1318 TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail,
1319 TAVOR_TNF_ERROR, "");
1320 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1321 "hw_init_query_fw_cmd_fail");
1322 TAVOR_TNF_EXIT(tavor_hw_init);
1323 return (DDI_FAILURE);
1324 }
1325
1326 if (tavor_fix_error_buf(state) != DDI_SUCCESS) {
1327 tavor_hw_fini(state, cleanup);
1328 TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail,
1329 TAVOR_TNF_ERROR, "");
1330 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1331 "hw_init_fixerrorbuf_fail");
1332 TAVOR_TNF_EXIT(tavor_hw_init);
1333 return (DDI_FAILURE);
1334 }
1335
1336 /* Validate that the FW version is appropriate */
1337 status = tavor_fw_version_check(state);
1338 if (status != DDI_SUCCESS) {
1339 if (state->ts_operational_mode == TAVOR_HCA_MODE) {
1340 cmn_err(CE_CONT, "Unsupported Tavor FW version: "
1341 "expected: %04d.%04d.%04d, "
1342 "actual: %04d.%04d.%04d\n",
1343 TAVOR_FW_VER_MAJOR,
1344 TAVOR_FW_VER_MINOR,
1345 TAVOR_FW_VER_SUBMINOR,
1346 state->ts_fw.fw_rev_major,
1347 state->ts_fw.fw_rev_minor,
1348 state->ts_fw.fw_rev_subminor);
1349 } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) {
1350 cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: "
1351 "expected: %04d.%04d.%04d, "
1352 "actual: %04d.%04d.%04d\n",
1353 TAVOR_COMPAT_FW_VER_MAJOR,
1354 TAVOR_COMPAT_FW_VER_MINOR,
1355 TAVOR_COMPAT_FW_VER_SUBMINOR,
1356 state->ts_fw.fw_rev_major,
1357 state->ts_fw.fw_rev_minor,
1358 state->ts_fw.fw_rev_subminor);
1359 } else {
1360 cmn_err(CE_CONT, "Unsupported FW version: "
1361 "%04d.%04d.%04d\n",
1362 state->ts_fw.fw_rev_major,
1363 state->ts_fw.fw_rev_minor,
1364 state->ts_fw.fw_rev_subminor);
1365 }
1366 tavor_hw_fini(state, cleanup);
1367 TNF_PROBE_0(tavor_hw_init_checkfwver_fail,
1368 TAVOR_TNF_ERROR, "");
1369 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1370 "hw_init_checkfwver_fail");
1371 TAVOR_TNF_EXIT(tavor_hw_init);
1372 return (DDI_FAILURE);
1373 }
1374
1375 drv_usecwait(10);
1376 retries = 1000; /* retry up to 1 second before giving up */
1377 retry:
1378 /* Call MOD_STAT_CFG to setup SRQ support (or disable) */
1379 status = tavor_mod_stat_cfg_cmd_post(state);
1380 if (status != DDI_SUCCESS) {
1381 if (retries > 0) {
1382 drv_usecwait(1000);
1383 retries--;
1384 goto retry;
1385 }
1386 cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n",
1387 status);
1388 tavor_hw_fini(state, cleanup);
1389 TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail,
1390 TAVOR_TNF_ERROR, "");
1391 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1392 "hw_init_mod_stat_cfg_cmd_fail");
1393 TAVOR_TNF_EXIT(tavor_hw_init);
1394 return (DDI_FAILURE);
1395 }
1396
1397 /* Figure out Tavor device limits */
1398 status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0,
1399 &state->ts_devlim, sizeof (tavor_hw_querydevlim_t),
1400 TAVOR_CMD_NOSLEEP_SPIN);
1401 if (status != TAVOR_CMD_SUCCESS) {
1402 cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n",
1403 status);
1404 tavor_hw_fini(state, cleanup);
1405 TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail,
1406 TAVOR_TNF_ERROR, "");
1407 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1408 "hw_init_query_devlim_cmd_fail");
1409 TAVOR_TNF_EXIT(tavor_hw_init);
1410 return (DDI_FAILURE);
1411 }
1412
1413 /* Initialize the Phase2 Tavor configuration profile */
1414 status = tavor_cfg_profile_init_phase2(state);
1415 if (status != DDI_SUCCESS) {
1416 tavor_hw_fini(state, cleanup);
1417 TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, "");
1418 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail");
1419 TAVOR_TNF_EXIT(tavor_hw_init);
1420 return (DDI_FAILURE);
1421 }
1422
1423 /* Second phase of init for Tavor configuration/resources */
1424 status = tavor_rsrc_init_phase2(state);
1425 if (status != DDI_SUCCESS) {
1426 tavor_hw_fini(state, cleanup);
1427 TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, "");
1428 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1429 "hw_init_rsrcinit2_fail");
1430 TAVOR_TNF_EXIT(tavor_hw_init);
1431 return (DDI_FAILURE);
1432 }
1433 cleanup = TAVOR_DRV_CLEANUP_LEVEL7;
1434
1435 /* Miscellaneous query information */
1436 status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0,
1437 &state->ts_adapter, sizeof (tavor_hw_queryadapter_t),
1438 TAVOR_CMD_NOSLEEP_SPIN);
1439 if (status != TAVOR_CMD_SUCCESS) {
1440 cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n",
1441 status);
1442 tavor_hw_fini(state, cleanup);
1443 TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail,
1444 TAVOR_TNF_ERROR, "");
1445 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1446 "hw_init_query_adapter_cmd_fail");
1447 TAVOR_TNF_EXIT(tavor_hw_init);
1448 return (DDI_FAILURE);
1449 }
1450
1451 /* Prepare configuration for Tavor INIT_HCA command */
1452 tavor_hca_config_setup(state, &state->ts_hcaparams);
1453
1454 /* Post command to init Tavor HCA */
1455 status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams,
1456 TAVOR_CMD_NOSLEEP_SPIN);
1457 if (status != TAVOR_CMD_SUCCESS) {
1458 cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n",
1459 status);
1460 tavor_hw_fini(state, cleanup);
1461 TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail,
1462 TAVOR_TNF_ERROR, "");
1463 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1464 "hw_init_init_hca_cmd_fail");
1465 TAVOR_TNF_EXIT(tavor_hw_init);
1466 return (DDI_FAILURE);
1467 }
1468 cleanup = TAVOR_DRV_CLEANUP_LEVEL8;
1469
1470 /* Allocate protection domain (PD) for Tavor internal use */
1471 status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP);
1472 if (status != DDI_SUCCESS) {
1473 tavor_hw_fini(state, cleanup);
1474 TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail,
1475 TAVOR_TNF_ERROR, "");
1476 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1477 "hw_init_internal_pd_alloc_fail");
1478 TAVOR_TNF_EXIT(tavor_hw_init);
1479 return (DDI_FAILURE);
1480 }
1481 cleanup = TAVOR_DRV_CLEANUP_LEVEL9;
1482
1483 /* Setup Tavor internal UAR pages (0 and 1) */
1484 status = tavor_internal_uarpgs_init(state);
1485 if (status != DDI_SUCCESS) {
1486 tavor_hw_fini(state, cleanup);
1487 TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail,
1488 TAVOR_TNF_ERROR, "");
1489 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1490 "hw_init_internal_uarpgs_alloc_fail");
1491 TAVOR_TNF_EXIT(tavor_hw_init);
1492 return (DDI_FAILURE);
1493 }
1494 cleanup = TAVOR_DRV_CLEANUP_LEVEL10;
1495
1496 /* Query and initialize the Tavor interrupt/MSI information */
1497 status = tavor_intr_or_msi_init(state);
1498 if (status != DDI_SUCCESS) {
1499 tavor_hw_fini(state, cleanup);
1500 TNF_PROBE_0(tavor_intr_or_msi_init_fail,
1501 TAVOR_TNF_ERROR, "");
1502 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1503 "intr_or_msi_init_fail");
1504 TAVOR_TNF_EXIT(tavor_hw_init);
1505 return (DDI_FAILURE);
1506 }
1507 cleanup = TAVOR_DRV_CLEANUP_LEVEL11;
1508
1509 /* Setup all of the Tavor EQs */
1510 status = tavor_eq_init_all(state);
1511 if (status != DDI_SUCCESS) {
1512 tavor_hw_fini(state, cleanup);
1513 TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, "");
1514 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1515 "hw_init_eqinitall_fail");
1516 TAVOR_TNF_EXIT(tavor_hw_init);
1517 return (DDI_FAILURE);
1518 }
1519 cleanup = TAVOR_DRV_CLEANUP_LEVEL12;
1520
1521 /* Set aside contexts for QP0 and QP1 */
1522 status = tavor_special_qp_contexts_reserve(state);
1523 if (status != DDI_SUCCESS) {
1524 tavor_hw_fini(state, cleanup);
1525 TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail,
1526 TAVOR_TNF_ERROR, "");
1527 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1528 "hw_init_reserve_special_qp_fail");
1529 TAVOR_TNF_EXIT(tavor_hw_init);
1530 return (DDI_FAILURE);
1531 }
1532 cleanup = TAVOR_DRV_CLEANUP_LEVEL13;
1533
1534 /* Initialize for multicast group handling */
1535 status = tavor_mcg_init(state);
1536 if (status != DDI_SUCCESS) {
1537 tavor_hw_fini(state, cleanup);
1538 TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, "");
1539 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail");
1540 TAVOR_TNF_EXIT(tavor_hw_init);
1541 return (DDI_FAILURE);
1542 }
1543 cleanup = TAVOR_DRV_CLEANUP_LEVEL14;
1544
1545 /* Initialize the Tavor IB port(s) */
1546 status = tavor_hca_port_init(state);
1547 if (status != DDI_SUCCESS) {
1548 tavor_hw_fini(state, cleanup);
1549 TNF_PROBE_0(tavor_hw_init_hca_port_init_fail,
1550 TAVOR_TNF_ERROR, "");
1551 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1552 "hw_init_hca_port_init_fail");
1553 TAVOR_TNF_EXIT(tavor_hw_init);
1554 return (DDI_FAILURE);
1555 }
1556 cleanup = TAVOR_DRV_CLEANUP_ALL;
1557
1558 /* Determine NodeGUID and SystemImageGUID */
1559 status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1560 &nodeinfo);
1561 if (status != TAVOR_CMD_SUCCESS) {
1562 cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n",
1563 status);
1564 tavor_hw_fini(state, cleanup);
1565 TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail,
1566 TAVOR_TNF_ERROR, "");
1567 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1568 "hw_init_getnodeinfo_cmd_fail");
1569 TAVOR_TNF_EXIT(tavor_hw_init);
1570 return (DDI_FAILURE);
1571 }
1572
1573 /*
1574 * If the NodeGUID value was set in OBP properties, then we use that
1575 * value. But we still print a message if the value we queried from
1576 * firmware does not match this value.
1577 *
1578 * Otherwise if OBP value is not set then we use the value from
1579 * firmware unconditionally.
1580 */
1581 if (state->ts_cfg_profile->cp_nodeguid) {
1582 state->ts_nodeguid = state->ts_cfg_profile->cp_nodeguid;
1583 } else {
1584 state->ts_nodeguid = nodeinfo.NodeGUID;
1585 }
1586
1587 if (state->ts_nodeguid != nodeinfo.NodeGUID) {
1588 cmn_err(CE_NOTE, "!NodeGUID value queried from firmware "
1589 "does not match value set by device property");
1590 }
1591
1592 /*
1593 * If the SystemImageGUID value was set in OBP properties, then we use
1594 * that value. But we still print a message if the value we queried
1595 * from firmware does not match this value.
1596 *
1597 * Otherwise if OBP value is not set then we use the value from
1598 * firmware unconditionally.
1599 */
1600 if (state->ts_cfg_profile->cp_sysimgguid) {
1601 state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid;
1602 } else {
1603 state->ts_sysimgguid = nodeinfo.SystemImageGUID;
1604 }
1605
1606 if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) {
1607 cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware "
1608 "does not match value set by device property");
1609 }
1610
1611 /* Get NodeDescription */
1612 status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1613 (sm_nodedesc_t *)&state->ts_nodedesc);
1614 if (status != TAVOR_CMD_SUCCESS) {
1615 cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n",
1616 status);
1617 tavor_hw_fini(state, cleanup);
1618 TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail,
1619 TAVOR_TNF_ERROR, "");
1620 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1621 "hw_init_getnodedesc_cmd_fail");
1622 TAVOR_TNF_EXIT(tavor_hw_init);
1623 return (DDI_FAILURE);
1624 }
1625
1626 TAVOR_TNF_EXIT(tavor_hw_init);
1627 return (DDI_SUCCESS);
1628 }
1629
1630
1631 /*
1632 * tavor_hw_fini()
1633 * Context: Only called from attach() and/or detach() path contexts
1634 */
1635 static void
1636 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup)
1637 {
1638 uint_t num_ports;
1639 int status;
1640
1641 TAVOR_TNF_ENTER(tavor_hw_fini);
1642
1643 switch (cleanup) {
1644 /*
1645 * If we add more driver initialization steps that should be cleaned
1646 * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the
1647 * first entry (i.e. corresponds to the last init step).
1648 */
1649 case TAVOR_DRV_CLEANUP_ALL:
1650 /* Shutdown the Tavor IB port(s) */
1651 num_ports = state->ts_cfg_profile->cp_num_ports;
1652 (void) tavor_hca_ports_shutdown(state, num_ports);
1653 /* FALLTHROUGH */
1654
1655 case TAVOR_DRV_CLEANUP_LEVEL14:
1656 /* Teardown resources used for multicast group handling */
1657 tavor_mcg_fini(state);
1658 /* FALLTHROUGH */
1659
1660 case TAVOR_DRV_CLEANUP_LEVEL13:
1661 /* Unreserve the special QP contexts */
1662 tavor_special_qp_contexts_unreserve(state);
1663 /* FALLTHROUGH */
1664
1665 case TAVOR_DRV_CLEANUP_LEVEL12:
1666 /*
1667 * Attempt to teardown all event queues (EQ). If we fail
1668 * here then print a warning message and return. Something
1669 * (either in HW or SW) has gone seriously wrong.
1670 */
1671 status = tavor_eq_fini_all(state);
1672 if (status != DDI_SUCCESS) {
1673 TAVOR_WARNING(state, "failed to teardown EQs");
1674 TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail,
1675 TAVOR_TNF_ERROR, "");
1676 TAVOR_TNF_EXIT(tavor_hw_fini);
1677 return;
1678 }
1679 /* FALLTHROUGH */
1680
1681 case TAVOR_DRV_CLEANUP_LEVEL11:
1682 status = tavor_intr_or_msi_fini(state);
1683 if (status != DDI_SUCCESS) {
1684 TAVOR_WARNING(state, "failed to free intr/MSI");
1685 TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail,
1686 TAVOR_TNF_ERROR, "");
1687 TAVOR_TNF_EXIT(tavor_hw_fini);
1688 return;
1689 }
1690 /* FALLTHROUGH */
1691
1692 case TAVOR_DRV_CLEANUP_LEVEL10:
1693 /* Free the resources for the Tavor internal UAR pages */
1694 tavor_internal_uarpgs_fini(state);
1695 /* FALLTHROUGH */
1696
1697 case TAVOR_DRV_CLEANUP_LEVEL9:
1698 /*
1699 * Free the PD that was used internally by Tavor software. If
1700 * we fail here then print a warning and return. Something
1701 * (probably software-related, but perhaps HW) has gone wrong.
1702 */
1703 status = tavor_pd_free(state, &state->ts_pdhdl_internal);
1704 if (status != DDI_SUCCESS) {
1705 TAVOR_WARNING(state, "failed to free internal PD");
1706 TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail,
1707 TAVOR_TNF_ERROR, "");
1708 TAVOR_TNF_EXIT(tavor_hw_fini);
1709 return;
1710 }
1711 /* FALLTHROUGH */
1712
1713 case TAVOR_DRV_CLEANUP_LEVEL8:
1714 /*
1715 * Post the CLOSE_HCA command to Tavor firmware. If we fail
1716 * here then print a warning and return. Something (either in
1717 * HW or SW) has gone seriously wrong.
1718 */
1719 status = tavor_close_hca_cmd_post(state,
1720 TAVOR_CMD_NOSLEEP_SPIN);
1721 if (status != TAVOR_CMD_SUCCESS) {
1722 TAVOR_WARNING(state, "failed to shutdown HCA");
1723 TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail,
1724 TAVOR_TNF_ERROR, "");
1725 TAVOR_TNF_EXIT(tavor_hw_fini);
1726 return;
1727 }
1728 /* FALLTHROUGH */
1729
1730 case TAVOR_DRV_CLEANUP_LEVEL7:
1731 /* Cleanup all the phase2 resources first */
1732 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL);
1733 /* FALLTHROUGH */
1734
1735 case TAVOR_DRV_CLEANUP_LEVEL6:
1736 /* Then cleanup the phase1 resources */
1737 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE);
1738 /* FALLTHROUGH */
1739
1740 case TAVOR_DRV_CLEANUP_LEVEL5:
1741 /*
1742 * Post the SYS_DIS command to Tavor firmware to shut
1743 * everything down again. If we fail here then print a
1744 * warning and return. Something (probably in HW, but maybe
1745 * in SW) has gone seriously wrong.
1746 */
1747 status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN);
1748 if (status != TAVOR_CMD_SUCCESS) {
1749 TAVOR_WARNING(state, "failed to shutdown hardware");
1750 TNF_PROBE_0(tavor_hw_fini_sys_dis_fail,
1751 TAVOR_TNF_ERROR, "");
1752 TAVOR_TNF_EXIT(tavor_hw_fini);
1753 return;
1754 }
1755 /* FALLTHROUGH */
1756
1757 case TAVOR_DRV_CLEANUP_LEVEL4:
1758 /* Teardown any resources allocated for the config profile */
1759 tavor_cfg_profile_fini(state);
1760 /* FALLTHROUGH */
1761
1762 case TAVOR_DRV_CLEANUP_LEVEL3:
1763 ddi_regs_map_free(&state->ts_reg_ddrhdl);
1764 /* FALLTHROUGH */
1765
1766 case TAVOR_DRV_CLEANUP_LEVEL2:
1767 ddi_regs_map_free(&state->ts_reg_uarhdl);
1768 /* FALLTHROUGH */
1769
1770 case TAVOR_DRV_CLEANUP_LEVEL1:
1771 case TAVOR_DRV_CLEANUP_LEVEL0:
1772 /*
1773 * LEVEL1 and LEVEL0 resources are freed in
1774 * tavor_drv_fini2().
1775 */
1776 break;
1777
1778 default:
1779 TAVOR_WARNING(state, "unexpected driver cleanup level");
1780 TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, "");
1781 TAVOR_TNF_EXIT(tavor_hw_fini);
1782 return;
1783 }
1784
1785 TAVOR_TNF_EXIT(tavor_hw_fini);
1786 }
1787
1788
1789 /*
1790 * tavor_soft_state_init()
1791 * Context: Only called from attach() path context
1792 */
1793 static int
1794 tavor_soft_state_init(tavor_state_t *state)
1795 {
1796 ibt_hca_attr_t *hca_attr;
1797 uint64_t maxval, val;
1798 ibt_hca_flags_t caps = IBT_HCA_NO_FLAGS;
1799 int status;
1800
1801 TAVOR_TNF_ENTER(tavor_soft_state_init);
1802
1803 /*
1804 * The ibc_hca_info_t struct is passed to the IBTF. This is the
1805 * routine where we initialize it. Many of the init values come from
1806 * either configuration variables or successful queries of the Tavor
1807 * hardware abilities
1808 */
1809 state->ts_ibtfinfo.hca_ci_vers = IBCI_V4;
1810 state->ts_ibtfinfo.hca_handle = (ibc_hca_hdl_t)state;
1811 state->ts_ibtfinfo.hca_ops = &tavor_ibc_ops;
1812
1813 hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
1814 state->ts_ibtfinfo.hca_attr = hca_attr;
1815
1816 hca_attr->hca_dip = state->ts_dip;
1817 hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major;
1818 hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor;
1819 hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor;
1820
1821 /*
1822 * Determine HCA capabilities:
1823 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST,
1824 * IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE,
1825 * or IBT_HCA_SHUTDOWN_PORT
1826 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID,
1827 * IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always
1828 * supported
1829 * All other features are conditionally supported, depending on the
1830 * status return by the Tavor HCA (in QUERY_DEV_LIM)
1831 */
1832 if (state->ts_devlim.ud_multi) {
1833 caps |= IBT_HCA_UD_MULTICAST;
1834 }
1835 if (state->ts_devlim.atomic) {
1836 caps |= IBT_HCA_ATOMICS_HCA;
1837 }
1838 if (state->ts_devlim.apm) {
1839 caps |= IBT_HCA_AUTO_PATH_MIG;
1840 }
1841 if (state->ts_devlim.pkey_v) {
1842 caps |= IBT_HCA_PKEY_CNTR;
1843 }
1844 if (state->ts_devlim.qkey_v) {
1845 caps |= IBT_HCA_QKEY_CNTR;
1846 }
1847 if (state->ts_cfg_profile->cp_srq_enable) {
1848 caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ;
1849 }
1850 caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT |
1851 IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE |
1852 IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE);
1853 hca_attr->hca_flags = caps;
1854 hca_attr->hca_flags2 = IBT_HCA2_DMA_MR;
1855
1856 /* Determine VendorID, DeviceID, and revision ID */
1857 hca_attr->hca_vendor_id = state->ts_adapter.vendor_id;
1858 hca_attr->hca_device_id = state->ts_adapter.device_id;
1859 hca_attr->hca_version_id = state->ts_adapter.rev_id;
1860
1861 /*
1862 * Determine number of available QPs and max QP size. Number of
1863 * available QPs is determined by subtracting the number of
1864 * "reserved QPs" (i.e. reserved for firmware use) from the
1865 * total number configured.
1866 */
1867 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
1868 hca_attr->hca_max_qp = val - ((uint64_t)1 <<
1869 state->ts_devlim.log_rsvd_qp);
1870 maxval = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz);
1871 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz);
1872 if (val > maxval) {
1873 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1874 TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail,
1875 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size "
1876 "exceeds device maximum", tnf_uint, maxsz, maxval);
1877 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1878 "soft_state_init_maxqpsz_toobig_fail");
1879 TAVOR_TNF_EXIT(tavor_soft_state_init);
1880 return (DDI_FAILURE);
1881 }
1882 hca_attr->hca_max_qp_sz = val;
1883
1884 /* Determine max scatter-gather size in WQEs */
1885 maxval = state->ts_devlim.max_sg;
1886 val = state->ts_cfg_profile->cp_wqe_max_sgl;
1887 if (val > maxval) {
1888 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1889 TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail,
1890 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl "
1891 "exceeds device maximum", tnf_uint, maxsgl, maxval);
1892 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1893 "soft_state_init_toomanysgl_fail");
1894 TAVOR_TNF_EXIT(tavor_soft_state_init);
1895 return (DDI_FAILURE);
1896 }
1897 /* If the rounded value for max SGL is too large, cap it */
1898 if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) {
1899 state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval;
1900 val = maxval;
1901 } else {
1902 val = state->ts_cfg_profile->cp_wqe_real_max_sgl;
1903 }
1904
1905 hca_attr->hca_max_sgl = val;
1906 hca_attr->hca_max_rd_sgl = 0; /* zero because RD is unsupported */
1907
1908 /*
1909 * Determine number of available CQs and max CQ size. Number of
1910 * available CQs is determined by subtracting the number of
1911 * "reserved CQs" (i.e. reserved for firmware use) from the
1912 * total number configured.
1913 */
1914 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq);
1915 hca_attr->hca_max_cq = val - ((uint64_t)1 <<
1916 state->ts_devlim.log_rsvd_cq);
1917 maxval = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz);
1918 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1;
1919 if (val > maxval) {
1920 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1921 TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail,
1922 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size "
1923 "exceeds device maximum", tnf_uint, maxsz, maxval);
1924 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1925 "soft_state_init_maxcqsz_toobig_fail");
1926 TAVOR_TNF_EXIT(tavor_soft_state_init);
1927 return (DDI_FAILURE);
1928 }
1929 hca_attr->hca_max_cq_sz = val;
1930
1931 /*
1932 * Determine number of available SRQs and max SRQ size. Number of
1933 * available SRQs is determined by subtracting the number of
1934 * "reserved SRQs" (i.e. reserved for firmware use) from the
1935 * total number configured.
1936 */
1937 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq);
1938 hca_attr->hca_max_srqs = val - ((uint64_t)1 <<
1939 state->ts_devlim.log_rsvd_srq);
1940 maxval = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz);
1941 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz);
1942
1943 if (val > maxval) {
1944 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1945 TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail,
1946 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size "
1947 "exceeds device maximum", tnf_uint, maxsz, maxval);
1948 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1949 "soft_state_init_maxsrqsz_toobig_fail");
1950 TAVOR_TNF_EXIT(tavor_soft_state_init);
1951 return (DDI_FAILURE);
1952 }
1953 hca_attr->hca_max_srqs_sz = val;
1954
1955 val = state->ts_cfg_profile->cp_srq_max_sgl;
1956 maxval = state->ts_devlim.max_sg;
1957 if (val > maxval) {
1958 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1959 TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail,
1960 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq "
1961 "sgl exceeds device maximum", tnf_uint, maxsgl, maxval);
1962 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1963 "soft_state_init_toomanysrqsgl_fail");
1964 TAVOR_TNF_EXIT(tavor_soft_state_init);
1965 return (DDI_FAILURE);
1966 }
1967 hca_attr->hca_max_srq_sgl = val;
1968
1969 /*
1970 * Determine supported HCA page sizes
1971 * XXX
1972 * For now we simply return the system pagesize as the only supported
1973 * pagesize
1974 */
1975 hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K :
1976 IBT_PAGE_4K);
1977
1978 /*
1979 * Determine number of available MemReg, MemWin, and their max size.
1980 * Number of available MRs and MWs is determined by subtracting
1981 * the number of "reserved MPTs" (i.e. reserved for firmware use)
1982 * from the total number configured for each.
1983 */
1984 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt);
1985 hca_attr->hca_max_memr = val - ((uint64_t)1 <<
1986 state->ts_devlim.log_rsvd_mpt);
1987 hca_attr->hca_max_mem_win = val - ((uint64_t)1 <<
1988 state->ts_devlim.log_rsvd_mpt);
1989 maxval = state->ts_devlim.log_max_mrw_sz;
1990 val = state->ts_cfg_profile->cp_log_max_mrw_sz;
1991 if (val > maxval) {
1992 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1993 TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail,
1994 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size "
1995 "exceeds device maximum", tnf_uint, maxsz, maxval);
1996 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1997 "soft_state_init_maxmrwsz_toobig_fail");
1998 TAVOR_TNF_EXIT(tavor_soft_state_init);
1999 return (DDI_FAILURE);
2000 }
2001 hca_attr->hca_max_memr_len = ((uint64_t)1 << val);
2002
2003 /* Determine RDMA/Atomic properties */
2004 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb);
2005 hca_attr->hca_max_rsc = val;
2006 val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp;
2007 hca_attr->hca_max_rdma_in_qp = val;
2008 val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp;
2009 hca_attr->hca_max_rdma_out_qp = val;
2010 hca_attr->hca_max_rdma_in_ee = 0;
2011 hca_attr->hca_max_rdma_out_ee = 0;
2012
2013 /*
2014 * Determine maximum number of raw IPv6 and Ether QPs. Set to 0
2015 * because neither type of raw QP is supported
2016 */
2017 hca_attr->hca_max_ipv6_qp = 0;
2018 hca_attr->hca_max_ether_qp = 0;
2019
2020 /* Determine max number of MCGs and max QP-per-MCG */
2021 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
2022 hca_attr->hca_max_mcg_qps = val;
2023 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg);
2024 hca_attr->hca_max_mcg = val;
2025 val = state->ts_cfg_profile->cp_num_qp_per_mcg;
2026 hca_attr->hca_max_qp_per_mcg = val;
2027
2028 /* Determine max number partitions (i.e. PKeys) */
2029 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2030 val = ((uint64_t)state->ts_cfg_profile->cp_num_ports <<
2031 state->ts_cfg_profile->cp_log_max_pkeytbl);
2032
2033 if (val > maxval) {
2034 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2035 TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail,
2036 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys "
2037 "exceeds device maximum", tnf_uint, maxpkey, maxval);
2038 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2039 "soft_state_init_toomanypkey_fail");
2040 TAVOR_TNF_EXIT(tavor_soft_state_init);
2041 return (DDI_FAILURE);
2042 }
2043 hca_attr->hca_max_partitions = val;
2044
2045 /* Determine number of ports */
2046 maxval = state->ts_devlim.num_ports;
2047 val = state->ts_cfg_profile->cp_num_ports;
2048 if ((val > maxval) || (val == 0)) {
2049 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2050 TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail,
2051 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports "
2052 "exceeds device maximum", tnf_uint, maxports, maxval);
2053 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2054 "soft_state_init_toomanyports_fail");
2055 TAVOR_TNF_EXIT(tavor_soft_state_init);
2056 return (DDI_FAILURE);
2057 }
2058 hca_attr->hca_nports = val;
2059
2060 /* Copy NodeGUID and SystemImageGUID from softstate */
2061 hca_attr->hca_node_guid = state->ts_nodeguid;
2062 hca_attr->hca_si_guid = state->ts_sysimgguid;
2063
2064 /*
2065 * Determine local ACK delay. Use the value suggested by the Tavor
2066 * hardware (from the QUERY_DEV_LIM command)
2067 */
2068 hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay;
2069
2070 /* Determine max SGID table and PKey table sizes */
2071 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl);
2072 hca_attr->hca_max_port_sgid_tbl_sz = val;
2073 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
2074 hca_attr->hca_max_port_pkey_tbl_sz = val;
2075
2076 /* Determine max number of PDs */
2077 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pd);
2078 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd);
2079 if (val > maxval) {
2080 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2081 TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail,
2082 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD "
2083 "exceeds device maximum", tnf_uint, maxpd, maxval);
2084 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2085 "soft_state_init_toomanypd_fail");
2086 TAVOR_TNF_EXIT(tavor_soft_state_init);
2087 return (DDI_FAILURE);
2088 }
2089 hca_attr->hca_max_pd = val;
2090
2091 /* Determine max number of Address Handles */
2092 maxval = ((uint64_t)1 << state->ts_devlim.log_max_av);
2093 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah);
2094 if (val > maxval) {
2095 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2096 TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail,
2097 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH "
2098 "exceeds device maximum", tnf_uint, maxah, maxval);
2099 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2100 "soft_state_init_toomanyah_fail");
2101 TAVOR_TNF_EXIT(tavor_soft_state_init);
2102 return (DDI_FAILURE);
2103 }
2104 hca_attr->hca_max_ah = val;
2105
2106 /* No RDDs or EECs (since Reliable Datagram is not supported) */
2107 hca_attr->hca_max_rdd = 0;
2108 hca_attr->hca_max_eec = 0;
2109
2110 /* Initialize lock for reserved UAR page access */
2111 mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER,
2112 DDI_INTR_PRI(state->ts_intrmsi_pri));
2113
2114 /* Initialize the flash fields */
2115 state->ts_fw_flashstarted = 0;
2116 mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER,
2117 DDI_INTR_PRI(state->ts_intrmsi_pri));
2118
2119 /* Initialize the lock for the info ioctl */
2120 mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER,
2121 DDI_INTR_PRI(state->ts_intrmsi_pri));
2122
2123 /* Initialize the AVL tree for QP number support */
2124 tavor_qpn_avl_init(state);
2125
2126 /* Initialize the kstat info structure */
2127 status = tavor_kstat_init(state);
2128 if (status != DDI_SUCCESS) {
2129 tavor_qpn_avl_fini(state);
2130 mutex_destroy(&state->ts_info_lock);
2131 mutex_destroy(&state->ts_fw_flashlock);
2132 mutex_destroy(&state->ts_uar_lock);
2133 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2134 TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail,
2135 TAVOR_TNF_ERROR, "");
2136 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2137 "soft_state_init_kstatinit_fail");
2138 TAVOR_TNF_EXIT(tavor_soft_state_init);
2139 return (DDI_FAILURE);
2140 }
2141
2142 TAVOR_TNF_EXIT(tavor_soft_state_init);
2143 return (DDI_SUCCESS);
2144 }
2145
2146
2147 /*
2148 * tavor_soft_state_fini()
2149 * Context: Called only from detach() path context
2150 */
2151 static void
2152 tavor_soft_state_fini(tavor_state_t *state)
2153 {
2154 TAVOR_TNF_ENTER(tavor_soft_state_fini);
2155
2156 /* Teardown the kstat info */
2157 tavor_kstat_fini(state);
2158
2159 /* Teardown the AVL tree for QP number support */
2160 tavor_qpn_avl_fini(state);
2161
2162 /* Free up info ioctl mutex */
2163 mutex_destroy(&state->ts_info_lock);
2164
2165 /* Free up flash mutex */
2166 mutex_destroy(&state->ts_fw_flashlock);
2167
2168 /* Free up the UAR page access mutex */
2169 mutex_destroy(&state->ts_uar_lock);
2170
2171 /* Free up the hca_attr struct */
2172 kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t));
2173
2174 TAVOR_TNF_EXIT(tavor_soft_state_fini);
2175 }
2176
2177
2178 /*
2179 * tavor_hca_config_setup()
2180 * Context: Only called from attach() path context
2181 */
2182 static void
2183 tavor_hca_config_setup(tavor_state_t *state,
2184 tavor_hw_initqueryhca_t *inithca)
2185 {
2186 tavor_rsrc_pool_info_t *rsrc_pool;
2187 uint64_t ddr_baseaddr, ddr_base_map_addr;
2188 uint64_t offset, addr;
2189 uint_t mcg_size;
2190
2191 TAVOR_TNF_ENTER(tavor_hca_config_setup);
2192
2193 /* Set "host endianness". Default is big endian */
2194 #ifdef _LITTLE_ENDIAN
2195 inithca->big_endian = 0;
2196 #else
2197 inithca->big_endian = 1;
2198 #endif
2199 /* No Address Vector Protection, but Port Checking on by default */
2200 inithca->udav_chk = TAVOR_UDAV_PROTECT_DISABLED;
2201 inithca->udav_port_chk = TAVOR_UDAV_PORTCHK_ENABLED;
2202
2203 ddr_baseaddr = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr;
2204 ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr;
2205
2206 /* Setup QPC table */
2207 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
2208 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2209 addr = ddr_base_map_addr + offset;
2210 inithca->context.qpc_baseaddr_h = (addr >> 32);
2211 inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7;
2212 inithca->context.log_num_qp = state->ts_cfg_profile->cp_log_num_qp;
2213
2214 /* Setup EEC table (initialize to zero - RD unsupported) */
2215 inithca->context.eec_baseaddr_h = 0;
2216 inithca->context.eec_baseaddr_l = 0;
2217 inithca->context.log_num_ee = 0;
2218
2219 /* Setup CQC table */
2220 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
2221 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2222 addr = ddr_base_map_addr + offset;
2223 inithca->context.cqc_baseaddr_h = (addr >> 32);
2224 inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2225 inithca->context.log_num_cq = state->ts_cfg_profile->cp_log_num_cq;
2226
2227 /* Setup SRQC table */
2228 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
2229 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2230 addr = ddr_base_map_addr + offset;
2231 inithca->context.srqc_baseaddr_h = (addr >> 32);
2232 inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2233 inithca->context.log_num_srq =
2234 state->ts_cfg_profile->cp_log_num_srq;
2235
2236 /* Setup EQPC table */
2237 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
2238 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2239 addr = ddr_base_map_addr + offset;
2240 inithca->context.eqpc_baseaddr = addr;
2241
2242 /* Setup EEEC table (initialize to zero - RD unsupported) */
2243 inithca->context.eeec_baseaddr = 0;
2244
2245 /* Setup EQC table */
2246 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
2247 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2248 addr = ddr_base_map_addr + offset;
2249 inithca->context.eqc_baseaddr_h = (addr >> 32);
2250 inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2251 inithca->context.log_num_eq = TAVOR_NUM_EQ_SHIFT;
2252
2253 /* Setup RDB table */
2254 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
2255 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2256 addr = ddr_base_map_addr + offset;
2257 inithca->context.rdb_baseaddr_h = (addr >> 32);
2258 inithca->context.rdb_baseaddr_l = 0;
2259
2260 /* Setup Multicast */
2261 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
2262 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2263 addr = ddr_base_map_addr + offset;
2264 inithca->multi.mc_baseaddr = addr;
2265 mcg_size = TAVOR_MCGMEM_SZ(state);
2266 inithca->multi.log_mc_tbl_ent = highbit(mcg_size) - 1;
2267 inithca->multi.mc_tbl_hash_sz =
2268 (1 << state->ts_cfg_profile->cp_log_num_mcg_hash);
2269 inithca->multi.mc_hash_fn = TAVOR_MCG_DEFAULT_HASH_FN;
2270 inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg;
2271
2272
2273 /* Setup TPT */
2274 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
2275 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2276 addr = ddr_base_map_addr + offset;
2277 inithca->tpt.mpt_baseaddr = addr;
2278 inithca->tpt.mttseg_sz = TAVOR_MTTSEG_SIZE_SHIFT;
2279 inithca->tpt.log_mpt_sz = state->ts_cfg_profile->cp_log_num_mpt;
2280 inithca->tpt.mtt_version = TAVOR_MTT_PG_WALK_VER;
2281
2282 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
2283 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2284 addr = ddr_base_map_addr + offset;
2285 inithca->tpt.mtt_baseaddr = addr;
2286
2287 /* Setup UAR */
2288 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
2289 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2290 addr = ddr_base_map_addr + offset;
2291 inithca->uar.uarscr_baseaddr = addr;
2292
2293 inithca->uar.uar_pg_sz = PAGESHIFT - 0xC;
2294
2295 TAVOR_TNF_EXIT(tavor_hca_config_setup);
2296 }
2297
2298
2299 /*
2300 * tavor_hca_port_init()
2301 * Context: Only called from attach() path context
2302 */
2303 static int
2304 tavor_hca_port_init(tavor_state_t *state)
2305 {
2306 tavor_hw_initib_t *portinits, *initib;
2307 tavor_cfg_profile_t *cfgprof;
2308 uint_t num_ports;
2309 int i, status;
2310 uint64_t maxval, val;
2311 uint64_t sysimgguid, nodeguid, portguid;
2312
2313 TAVOR_TNF_ENTER(tavor_hca_port_init);
2314
2315 cfgprof = state->ts_cfg_profile;
2316
2317 /* Get number of HCA ports */
2318 num_ports = cfgprof->cp_num_ports;
2319
2320 /* Allocate space for Tavor port init struct(s) */
2321 portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports *
2322 sizeof (tavor_hw_initib_t), KM_SLEEP);
2323
2324 /* Post command to initialize Tavor HCA port */
2325 for (i = 0; i < num_ports; i++) {
2326 initib = &portinits[i];
2327
2328 /*
2329 * Determine whether we need to override the firmware's
2330 * default SystemImageGUID setting.
2331 */
2332 sysimgguid = cfgprof->cp_sysimgguid;
2333 if (sysimgguid != 0) {
2334 initib->set_sysimg_guid = 1;
2335 initib->sysimg_guid = sysimgguid;
2336 }
2337
2338 /*
2339 * Determine whether we need to override the firmware's
2340 * default NodeGUID setting.
2341 */
2342 nodeguid = cfgprof->cp_nodeguid;
2343 if (nodeguid != 0) {
2344 initib->set_node_guid = 1;
2345 initib->node_guid = nodeguid;
2346 }
2347
2348 /*
2349 * Determine whether we need to override the firmware's
2350 * default PortGUID setting.
2351 */
2352 portguid = cfgprof->cp_portguid[i];
2353 if (portguid != 0) {
2354 initib->set_port_guid0 = 1;
2355 initib->guid0 = portguid;
2356 }
2357
2358 /* Validate max MTU size */
2359 maxval = state->ts_devlim.max_mtu;
2360 val = cfgprof->cp_max_mtu;
2361 if (val > maxval) {
2362 TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail,
2363 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2364 "MTU size exceeds device maximum", tnf_uint,
2365 maxmtu, maxval);
2366 TAVOR_TNF_EXIT(tavor_hca_port_init);
2367 goto init_ports_fail;
2368 }
2369 initib->mtu_cap = val;
2370
2371 /* Validate the max port width */
2372 maxval = state->ts_devlim.max_port_width;
2373 val = cfgprof->cp_max_port_width;
2374 if (val > maxval) {
2375 TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail,
2376 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2377 "port width exceeds device maximum", tnf_uint,
2378 maxportwidth, maxval);
2379 TAVOR_TNF_EXIT(tavor_hca_port_init);
2380 goto init_ports_fail;
2381 }
2382 initib->port_width_cap = val;
2383
2384 /* Validate max VL cap size */
2385 maxval = state->ts_devlim.max_vl;
2386 val = cfgprof->cp_max_vlcap;
2387 if (val > maxval) {
2388 TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail,
2389 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2390 "VLcap size exceeds device maximum", tnf_uint,
2391 maxvlcap, maxval);
2392 TAVOR_TNF_EXIT(tavor_hca_port_init);
2393 goto init_ports_fail;
2394 }
2395 initib->vl_cap = val;
2396
2397 /* Validate max GID table size */
2398 maxval = ((uint64_t)1 << state->ts_devlim.log_max_gid);
2399 val = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl);
2400 if (val > maxval) {
2401 TNF_PROBE_2(tavor_hca_port_init_gidtable_fail,
2402 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2403 "GID table size exceeds device maximum", tnf_uint,
2404 maxgidtbl, maxval);
2405 TAVOR_TNF_EXIT(tavor_hca_port_init);
2406 goto init_ports_fail;
2407 }
2408 initib->max_gid = val;
2409
2410 /* Validate max PKey table size */
2411 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2412 val = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl);
2413 if (val > maxval) {
2414 TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail,
2415 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2416 "PKey table size exceeds device maximum", tnf_uint,
2417 maxpkeytbl, maxval);
2418 TAVOR_TNF_EXIT(tavor_hca_port_init);
2419 goto init_ports_fail;
2420 }
2421 initib->max_pkey = val;
2422
2423 /*
2424 * Post the INIT_IB command to Tavor firmware. When this
2425 * command completes, the corresponding Tavor port will be
2426 * physically "Up" and initialized.
2427 */
2428 status = tavor_init_ib_cmd_post(state, initib, i + 1,
2429 TAVOR_CMD_NOSLEEP_SPIN);
2430 if (status != TAVOR_CMD_SUCCESS) {
2431 cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command "
2432 "failed: %08x\n", i + 1, status);
2433 TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail,
2434 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2435 tnf_uint, port, i + 1);
2436 TAVOR_TNF_EXIT(tavor_hca_port_init);
2437 goto init_ports_fail;
2438 }
2439 }
2440
2441 /* Free up the memory for Tavor port init struct(s), return success */
2442 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2443 TAVOR_TNF_EXIT(tavor_hca_port_init);
2444 return (DDI_SUCCESS);
2445
2446 init_ports_fail:
2447 /*
2448 * Free up the memory for Tavor port init struct(s), shutdown any
2449 * successfully initialized ports, and return failure
2450 */
2451 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2452 (void) tavor_hca_ports_shutdown(state, i);
2453
2454 TAVOR_TNF_EXIT(tavor_hca_port_init);
2455 return (DDI_FAILURE);
2456 }
2457
2458
2459 /*
2460 * tavor_hca_ports_shutdown()
2461 * Context: Only called from attach() and/or detach() path contexts
2462 */
2463 static int
2464 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init)
2465 {
2466 int i, status;
2467
2468 TAVOR_TNF_ENTER(tavor_hca_ports_shutdown);
2469
2470 /*
2471 * Post commands to shutdown all init'd Tavor HCA ports. Note: if
2472 * any of these commands fail for any reason, it would be entirely
2473 * unexpected and probably indicative a serious problem (HW or SW).
2474 * Although we do return void from this function, this type of failure
2475 * should not go unreported. That is why we have the warning message
2476 * and the detailed TNF information.
2477 */
2478 for (i = 0; i < num_init; i++) {
2479 status = tavor_close_ib_cmd_post(state, i + 1,
2480 TAVOR_CMD_NOSLEEP_SPIN);
2481 if (status != TAVOR_CMD_SUCCESS) {
2482 TAVOR_WARNING(state, "failed to shutdown HCA port");
2483 TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail,
2484 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2485 tnf_uint, port, i + 1);
2486 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2487 return (status);
2488 }
2489 }
2490
2491 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2492
2493 return (TAVOR_CMD_SUCCESS);
2494 }
2495
2496
2497 /*
2498 * tavor_internal_uarpgs_init
2499 * Context: Only called from attach() path context
2500 */
2501 static int
2502 tavor_internal_uarpgs_init(tavor_state_t *state)
2503 {
2504 int status;
2505
2506 TAVOR_TNF_ENTER(tavor_internal_uarpgs_init);
2507
2508 /*
2509 * Save away reserved Tavor UAR page #0. This UAR page is not to
2510 * be used by software.
2511 */
2512 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2513 &state->ts_uarpg0_rsrc_rsrvd);
2514 if (status != DDI_SUCCESS) {
2515 TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2516 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2517 return (DDI_FAILURE);
2518 }
2519
2520 /*
2521 * Save away Tavor UAR page #1 (for internal use). This UAR page is
2522 * the privileged UAR page through which all kernel generated
2523 * doorbells will be rung.
2524 */
2525 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2526 &state->ts_uarpg1_rsrc);
2527 if (status != DDI_SUCCESS) {
2528 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2529 TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2530 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2531 return (DDI_FAILURE);
2532 }
2533
2534 /* Setup pointer to UAR page #1 doorbells */
2535 state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr;
2536
2537 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2538 return (DDI_SUCCESS);
2539 }
2540
2541
2542 /*
2543 * tavor_internal_uarpgs_fini
2544 * Context: Only called from attach() and/or detach() path contexts
2545 */
2546 static void
2547 tavor_internal_uarpgs_fini(tavor_state_t *state)
2548 {
2549 TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini);
2550
2551 /* Free up Tavor UAR page #1 (kernel driver doorbells) */
2552 tavor_rsrc_free(state, &state->ts_uarpg1_rsrc);
2553
2554 /* Free up Tavor UAR page #0 (reserved) */
2555 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2556
2557 TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini);
2558 }
2559
2560
2561 /*
2562 * tavor_special_qp_contexts_reserve()
2563 * Context: Only called from attach() path context
2564 */
2565 static int
2566 tavor_special_qp_contexts_reserve(tavor_state_t *state)
2567 {
2568 tavor_rsrc_t *qp0_rsrc, *qp1_rsrc;
2569 int status;
2570
2571 TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve);
2572
2573 /* Initialize the lock used for special QP rsrc management */
2574 mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER,
2575 DDI_INTR_PRI(state->ts_intrmsi_pri));
2576
2577 /*
2578 * Reserve contexts for QP0. These QP contexts will be setup to
2579 * act as aliases for the real QP0. Note: We are required to grab
2580 * two QPs (one per port) even if we are operating in single-port
2581 * mode.
2582 */
2583 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc);
2584 if (status != DDI_SUCCESS) {
2585 mutex_destroy(&state->ts_spec_qplock);
2586 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail,
2587 TAVOR_TNF_ERROR, "");
2588 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2589 return (DDI_FAILURE);
2590 }
2591 state->ts_spec_qp0 = qp0_rsrc;
2592
2593 /*
2594 * Reserve contexts for QP1. These QP contexts will be setup to
2595 * act as aliases for the real QP1. Note: We are required to grab
2596 * two QPs (one per port) even if we are operating in single-port
2597 * mode.
2598 */
2599 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc);
2600 if (status != DDI_SUCCESS) {
2601 tavor_rsrc_free(state, &qp0_rsrc);
2602 mutex_destroy(&state->ts_spec_qplock);
2603 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail,
2604 TAVOR_TNF_ERROR, "");
2605 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2606 return (DDI_FAILURE);
2607 }
2608 state->ts_spec_qp1 = qp1_rsrc;
2609
2610 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2611 return (DDI_SUCCESS);
2612 }
2613
2614
2615 /*
2616 * tavor_special_qp_contexts_unreserve()
2617 * Context: Only called from attach() and/or detach() path contexts
2618 */
2619 static void
2620 tavor_special_qp_contexts_unreserve(tavor_state_t *state)
2621 {
2622 TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve);
2623
2624 /* Unreserve contexts for QP1 */
2625 tavor_rsrc_free(state, &state->ts_spec_qp1);
2626
2627 /* Unreserve contexts for QP0 */
2628 tavor_rsrc_free(state, &state->ts_spec_qp0);
2629
2630 /* Destroy the lock used for special QP rsrc management */
2631 mutex_destroy(&state->ts_spec_qplock);
2632
2633 TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve);
2634 }
2635
2636
2637 /*
2638 * tavor_sw_reset()
2639 * Context: Currently called only from attach() path context
2640 */
2641 static int
2642 tavor_sw_reset(tavor_state_t *state)
2643 {
2644 dev_info_t *dip, *pdip;
2645 ddi_acc_handle_t hdl = state->ts_pci_cfghdl, phdl;
2646 uint32_t reset_delay;
2647 int status, i;
2648
2649 TAVOR_TNF_ENTER(tavor_sw_reset);
2650
2651 /*
2652 * If the configured software reset delay is set to zero, then we
2653 * will not attempt a software reset of the Tavor device.
2654 */
2655 reset_delay = state->ts_cfg_profile->cp_sw_reset_delay;
2656 if (reset_delay == 0) {
2657 TAVOR_TNF_EXIT(tavor_sw_reset);
2658 return (DDI_SUCCESS);
2659 }
2660
2661 /*
2662 * Get dip for HCA device _and_ parent device as well. Parent access
2663 * is necessary here because software reset of the Tavor hardware
2664 * will reinitialize both the config registers of the PCI bridge
2665 * (parent, if it exists) and the IB HCA (self)
2666 */
2667 dip = state->ts_dip;
2668 pdip = ddi_get_parent(dip);
2669
2670 /* Query the PCI capabilities of the HCA device */
2671 tavor_pci_capability_list(state, hdl);
2672
2673 /*
2674 * Read all PCI config info (reg0...reg63). Note: According to the
2675 * Tavor software reset application note, we should not read or
2676 * restore the values in reg22 and reg23.
2677 */
2678 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2679 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2680 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2681 state->ts_cfg_data[i] = pci_config_get32(hdl, i << 2);
2682 }
2683 }
2684
2685 if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2686 /*
2687 * Setup for PCI config read/write of bridge device
2688 */
2689 status = pci_config_setup(pdip, &phdl);
2690 if (status != DDI_SUCCESS) {
2691 TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail,
2692 TAVOR_TNF_ERROR, "");
2693 TAVOR_TNF_EXIT(tavor_sw_reset);
2694 return (DDI_FAILURE);
2695 }
2696
2697 /*
2698 * Read all PCI config info (reg0...reg63). Note: According to
2699 * the Tavor software reset application note, we should not
2700 * read or restore the values in reg22 and reg23.
2701 */
2702 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2703 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2704 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2705 state->ts_cfg_pdata[i] =
2706 pci_config_get32(phdl, i << 2);
2707 }
2708 }
2709 }
2710
2711 /*
2712 * Perform the software reset (by writing 1 at offset 0xF0010)
2713 */
2714 ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset,
2715 TAVOR_SW_RESET_START);
2716
2717 drv_usecwait(reset_delay);
2718
2719 if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2720 /*
2721 * Bridge exists, so wait for the bridge to become ready.
2722 *
2723 * The above delay is necessary to avoid system panic from
2724 * Master Abort. If the device is accessed before this delay,
2725 * device will not respond to config cycles and they will be
2726 * terminate with a Master Abort which will panic the system.
2727 * Below is the loop we use to poll status from the device to
2728 * determine if it is OK to proceed.
2729 */
2730 i = 0;
2731 while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2732 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2733 }
2734
2735 /*
2736 * Write all the PCI config registers back into each device
2737 * (except for reg22 and reg23 - see above)
2738 */
2739 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2740 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2741 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2742 pci_config_put32(phdl, i << 2,
2743 state->ts_cfg_pdata[i]);
2744 }
2745 }
2746
2747 /*
2748 * Tear down the config setup (for bridge device)
2749 */
2750 pci_config_teardown(&phdl);
2751
2752 /* No Bridge Device */
2753 } else {
2754 /*
2755 * Bridge does not exist, so instead wait for the device itself
2756 * to become ready.
2757 *
2758 * The above delay is necessary to avoid system panic from
2759 * Master Abort. If the device is accessed before this delay,
2760 * device will not respond to config cycles and they will be
2761 * terminate with a Master Abort which will panic the system.
2762 * Below is the loop we use to poll status from the device to
2763 * determine if it is OK to proceed.
2764 */
2765 i = 0;
2766 while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2767 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2768 }
2769 }
2770
2771 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2772 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2773 (i != TAVOR_SW_RESET_REG23_RSVD)) {
2774 pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]);
2775 }
2776 }
2777
2778 TAVOR_TNF_EXIT(tavor_sw_reset);
2779 return (DDI_SUCCESS);
2780 }
2781
2782
2783 /*
2784 * tavor_mcg_init()
2785 * Context: Only called from attach() path context
2786 */
2787 static int
2788 tavor_mcg_init(tavor_state_t *state)
2789 {
2790 uint_t mcg_tmp_sz;
2791
2792 TAVOR_TNF_ENTER(tavor_mcg_init);
2793
2794 /*
2795 * Allocate space for the MCG temporary copy buffer. This is
2796 * used by the Attach/Detach Multicast Group code
2797 */
2798 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2799 state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP);
2800
2801 /*
2802 * Initialize the multicast group mutex. This ensures atomic
2803 * access to add, modify, and remove entries in the multicast
2804 * group hash lists.
2805 */
2806 mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER,
2807 DDI_INTR_PRI(state->ts_intrmsi_pri));
2808
2809 TAVOR_TNF_EXIT(tavor_mcg_init);
2810 return (DDI_SUCCESS);
2811 }
2812
2813
2814 /*
2815 * tavor_mcg_fini()
2816 * Context: Only called from attach() and/or detach() path contexts
2817 */
2818 static void
2819 tavor_mcg_fini(tavor_state_t *state)
2820 {
2821 uint_t mcg_tmp_sz;
2822
2823 TAVOR_TNF_ENTER(tavor_mcg_fini);
2824
2825 /* Free up the space used for the MCG temporary copy buffer */
2826 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2827 kmem_free(state->ts_mcgtmp, mcg_tmp_sz);
2828
2829 /* Destroy the multicast group mutex */
2830 mutex_destroy(&state->ts_mcglock);
2831
2832 TAVOR_TNF_EXIT(tavor_mcg_fini);
2833 }
2834
2835
2836 /*
2837 * tavor_fw_version_check()
2838 * Context: Only called from attach() path context
2839 */
2840 static int
2841 tavor_fw_version_check(tavor_state_t *state)
2842 {
2843 uint_t tavor_fw_ver_major;
2844 uint_t tavor_fw_ver_minor;
2845 uint_t tavor_fw_ver_subminor;
2846
2847 /*
2848 * Depending on which version of driver we have attached, the firmware
2849 * version checks will be different. We set up the comparison values
2850 * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware
2851 * running in tavor mode).
2852 */
2853 switch (state->ts_operational_mode) {
2854 case TAVOR_HCA_MODE:
2855 tavor_fw_ver_major = TAVOR_FW_VER_MAJOR;
2856 tavor_fw_ver_minor = TAVOR_FW_VER_MINOR;
2857 tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR;
2858 break;
2859
2860 case TAVOR_COMPAT_MODE:
2861 tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR;
2862 tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR;
2863 tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR;
2864 break;
2865
2866 default:
2867 return (DDI_FAILURE);
2868 }
2869
2870 /*
2871 * If FW revision major number is less than acceptable,
2872 * return failure, else if greater return success. If
2873 * the major numbers are equal than check the minor number
2874 */
2875 if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) {
2876 return (DDI_FAILURE);
2877 } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) {
2878 return (DDI_SUCCESS);
2879 }
2880 /*
2881 * Do the same check as above, except for minor revision numbers
2882 * If the minor numbers are equal than check the subminor number
2883 */
2884 if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) {
2885 return (DDI_FAILURE);
2886 } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) {
2887 return (DDI_SUCCESS);
2888 }
2889
2890 /*
2891 * Once again we do the same check as above, except for the subminor
2892 * revision number. If the subminor numbers are equal here, then
2893 * these are the same firmware version, return success
2894 */
2895 if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) {
2896 return (DDI_FAILURE);
2897 } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) {
2898 return (DDI_SUCCESS);
2899 }
2900
2901 return (DDI_SUCCESS);
2902 }
2903
2904
2905 /*
2906 * tavor_device_info_report()
2907 * Context: Only called from attach() path context
2908 */
2909 static void
2910 tavor_device_info_report(tavor_state_t *state)
2911 {
2912 cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, "
2913 "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major,
2914 state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor,
2915 state->ts_adapter.rev_id);
2916 cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n",
2917 state->ts_instance, state->ts_nodedesc, state->ts_nodeguid);
2918 }
2919
2920
2921 /*
2922 * tavor_pci_capability_list()
2923 * Context: Only called from attach() path context
2924 */
2925 static void
2926 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl)
2927 {
2928 uint_t offset, data;
2929
2930 TAVOR_TNF_ENTER(tavor_pci_capability_list);
2931
2932 /*
2933 * Check for the "PCI Capabilities" bit in the "Status Register".
2934 * Bit 4 in this register indicates the presence of a "PCI
2935 * Capabilities" list.
2936 */
2937 data = pci_config_get16(hdl, 0x6);
2938 if ((data & 0x10) == 0) {
2939 TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, "");
2940 TAVOR_TNF_EXIT(tavor_pci_capability_list);
2941 return;
2942 }
2943
2944 /*
2945 * Starting from offset 0x34 in PCI config space, find the
2946 * head of "PCI capabilities" list, and walk the list. If
2947 * capabilities of a known type are encountered (e.g.
2948 * "PCI-X Capability"), then call the appropriate handler
2949 * function.
2950 */
2951 offset = pci_config_get8(hdl, 0x34);
2952 while (offset != 0x0) {
2953 data = pci_config_get8(hdl, offset);
2954
2955 /*
2956 * Check for known capability types. Tavor has the
2957 * following:
2958 * o VPD Capability (0x03)
2959 * o PCI-X Capability (0x07)
2960 * o MSI Capability (0x05)
2961 * o MSIX Capability (0x11)
2962 */
2963 switch (data) {
2964 case 0x03:
2965 tavor_pci_capability_vpd(state, hdl, offset);
2966 break;
2967 case 0x07:
2968 tavor_pci_capability_pcix(state, hdl, offset);
2969 break;
2970 case 0x05:
2971 break;
2972 default:
2973 break;
2974 }
2975
2976 /* Get offset of next entry in list */
2977 offset = pci_config_get8(hdl, offset + 1);
2978 }
2979
2980 TAVOR_TNF_EXIT(tavor_pci_capability_list);
2981 }
2982
2983 /*
2984 * tavor_pci_read_vpd()
2985 * Context: Only called from attach() path context
2986 * utility routine for tavor_pci_capability_vpd()
2987 */
2988 static int
2989 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr,
2990 uint32_t *data)
2991 {
2992 int retry = 4; /* retry counter for EEPROM poll */
2993 uint32_t val;
2994 int vpd_addr = offset + 2;
2995 int vpd_data = offset + 4;
2996
2997 TAVOR_TNF_ENTER(tavor_pci_read_vpd);
2998
2999 /*
3000 * In order to read a 32-bit value from VPD, we are to write down
3001 * the address (offset in the VPD itself) to the address register.
3002 * To signal the read, we also clear bit 31. We then poll on bit 31
3003 * and when it is set, we can then read our 4 bytes from the data
3004 * register.
3005 */
3006 (void) pci_config_put32(hdl, offset, addr << 16);
3007 do {
3008 drv_usecwait(1000);
3009 val = pci_config_get16(hdl, vpd_addr);
3010 if ((val >> 15) & 0x01) {
3011 *data = pci_config_get32(hdl, vpd_data);
3012 TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3013 return (DDI_SUCCESS);
3014 }
3015 } while (--retry);
3016
3017 TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, "");
3018 TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3019 return (DDI_FAILURE);
3020 }
3021
3022
3023 /*
3024 * tavor_pci_capability_vpd()
3025 * Context: Only called from attach() path context
3026 */
3027 static void
3028 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl,
3029 uint_t offset)
3030 {
3031 uint8_t name_length;
3032 uint8_t pn_length;
3033 int i, err = 0;
3034 int vpd_str_id = 0;
3035 int vpd_ro_desc;
3036 int vpd_ro_pn_desc;
3037 #ifndef _LITTLE_ENDIAN
3038 uint32_t data32;
3039 #endif /* _LITTLE_ENDIAN */
3040 union {
3041 uint32_t vpd_int[TAVOR_VPD_HDR_DWSIZE];
3042 uchar_t vpd_char[TAVOR_VPD_HDR_BSIZE];
3043 } vpd;
3044
3045 TAVOR_TNF_ENTER(tavor_pci_capability_vpd);
3046
3047 /*
3048 * Read Vital Product Data (VPD) from PCI-X capability.
3049 */
3050 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3051 err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]);
3052 if (err != DDI_SUCCESS) {
3053 cmn_err(CE_NOTE, "!VPD read failed\n");
3054 goto out;
3055 }
3056 }
3057
3058 #ifndef _LITTLE_ENDIAN
3059 /*
3060 * Need to swap bytes for big endian.
3061 */
3062 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3063 data32 = vpd.vpd_int[i];
3064 vpd.vpd_char[(i << 2) + 3] =
3065 (uchar_t)((data32 & 0xFF000000) >> 24);
3066 vpd.vpd_char[(i << 2) + 2] =
3067 (uchar_t)((data32 & 0x00FF0000) >> 16);
3068 vpd.vpd_char[(i << 2) + 1] =
3069 (uchar_t)((data32 & 0x0000FF00) >> 8);
3070 vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF);
3071 }
3072 #endif /* _LITTLE_ENDIAN */
3073
3074 /* Check for VPD String ID Tag */
3075 if (vpd.vpd_char[vpd_str_id] == 0x82) {
3076 /* get the product name */
3077 name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1];
3078 if (name_length > sizeof (state->ts_hca_name)) {
3079 cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n",
3080 name_length);
3081 goto out;
3082 }
3083 (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3],
3084 name_length);
3085 state->ts_hca_name[name_length] = 0;
3086
3087 /* get the part number */
3088 vpd_ro_desc = name_length + 3; /* read-only tag location */
3089 vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */
3090 /*
3091 * Verify read-only tag and Part Number keyword.
3092 */
3093 if (vpd.vpd_char[vpd_ro_desc] != 0x90 ||
3094 (vpd.vpd_char[vpd_ro_pn_desc] != 'P' &&
3095 vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) {
3096 cmn_err(CE_NOTE, "!VPD Part Number not found\n");
3097 goto out;
3098 }
3099
3100 pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2];
3101 if (pn_length > sizeof (state->ts_hca_pn)) {
3102 cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n",
3103 name_length);
3104 goto out;
3105 }
3106 (void) memcpy(state->ts_hca_pn,
3107 &vpd.vpd_char[vpd_ro_pn_desc + 3],
3108 pn_length);
3109 state->ts_hca_pn[pn_length] = 0;
3110 state->ts_hca_pn_len = pn_length;
3111 } else {
3112 /* Wrong VPD String ID Tag */
3113 cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n",
3114 vpd.vpd_char[0]);
3115 goto out;
3116 }
3117 TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3118 return;
3119 out:
3120 state->ts_hca_pn_len = 0;
3121 TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, "");
3122 TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3123 }
3124
3125 /*
3126 * tavor_pci_capability_pcix()
3127 * Context: Only called from attach() path context
3128 */
3129 static void
3130 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl,
3131 uint_t offset)
3132 {
3133 uint_t command, status;
3134 int max_out_splt_trans, max_mem_rd_byte_cnt;
3135 int designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt;
3136
3137 TAVOR_TNF_ENTER(tavor_pci_capability_pcix);
3138
3139 /*
3140 * Query the current values for the PCI-X Command Register and
3141 * the PCI-X Status Register.
3142 */
3143 command = pci_config_get16(hdl, offset + 2);
3144 status = pci_config_get32(hdl, offset + 4);
3145
3146 /*
3147 * Check for config property specifying "maximum outstanding
3148 * split transactions". If the property is defined and valid
3149 * (i.e. no larger than the so-called "designed maximum"),
3150 * then use the specified value to update the PCI-X Command Register.
3151 * Otherwise, extract the value from the Tavor config profile.
3152 */
3153 designed_max_out_splt_trans = ((status >> 23) & 7);
3154 max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3155 DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1);
3156 if ((max_out_splt_trans != -1) &&
3157 ((max_out_splt_trans < 0) ||
3158 (max_out_splt_trans > designed_max_out_splt_trans))) {
3159 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-"
3160 "split-trans\" (%d) invalid or exceeds device maximum"
3161 " (%d), using default value (%d)\n", state->ts_instance,
3162 max_out_splt_trans, designed_max_out_splt_trans,
3163 state->ts_cfg_profile->cp_max_out_splt_trans);
3164 max_out_splt_trans =
3165 state->ts_cfg_profile->cp_max_out_splt_trans;
3166 } else if (max_out_splt_trans == -1) {
3167 max_out_splt_trans =
3168 state->ts_cfg_profile->cp_max_out_splt_trans;
3169 }
3170
3171 /*
3172 * The config profile setting for max_out_splt_trans is determined
3173 * based on arch. Check tavor_cfg.c for more information. A value of
3174 * '-1' in the patchable variable means "do not change". A value of
3175 * '0' means 1 outstanding splt trans and other values as defined by
3176 * PCI. So we do one more check here, that if 'max_out_splt_trans' is
3177 * -1 (ie: < 0) we do not set the PCI command and leave it at the
3178 * default.
3179 */
3180 if (max_out_splt_trans >= 0) {
3181 command = ((command & 0xFF8F) | max_out_splt_trans << 4);
3182 }
3183
3184 /*
3185 * Check for config property specifying "maximum memory read
3186 * byte count. If the property is defined and valid
3187 * (i.e. no larger than the so-called "designed maximum"),
3188 * then use the specified value to update the PCI-X Command Register.
3189 * Otherwise, extract the value from the Tavor config profile.
3190 */
3191 designed_max_mem_rd_byte_cnt = ((status >> 21) & 3);
3192 max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3193 DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1);
3194 if ((max_mem_rd_byte_cnt != -1) &&
3195 ((max_mem_rd_byte_cnt < 0) ||
3196 (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) {
3197 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-"
3198 "count\" (%d) invalid or exceeds device maximum"
3199 " (%d), using default value (%d)\n", state->ts_instance,
3200 max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt,
3201 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt);
3202 max_mem_rd_byte_cnt =
3203 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3204 } else if (max_mem_rd_byte_cnt == -1) {
3205 max_mem_rd_byte_cnt =
3206 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3207 }
3208
3209 /*
3210 * The config profile setting for max_mem_rd_byte_cnt is determined
3211 * based on arch. Check tavor_cfg.c for more information. A value of
3212 * '-1' in the patchable variable means "do not change". A value of
3213 * '0' means minimum (512B) read, and other values as defined by
3214 * PCI. So we do one more check here, that if 'max_mem_rd_byte_cnt' is
3215 * -1 (ie: < 0) we do not set the PCI command and leave it at the
3216 * default.
3217 */
3218 if (max_mem_rd_byte_cnt >= 0) {
3219 command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2);
3220 }
3221
3222 /*
3223 * Update the PCI-X Command Register with the newly configured
3224 * values.
3225 */
3226 pci_config_put16(hdl, offset + 2, command);
3227
3228 TAVOR_TNF_EXIT(tavor_pci_capability_pcix);
3229 }
3230
3231
3232 /*
3233 * tavor_intr_or_msi_init()
3234 * Context: Only called from attach() path context
3235 */
3236 static int
3237 tavor_intr_or_msi_init(tavor_state_t *state)
3238 {
3239 int status;
3240
3241 TAVOR_TNF_ENTER(tavor_intr_or_msi_init);
3242
3243 /* Query for the list of supported interrupt event types */
3244 status = ddi_intr_get_supported_types(state->ts_dip,
3245 &state->ts_intr_types_avail);
3246 if (status != DDI_SUCCESS) {
3247 TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail,
3248 TAVOR_TNF_ERROR, "");
3249 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3250 return (DDI_FAILURE);
3251 }
3252
3253 /*
3254 * If Tavor/Arbel supports MSI in this system (and, if it
3255 * hasn't been overridden by a configuration variable), then
3256 * the default behavior is to use a single MSI. Otherwise,
3257 * fallback to using legacy interrupts. Also, if MSI allocatis chosen,
3258 * but fails for whatever reasons, then fallback to using legacy
3259 * interrupts.
3260 */
3261 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3262 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3263 status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI);
3264 if (status == DDI_SUCCESS) {
3265 state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI;
3266 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3267 return (DDI_SUCCESS);
3268 }
3269 }
3270
3271 /*
3272 * MSI interrupt allocation failed, or was not available. Fallback to
3273 * legacy interrupt support.
3274 */
3275 if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) {
3276 status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED);
3277 if (status == DDI_SUCCESS) {
3278 state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED;
3279 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3280 return (DDI_SUCCESS);
3281 }
3282 }
3283
3284 /*
3285 * Neither MSI or legacy interrupts were successful. return failure.
3286 */
3287 TAVOR_TNF_EXIT(tavor_intr_or_msi_setup);
3288 return (DDI_FAILURE);
3289 }
3290
3291 /*
3292 * tavor_add_intrs()
3293 * Context: Only called from attach() patch context
3294 */
3295 static int
3296 tavor_add_intrs(tavor_state_t *state, int intr_type)
3297 {
3298 int status;
3299
3300 TAVOR_TNF_ENTER(tavor_add_intrs);
3301
3302 /* Get number of interrupts/MSI supported */
3303 status = ddi_intr_get_nintrs(state->ts_dip, intr_type,
3304 &state->ts_intrmsi_count);
3305 if (status != DDI_SUCCESS) {
3306 TNF_PROBE_0(tavor_add_intrs_getnintrs_fail,
3307 TAVOR_TNF_ERROR, "");
3308 TAVOR_TNF_EXIT(tavor_add_intrs);
3309 return (DDI_FAILURE);
3310 }
3311
3312 /* Get number of available interrupts/MSI */
3313 status = ddi_intr_get_navail(state->ts_dip, intr_type,
3314 &state->ts_intrmsi_avail);
3315 if (status != DDI_SUCCESS) {
3316 TNF_PROBE_0(tavor_add_intrs_getnavail_fail,
3317 TAVOR_TNF_ERROR, "");
3318 TAVOR_TNF_EXIT(tavor_add_intrs);
3319 return (DDI_FAILURE);
3320 }
3321
3322 /* Ensure that we have at least one (1) usable MSI or interrupt */
3323 if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) {
3324 TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail,
3325 TAVOR_TNF_ERROR, "");
3326 TAVOR_TNF_EXIT(tavor_add_intrs);
3327 return (DDI_FAILURE);
3328 }
3329
3330 /* Attempt to allocate a single interrupt/MSI handle */
3331 status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl,
3332 intr_type, 0, 1, &state->ts_intrmsi_allocd,
3333 DDI_INTR_ALLOC_STRICT);
3334 if (status != DDI_SUCCESS) {
3335 TNF_PROBE_0(tavor_add_intrs_intralloc_fail,
3336 TAVOR_TNF_ERROR, "");
3337 TAVOR_TNF_EXIT(tavor_add_intrs);
3338 return (DDI_FAILURE);
3339 }
3340
3341 /* Ensure that we have allocated at least one (1) MSI or interrupt */
3342 if (state->ts_intrmsi_allocd < 1) {
3343 TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail,
3344 TAVOR_TNF_ERROR, "");
3345 TAVOR_TNF_EXIT(tavor_add_intrs);
3346 return (DDI_FAILURE);
3347 }
3348
3349 /*
3350 * Extract the priority for the allocated interrupt/MSI. This
3351 * will be used later when initializing certain mutexes.
3352 */
3353 status = ddi_intr_get_pri(state->ts_intrmsi_hdl,
3354 &state->ts_intrmsi_pri);
3355 if (status != DDI_SUCCESS) {
3356 /* Free the allocated interrupt/MSI handle */
3357 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3358
3359 TNF_PROBE_0(tavor_add_intrs_getpri_fail,
3360 TAVOR_TNF_ERROR, "");
3361 TAVOR_TNF_EXIT(tavor_add_intrs);
3362 return (DDI_FAILURE);
3363 }
3364
3365 /* Make sure the interrupt/MSI priority is below 'high level' */
3366 if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) {
3367 /* Free the allocated interrupt/MSI handle */
3368 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3369
3370 TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail,
3371 TAVOR_TNF_ERROR, "");
3372 TAVOR_TNF_EXIT(tavor_add_intrs);
3373 return (DDI_FAILURE);
3374 }
3375
3376 /* Get add'l capability information regarding interrupt/MSI */
3377 status = ddi_intr_get_cap(state->ts_intrmsi_hdl,
3378 &state->ts_intrmsi_cap);
3379 if (status != DDI_SUCCESS) {
3380 /* Free the allocated interrupt/MSI handle */
3381 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3382
3383 TNF_PROBE_0(tavor_add_intrs_getcap_fail,
3384 TAVOR_TNF_ERROR, "");
3385 TAVOR_TNF_EXIT(tavor_add_intrs);
3386 return (DDI_FAILURE);
3387 }
3388
3389 TAVOR_TNF_EXIT(tavor_add_intrs);
3390 return (DDI_SUCCESS);
3391 }
3392
3393
3394 /*
3395 * tavor_intr_or_msi_fini()
3396 * Context: Only called from attach() and/or detach() path contexts
3397 */
3398 static int
3399 tavor_intr_or_msi_fini(tavor_state_t *state)
3400 {
3401 int status;
3402
3403 TAVOR_TNF_ENTER(tavor_intr_or_msi_fini);
3404
3405 /* Free the allocated interrupt/MSI handle */
3406 status = ddi_intr_free(state->ts_intrmsi_hdl);
3407 if (status != DDI_SUCCESS) {
3408 TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail,
3409 TAVOR_TNF_ERROR, "");
3410 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3411 return (DDI_FAILURE);
3412 }
3413
3414 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3415 return (DDI_SUCCESS);
3416 }
3417
3418
3419 /* Disable Tavor interrupts */
3420 static int
3421 tavor_intr_disable(tavor_state_t *state)
3422 {
3423 ushort_t msi_ctrl = 0, caps_ctrl = 0;
3424 ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl;
3425 ASSERT(pci_cfg_hdl != NULL);
3426 ASSERT(state->ts_intr_types_avail &
3427 (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI));
3428
3429 /*
3430 * Check if MSI interrupts are used. If so, disable MSI interupts.
3431 * If not, since Tavor doesn't support MSI-X interrupts, assuming the
3432 * legacy interrupt is used instead, disable the legacy interrupt.
3433 */
3434 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3435 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3436
3437 if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI,
3438 &caps_ctrl) == DDI_SUCCESS)) {
3439 if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL,
3440 caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16)
3441 return (DDI_FAILURE);
3442 }
3443 ASSERT(msi_ctrl != 0);
3444
3445 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
3446 return (DDI_SUCCESS);
3447
3448 if (msi_ctrl & PCI_MSI_PVM_MASK) {
3449 int offset = (msi_ctrl & PCI_MSI_64BIT_MASK) ?
3450 PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK;
3451
3452 /* Clear all inums in MSI */
3453 PCI_CAP_PUT32(pci_cfg_hdl, NULL, caps_ctrl,
3454 offset, 0x0);
3455 }
3456
3457 /* Disable MSI interrupts */
3458 msi_ctrl &= ~PCI_MSI_ENABLE_BIT;
3459 PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSI_CTRL,
3460 msi_ctrl);
3461
3462 } else {
3463 uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM);
3464 ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED);
3465
3466 /* Disable the legacy interrupts */
3467 cmdreg |= PCI_COMM_INTX_DISABLE;
3468 pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg);
3469 }
3470
3471 return (DDI_SUCCESS);
3472 }
3473
3474 /* Tavor quiesce(9F) entry */
3475 static int
3476 tavor_quiesce(dev_info_t *dip)
3477 {
3478 tavor_state_t *state = ddi_get_soft_state(tavor_statep,
3479 DEVI(dip)->devi_instance);
3480 ASSERT(state != NULL);
3481
3482 /* start fastreboot */
3483 state->ts_quiescing = B_TRUE;
3484
3485 /* If it's in maintenance mode, do nothing but return with SUCCESS */
3486 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
3487 return (DDI_SUCCESS);
3488 }
3489
3490 /* Shutdown HCA ports */
3491 if (tavor_hca_ports_shutdown(state,
3492 state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) {
3493 state->ts_quiescing = B_FALSE;
3494 return (DDI_FAILURE);
3495 }
3496
3497 /* Close HCA */
3498 if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3499 TAVOR_CMD_SUCCESS) {
3500 state->ts_quiescing = B_FALSE;
3501 return (DDI_FAILURE);
3502 }
3503
3504 /* Shutdown FW */
3505 if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3506 TAVOR_CMD_SUCCESS) {
3507 state->ts_quiescing = B_FALSE;
3508 return (DDI_FAILURE);
3509 }
3510
3511 /* Disable interrupts */
3512 if (tavor_intr_disable(state) != DDI_SUCCESS) {
3513 state->ts_quiescing = B_FALSE;
3514 return (DDI_FAILURE);
3515 }
3516
3517 /* SW-reset */
3518 if (tavor_sw_reset(state) != DDI_SUCCESS) {
3519 state->ts_quiescing = B_FALSE;
3520 return (DDI_FAILURE);
3521 }
3522
3523 return (DDI_SUCCESS);
3524 }