1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * tavor.c
  28  *    Tavor (InfiniBand) HCA Driver attach/detach Routines
  29  *
  30  *    Implements all the routines necessary for the attach, setup,
  31  *    initialization (and subsequent possible teardown and detach) of the
  32  *    Tavor InfiniBand HCA driver.
  33  */
  34 
  35 #include <sys/types.h>
  36 #include <sys/file.h>
  37 #include <sys/open.h>
  38 #include <sys/conf.h>
  39 #include <sys/ddi.h>
  40 #include <sys/sunddi.h>
  41 #include <sys/modctl.h>
  42 #include <sys/stat.h>
  43 #include <sys/pci.h>
  44 #include <sys/pci_cap.h>
  45 #include <sys/bitmap.h>
  46 #include <sys/policy.h>
  47 
  48 #include <sys/ib/adapters/tavor/tavor.h>
  49 #include <sys/pci.h>
  50 
  51 /* Tavor HCA State Pointer */
  52 void *tavor_statep;
  53 
  54 /*
  55  * The Tavor "userland resource database" is common to instances of the
  56  * Tavor HCA driver.  This structure "tavor_userland_rsrc_db" contains all
  57  * the necessary information to maintain it.
  58  */
  59 tavor_umap_db_t tavor_userland_rsrc_db;
  60 
  61 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t);
  62 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t);
  63 static int tavor_open(dev_t *, int, int, cred_t *);
  64 static int tavor_close(dev_t, int, int, cred_t *);
  65 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
  66 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance);
  67 static void tavor_drv_fini(tavor_state_t *state);
  68 static void tavor_drv_fini2(tavor_state_t *state);
  69 static int tavor_isr_init(tavor_state_t *state);
  70 static void tavor_isr_fini(tavor_state_t *state);
  71 static int tavor_hw_init(tavor_state_t *state);
  72 static void tavor_hw_fini(tavor_state_t *state,
  73     tavor_drv_cleanup_level_t cleanup);
  74 static int tavor_soft_state_init(tavor_state_t *state);
  75 static void tavor_soft_state_fini(tavor_state_t *state);
  76 static int tavor_hca_port_init(tavor_state_t *state);
  77 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init);
  78 static void tavor_hca_config_setup(tavor_state_t *state,
  79     tavor_hw_initqueryhca_t *inithca);
  80 static int tavor_internal_uarpgs_init(tavor_state_t *state);
  81 static void tavor_internal_uarpgs_fini(tavor_state_t *state);
  82 static int tavor_special_qp_contexts_reserve(tavor_state_t *state);
  83 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state);
  84 static int tavor_sw_reset(tavor_state_t *state);
  85 static int tavor_mcg_init(tavor_state_t *state);
  86 static void tavor_mcg_fini(tavor_state_t *state);
  87 static int tavor_fw_version_check(tavor_state_t *state);
  88 static void tavor_device_info_report(tavor_state_t *state);
  89 static void tavor_pci_capability_list(tavor_state_t *state,
  90     ddi_acc_handle_t hdl);
  91 static void tavor_pci_capability_vpd(tavor_state_t *state,
  92     ddi_acc_handle_t hdl, uint_t offset);
  93 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset,
  94     uint32_t addr, uint32_t *data);
  95 static void tavor_pci_capability_pcix(tavor_state_t *state,
  96     ddi_acc_handle_t hdl, uint_t offset);
  97 static int tavor_intr_or_msi_init(tavor_state_t *state);
  98 static int tavor_add_intrs(tavor_state_t *state, int intr_type);
  99 static int tavor_intr_or_msi_fini(tavor_state_t *state);
 100 
 101 /* X86 fastreboot support */
 102 static int tavor_intr_disable(tavor_state_t *);
 103 static int tavor_quiesce(dev_info_t *);
 104 
 105 /* Character/Block Operations */
 106 static struct cb_ops tavor_cb_ops = {
 107         tavor_open,             /* open */
 108         tavor_close,            /* close */
 109         nodev,                  /* strategy (block) */
 110         nodev,                  /* print (block) */
 111         nodev,                  /* dump (block) */
 112         nodev,                  /* read */
 113         nodev,                  /* write */
 114         tavor_ioctl,            /* ioctl */
 115         tavor_devmap,           /* devmap */
 116         NULL,                   /* mmap */
 117         nodev,                  /* segmap */
 118         nochpoll,               /* chpoll */
 119         ddi_prop_op,            /* prop_op */
 120         NULL,                   /* streams */
 121         D_NEW | D_MP |
 122         D_64BIT | D_HOTPLUG |
 123         D_DEVMAP,               /* flags */
 124         CB_REV                  /* rev */
 125 };
 126 
 127 /* Driver Operations */
 128 static struct dev_ops tavor_ops = {
 129         DEVO_REV,               /* struct rev */
 130         0,                      /* refcnt */
 131         tavor_getinfo,          /* getinfo */
 132         nulldev,                /* identify */
 133         nulldev,                /* probe */
 134         tavor_attach,           /* attach */
 135         tavor_detach,           /* detach */
 136         nodev,                  /* reset */
 137         &tavor_cb_ops,              /* cb_ops */
 138         NULL,                   /* bus_ops */
 139         nodev,                  /* power */
 140         tavor_quiesce,          /* devo_quiesce */
 141 };
 142 
 143 /* Module Driver Info */
 144 static struct modldrv tavor_modldrv = {
 145         &mod_driverops,
 146         "Tavor InfiniBand HCA Driver",
 147         &tavor_ops
 148 };
 149 
 150 /* Module Linkage */
 151 static struct modlinkage tavor_modlinkage = {
 152         MODREV_1,
 153         &tavor_modldrv,
 154         NULL
 155 };
 156 
 157 /*
 158  * This extern refers to the ibc_operations_t function vector that is defined
 159  * in the tavor_ci.c file.
 160  */
 161 extern ibc_operations_t tavor_ibc_ops;
 162 
 163 #ifndef NPROBE
 164 extern int tnf_mod_load(void);
 165 extern int tnf_mod_unload(struct modlinkage *mlp);
 166 #endif
 167 
 168 
 169 /*
 170  * _init()
 171  */
 172 int
 173 _init()
 174 {
 175         int     status;
 176 
 177 #ifndef NPROBE
 178         (void) tnf_mod_load();
 179 #endif
 180         TAVOR_TNF_ENTER(tavor_init);
 181 
 182         status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t),
 183             (size_t)TAVOR_INITIAL_STATES);
 184         if (status != 0) {
 185                 TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, "");
 186                 TAVOR_TNF_EXIT(tavor_init);
 187 #ifndef NPROBE
 188                 (void) tnf_mod_unload(&tavor_modlinkage);
 189 #endif
 190                 return (status);
 191         }
 192 
 193         status = ibc_init(&tavor_modlinkage);
 194         if (status != 0) {
 195                 TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, "");
 196                 ddi_soft_state_fini(&tavor_statep);
 197                 TAVOR_TNF_EXIT(tavor_init);
 198 #ifndef NPROBE
 199                 (void) tnf_mod_unload(&tavor_modlinkage);
 200 #endif
 201                 return (status);
 202         }
 203         status = mod_install(&tavor_modlinkage);
 204         if (status != 0) {
 205                 TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, "");
 206                 ibc_fini(&tavor_modlinkage);
 207                 ddi_soft_state_fini(&tavor_statep);
 208                 TAVOR_TNF_EXIT(tavor_init);
 209 #ifndef NPROBE
 210                 (void) tnf_mod_unload(&tavor_modlinkage);
 211 #endif
 212                 return (status);
 213         }
 214 
 215         /* Initialize the Tavor "userland resources database" */
 216         tavor_umap_db_init();
 217 
 218         TAVOR_TNF_EXIT(tavor_init);
 219         return (status);
 220 }
 221 
 222 
 223 /*
 224  * _info()
 225  */
 226 int
 227 _info(struct modinfo *modinfop)
 228 {
 229         int     status;
 230 
 231         TAVOR_TNF_ENTER(tavor_info);
 232         status = mod_info(&tavor_modlinkage, modinfop);
 233         TAVOR_TNF_EXIT(tavor_info);
 234         return (status);
 235 }
 236 
 237 
 238 /*
 239  * _fini()
 240  */
 241 int
 242 _fini()
 243 {
 244         int     status;
 245 
 246         TAVOR_TNF_ENTER(tavor_fini);
 247 
 248         status = mod_remove(&tavor_modlinkage);
 249         if (status != 0) {
 250                 TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, "");
 251                 TAVOR_TNF_EXIT(tavor_fini);
 252                 return (status);
 253         }
 254 
 255         /* Destroy the Tavor "userland resources database" */
 256         tavor_umap_db_fini();
 257 
 258         ibc_fini(&tavor_modlinkage);
 259         ddi_soft_state_fini(&tavor_statep);
 260 #ifndef NPROBE
 261         (void) tnf_mod_unload(&tavor_modlinkage);
 262 #endif
 263         TAVOR_TNF_EXIT(tavor_fini);
 264         return (status);
 265 }
 266 
 267 
 268 /*
 269  * tavor_getinfo()
 270  */
 271 /* ARGSUSED */
 272 static int
 273 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
 274 {
 275         dev_t           dev;
 276         tavor_state_t   *state;
 277         minor_t         instance;
 278 
 279         TAVOR_TNF_ENTER(tavor_getinfo);
 280 
 281         switch (cmd) {
 282         case DDI_INFO_DEVT2DEVINFO:
 283                 dev = (dev_t)arg;
 284                 instance = TAVOR_DEV_INSTANCE(dev);
 285                 state = ddi_get_soft_state(tavor_statep, instance);
 286                 if (state == NULL) {
 287                         TNF_PROBE_0(tavor_getinfo_gss_fail,
 288                             TAVOR_TNF_ERROR, "");
 289                         TAVOR_TNF_EXIT(tavor_getinfo);
 290                         return (DDI_FAILURE);
 291                 }
 292                 *result = (void *)state->ts_dip;
 293                 return (DDI_SUCCESS);
 294 
 295         case DDI_INFO_DEVT2INSTANCE:
 296                 dev = (dev_t)arg;
 297                 instance = TAVOR_DEV_INSTANCE(dev);
 298                 *result = (void *)(uintptr_t)instance;
 299                 return (DDI_SUCCESS);
 300 
 301         default:
 302                 TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, "");
 303                 break;
 304         }
 305 
 306         TAVOR_TNF_EXIT(tavor_getinfo);
 307         return (DDI_FAILURE);
 308 }
 309 
 310 
 311 /*
 312  * tavor_open()
 313  */
 314 /* ARGSUSED */
 315 static int
 316 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp)
 317 {
 318         tavor_state_t           *state;
 319         tavor_rsrc_t            *rsrcp;
 320         tavor_umap_db_entry_t   *umapdb, *umapdb2;
 321         minor_t                 instance;
 322         uint64_t                key, value;
 323         uint_t                  tr_indx;
 324         dev_t                   dev;
 325         int                     status;
 326 
 327         TAVOR_TNF_ENTER(tavor_open);
 328 
 329         instance = TAVOR_DEV_INSTANCE(*devp);
 330         state = ddi_get_soft_state(tavor_statep, instance);
 331         if (state == NULL) {
 332                 TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, "");
 333                 TAVOR_TNF_EXIT(tavor_open);
 334                 return (ENXIO);
 335         }
 336 
 337         /*
 338          * Only allow driver to be opened for character access, and verify
 339          * whether exclusive access is allowed.
 340          */
 341         if ((otyp != OTYP_CHR) || ((flag & FEXCL) &&
 342             secpolicy_excl_open(credp) != 0)) {
 343                 TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, "");
 344                 TAVOR_TNF_EXIT(tavor_open);
 345                 return (EINVAL);
 346         }
 347 
 348         /*
 349          * Search for the current process PID in the "userland resources
 350          * database".  If it is not found, then attempt to allocate a UAR
 351          * page and add the ("key", "value") pair to the database.
 352          * Note:  As a last step we always return a devp appropriate for
 353          * the open.  Either we return a new minor number (based on the
 354          * instance and the UAR page index) or we return the current minor
 355          * number for the given client process.
 356          *
 357          * We also add an entry to the database to allow for lookup from
 358          * "dev_t" to the current process PID.  This is necessary because,
 359          * under certain circumstance, the process PID that calls the Tavor
 360          * close() entry point may not be the same as the one who called
 361          * open().  Specifically, this can happen if a child process calls
 362          * the Tavor's open() entry point, gets a UAR page, maps it out (using
 363          * mmap()), and then exits without calling munmap().  Because mmap()
 364          * adds a reference to the file descriptor, at the exit of the child
 365          * process the file descriptor is "inherited" by the parent (and will
 366          * be close()'d by the parent's PID only when it exits).
 367          *
 368          * Note: We use the tavor_umap_db_find_nolock() and
 369          * tavor_umap_db_add_nolock() database access routines below (with
 370          * an explicit mutex_enter of the database lock - "tdl_umapdb_lock")
 371          * to ensure that the multiple accesses (in this case searching for,
 372          * and then adding _two_ database entries) can be done atomically.
 373          */
 374         key = ddi_get_pid();
 375         mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
 376         status = tavor_umap_db_find_nolock(instance, key,
 377             MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
 378         if (status != DDI_SUCCESS) {
 379                 /*
 380                  * If we are in 'maintenance mode', we cannot alloc a UAR page.
 381                  * But we still need some rsrcp value, and a mostly unique
 382                  * tr_indx value.  So we set rsrcp to NULL for maintenance
 383                  * mode, and use a rolling count for tr_indx.  The field
 384                  * 'ts_open_tr_indx' is used only in this maintenance mode
 385                  * condition.
 386                  *
 387                  * Otherwise, if we are in operational mode then we allocate
 388                  * the UAR page as normal, and use the rsrcp value and tr_indx
 389                  * value from that allocation.
 390                  */
 391                 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 392                         rsrcp = NULL;
 393                         tr_indx = state->ts_open_tr_indx++;
 394                 } else {
 395                         /* Allocate a new UAR page for this process */
 396                         status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1,
 397                             TAVOR_NOSLEEP, &rsrcp);
 398                         if (status != DDI_SUCCESS) {
 399                                 mutex_exit(
 400                                     &tavor_userland_rsrc_db.tdl_umapdb_lock);
 401                                 TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail,
 402                                     TAVOR_TNF_ERROR, "");
 403                                 TAVOR_TNF_EXIT(tavor_open);
 404                                 return (EAGAIN);
 405                         }
 406 
 407                         tr_indx = rsrcp->tr_indx;
 408                 }
 409 
 410                 /*
 411                  * Allocate an entry to track the UAR page resource in the
 412                  * "userland resources database".
 413                  */
 414                 umapdb = tavor_umap_db_alloc(instance, key,
 415                     MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp);
 416                 if (umapdb == NULL) {
 417                         mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
 418                         /* If in "maintenance mode", don't free the rsrc */
 419                         if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 420                                 tavor_rsrc_free(state, &rsrcp);
 421                         }
 422                         TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
 423                             TAVOR_TNF_ERROR, "");
 424                         TAVOR_TNF_EXIT(tavor_open);
 425                         return (EAGAIN);
 426                 }
 427 
 428                 /*
 429                  * Create a new device number.  Minor number is a function of
 430                  * the UAR page index (15 bits) and the device instance number
 431                  * (3 bits).
 432                  */
 433                 dev = makedevice(getmajor(*devp), (tr_indx <<
 434                     TAVOR_MINORNUM_SHIFT) | instance);
 435 
 436                 /*
 437                  * Allocate another entry in the "userland resources database"
 438                  * to track the association of the device number (above) to
 439                  * the current process ID (in "key").
 440                  */
 441                 umapdb2 = tavor_umap_db_alloc(instance, dev,
 442                     MLNX_UMAP_PID_RSRC, (uint64_t)key);
 443                 if (umapdb2 == NULL) {
 444                         mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
 445                         tavor_umap_db_free(umapdb);
 446                         /* If in "maintenance mode", don't free the rsrc */
 447                         if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 448                                 tavor_rsrc_free(state, &rsrcp);
 449                         }
 450                         TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
 451                             TAVOR_TNF_ERROR, "");
 452                         TAVOR_TNF_EXIT(tavor_open);
 453                         return (EAGAIN);
 454                 }
 455 
 456                 /* Add the entries to the database */
 457                 tavor_umap_db_add_nolock(umapdb);
 458                 tavor_umap_db_add_nolock(umapdb2);
 459 
 460         } else {
 461                 /*
 462                  * Return the same device number as on the original open()
 463                  * call.  This was calculated as a function of the UAR page
 464                  * index (top 16 bits) and the device instance number
 465                  */
 466                 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
 467                 dev = makedevice(getmajor(*devp), (rsrcp->tr_indx <<
 468                     TAVOR_MINORNUM_SHIFT) | instance);
 469         }
 470         mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
 471 
 472         *devp = dev;
 473 
 474         TAVOR_TNF_EXIT(tavor_open);
 475         return (0);
 476 }
 477 
 478 
 479 /*
 480  * tavor_close()
 481  */
 482 /* ARGSUSED */
 483 static int
 484 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp)
 485 {
 486         tavor_state_t           *state;
 487         tavor_rsrc_t            *rsrcp;
 488         tavor_umap_db_entry_t   *umapdb;
 489         tavor_umap_db_priv_t    *priv;
 490         minor_t                 instance;
 491         uint64_t                key, value;
 492         int                     status;
 493 
 494         TAVOR_TNF_ENTER(tavor_close);
 495 
 496         instance = TAVOR_DEV_INSTANCE(dev);
 497         state = ddi_get_soft_state(tavor_statep, instance);
 498         if (state == NULL) {
 499                 TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, "");
 500                 TAVOR_TNF_EXIT(tavor_close);
 501                 return (ENXIO);
 502         }
 503 
 504         /*
 505          * Search for "dev_t" in the "userland resources database".  As
 506          * explained above in tavor_open(), we can't depend on using the
 507          * current process ID here to do the lookup because the process
 508          * that ultimately closes may not be the same one who opened
 509          * (because of inheritance).
 510          * So we lookup the "dev_t" (which points to the PID of the process
 511          * that opened), and we remove the entry from the database (and free
 512          * it up).  Then we do another query based on the PID value.  And when
 513          * we find that database entry, we free it up too and then free the
 514          * Tavor UAR page resource.
 515          *
 516          * Note: We use the tavor_umap_db_find_nolock() database access
 517          * routine below (with an explicit mutex_enter of the database lock)
 518          * to ensure that the multiple accesses (which attempt to remove the
 519          * two database entries) can be done atomically.
 520          *
 521          * This works the same in both maintenance mode and HCA mode, except
 522          * for the call to tavor_rsrc_free().  In the case of maintenance mode,
 523          * this call is not needed, as it was not allocated in tavor_open()
 524          * above.
 525          */
 526         key = dev;
 527         mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
 528         status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC,
 529             &value, TAVOR_UMAP_DB_REMOVE, &umapdb);
 530         if (status == DDI_SUCCESS) {
 531                 /*
 532                  * If the "tdb_priv" field is non-NULL, it indicates that
 533                  * some "on close" handling is still necessary.  Call
 534                  * tavor_umap_db_handle_onclose_cb() to do the handling (i.e.
 535                  * to invoke all the registered callbacks).  Then free up
 536                  * the resources associated with "tdb_priv" and continue
 537                  * closing.
 538                  */
 539                 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv;
 540                 if (priv != NULL) {
 541                         tavor_umap_db_handle_onclose_cb(priv);
 542                         kmem_free(priv, sizeof (tavor_umap_db_priv_t));
 543                         umapdb->tdbe_common.tdb_priv = (void *)NULL;
 544                 }
 545 
 546                 tavor_umap_db_free(umapdb);
 547 
 548                 /*
 549                  * Now do another lookup using PID as the key (copy it from
 550                  * "value").  When this lookup is complete, the "value" field
 551                  * will contain the tavor_rsrc_t pointer for the UAR page
 552                  * resource.
 553                  */
 554                 key = value;
 555                 status = tavor_umap_db_find_nolock(instance, key,
 556                     MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
 557                     &umapdb);
 558                 if (status == DDI_SUCCESS) {
 559                         tavor_umap_db_free(umapdb);
 560                         /* If in "maintenance mode", don't free the rsrc */
 561                         if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 562                                 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
 563                                 tavor_rsrc_free(state, &rsrcp);
 564                         }
 565                 }
 566         }
 567         mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
 568 
 569         TAVOR_TNF_EXIT(tavor_close);
 570         return (0);
 571 }
 572 
 573 
 574 /*
 575  * tavor_attach()
 576  *    Context: Only called from attach() path context
 577  */
 578 static int
 579 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 580 {
 581         tavor_state_t   *state;
 582         ibc_clnt_hdl_t  tmp_ibtfpriv;
 583         ibc_status_t    ibc_status;
 584         int             instance;
 585         int             status;
 586 
 587         TAVOR_TNF_ENTER(tavor_attach);
 588 
 589 #ifdef __lock_lint
 590         (void) tavor_quiesce(dip);
 591 #endif
 592 
 593         switch (cmd) {
 594         case DDI_ATTACH:
 595                 instance = ddi_get_instance(dip);
 596                 status = ddi_soft_state_zalloc(tavor_statep, instance);
 597                 if (status != DDI_SUCCESS) {
 598                         TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, "");
 599                         cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
 600                             "attach_ssz_fail", instance);
 601                         goto fail_attach_nomsg;
 602 
 603                 }
 604                 state = ddi_get_soft_state(tavor_statep, instance);
 605                 if (state == NULL) {
 606                         ddi_soft_state_free(tavor_statep, instance);
 607                         TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, "");
 608                         cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
 609                             "attach_gss_fail", instance);
 610                         goto fail_attach_nomsg;
 611                 }
 612 
 613                 /* clear the attach error buffer */
 614                 TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf);
 615 
 616                 /*
 617                  * Initialize Tavor driver and hardware.
 618                  *
 619                  * Note: If this initialization fails we may still wish to
 620                  * create a device node and remain operational so that Tavor
 621                  * firmware can be updated/flashed (i.e. "maintenance mode").
 622                  * If this is the case, then "ts_operational_mode" will be
 623                  * equal to TAVOR_MAINTENANCE_MODE.  We will not attempt to
 624                  * attach to the IBTF or register with the IBMF (i.e. no
 625                  * InfiniBand interfaces will be enabled).
 626                  */
 627                 status = tavor_drv_init(state, dip, instance);
 628                 if ((status != DDI_SUCCESS) &&
 629                     (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
 630                         TNF_PROBE_0(tavor_attach_drvinit_fail,
 631                             TAVOR_TNF_ERROR, "");
 632                         goto fail_attach;
 633                 }
 634 
 635                 /* Create the minor node for device */
 636                 status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance,
 637                     DDI_PSEUDO, 0);
 638                 if (status != DDI_SUCCESS) {
 639                         tavor_drv_fini(state);
 640                         TAVOR_ATTACH_MSG(state->ts_attach_buf,
 641                             "attach_create_mn_fail");
 642                         TNF_PROBE_0(tavor_attach_create_mn_fail,
 643                             TAVOR_TNF_ERROR, "");
 644                         goto fail_attach;
 645                 }
 646 
 647                 /*
 648                  * If we are in "maintenance mode", then we don't want to
 649                  * register with the IBTF.  All InfiniBand interfaces are
 650                  * uninitialized, and the device is only capable of handling
 651                  * requests to update/flash firmware (or test/debug requests).
 652                  */
 653                 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 654 
 655                         /* Attach to InfiniBand Transport Framework (IBTF) */
 656                         ibc_status = ibc_attach(&tmp_ibtfpriv,
 657                             &state->ts_ibtfinfo);
 658                         if (ibc_status != IBC_SUCCESS) {
 659                                 ddi_remove_minor_node(dip, "devctl");
 660                                 tavor_drv_fini(state);
 661                                 TNF_PROBE_0(tavor_attach_ibcattach_fail,
 662                                     TAVOR_TNF_ERROR, "");
 663                                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
 664                                     "attach_ibcattach_fail");
 665                                 goto fail_attach;
 666                         }
 667 
 668                         /*
 669                          * Now that we've successfully attached to the IBTF,
 670                          * we enable all appropriate asynch and CQ events to
 671                          * be forwarded to the IBTF.
 672                          */
 673                         TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv);
 674 
 675                         ibc_post_attach(state->ts_ibtfpriv);
 676 
 677                         /* Register agents with IB Mgmt Framework (IBMF) */
 678                         status = tavor_agent_handlers_init(state);
 679                         if (status != DDI_SUCCESS) {
 680                                 (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH);
 681                                 TAVOR_QUIESCE_IBTF_CALLB(state);
 682                                 if (state->ts_in_evcallb != 0) {
 683                                         TAVOR_WARNING(state, "unable to "
 684                                             "quiesce Tavor IBTF callbacks");
 685                                 }
 686                                 ibc_detach(tmp_ibtfpriv);
 687                                 ddi_remove_minor_node(dip, "devctl");
 688                                 tavor_drv_fini(state);
 689                                 TNF_PROBE_0(tavor_attach_agentinit_fail,
 690                                     TAVOR_TNF_ERROR, "");
 691                                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
 692                                     "attach_agentinit_fail");
 693                                 goto fail_attach;
 694                         }
 695                 }
 696 
 697                 /* Report that driver was loaded */
 698                 ddi_report_dev(dip);
 699 
 700                 /* Send device information to log file */
 701                 tavor_device_info_report(state);
 702 
 703                 /* Report attach in maintenance mode, if appropriate */
 704                 if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
 705                         cmn_err(CE_NOTE, "tavor%d: driver attached "
 706                             "(for maintenance mode only)", state->ts_instance);
 707                 }
 708 
 709                 TAVOR_TNF_EXIT(tavor_attach);
 710                 return (DDI_SUCCESS);
 711 
 712         case DDI_RESUME:
 713                 /* Add code here for DDI_RESUME XXX */
 714                 TAVOR_TNF_EXIT(tavor_attach);
 715                 return (DDI_FAILURE);
 716 
 717         default:
 718                 TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, "");
 719                 break;
 720         }
 721 
 722 fail_attach:
 723         cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance,
 724             state->ts_attach_buf);
 725         tavor_drv_fini2(state);
 726         ddi_soft_state_free(tavor_statep, instance);
 727 fail_attach_nomsg:
 728         TAVOR_TNF_EXIT(tavor_attach);
 729         return (DDI_FAILURE);
 730 }
 731 
 732 
 733 /*
 734  * tavor_detach()
 735  *    Context: Only called from detach() path context
 736  */
 737 static int
 738 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 739 {
 740         tavor_state_t   *state;
 741         ibc_clnt_hdl_t  tmp_ibtfpriv;
 742         ibc_status_t    ibc_status;
 743         int             instance, status;
 744 
 745         TAVOR_TNF_ENTER(tavor_detach);
 746 
 747         instance = ddi_get_instance(dip);
 748         state = ddi_get_soft_state(tavor_statep, instance);
 749         if (state == NULL) {
 750                 TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, "");
 751                 TAVOR_TNF_EXIT(tavor_detach);
 752                 return (DDI_FAILURE);
 753         }
 754 
 755         switch (cmd) {
 756         case DDI_DETACH:
 757                 /*
 758                  * If we are in "maintenance mode", then we do not want to
 759                  * do teardown for any of the InfiniBand interfaces.
 760                  * Specifically, this means not detaching from IBTF (we never
 761                  * attached to begin with) and not deregistering from IBMF.
 762                  */
 763                 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 764                         /* Unregister agents from IB Mgmt Framework (IBMF) */
 765                         status = tavor_agent_handlers_fini(state);
 766                         if (status != DDI_SUCCESS) {
 767                                 TNF_PROBE_0(tavor_detach_agentfini_fail,
 768                                     TAVOR_TNF_ERROR, "");
 769                                 TAVOR_TNF_EXIT(tavor_detach);
 770                                 return (DDI_FAILURE);
 771                         }
 772 
 773                         /*
 774                          * Attempt the "pre-detach" from InfiniBand Transport
 775                          * Framework (IBTF).  At this point the IBTF is still
 776                          * capable of handling incoming asynch and completion
 777                          * events.  This "pre-detach" is primarily a mechanism
 778                          * to notify the appropriate IBTF clients that the
 779                          * HCA is being removed/offlined.
 780                          */
 781                         ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd);
 782                         if (ibc_status != IBC_SUCCESS) {
 783                                 status = tavor_agent_handlers_init(state);
 784                                 if (status != DDI_SUCCESS) {
 785                                         TAVOR_WARNING(state, "failed to "
 786                                             "restart Tavor agents");
 787                                 }
 788                                 TNF_PROBE_0(tavor_detach_ibcpredetach_fail,
 789                                     TAVOR_TNF_ERROR, "");
 790                                 TAVOR_TNF_EXIT(tavor_detach);
 791                                 return (DDI_FAILURE);
 792                         }
 793 
 794                         /*
 795                          * Before we can fully detach from the IBTF we need to
 796                          * ensure that we have handled all outstanding event
 797                          * callbacks.  This is accomplished by quiescing the
 798                          * event callback mechanism.  Note: if we are unable
 799                          * to successfully quiesce the callbacks, then this is
 800                          * an indication that something has probably gone
 801                          * seriously wrong.  We print out a warning, but
 802                          * continue.
 803                          */
 804                         tmp_ibtfpriv = state->ts_ibtfpriv;
 805                         TAVOR_QUIESCE_IBTF_CALLB(state);
 806                         if (state->ts_in_evcallb != 0) {
 807                                 TAVOR_WARNING(state, "unable to quiesce Tavor "
 808                                     "IBTF callbacks");
 809                         }
 810 
 811                         /* Complete the detach from the IBTF */
 812                         ibc_detach(tmp_ibtfpriv);
 813                 }
 814 
 815                 /* Remove the minor node for device */
 816                 ddi_remove_minor_node(dip, "devctl");
 817 
 818                 /*
 819                  * Only call tavor_drv_fini() if we are in Tavor HCA mode.
 820                  * (Because if we are in "maintenance mode", then we never
 821                  * successfully finished init.)  Only report successful
 822                  * detach for normal HCA mode.
 823                  */
 824                 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
 825                         /* Cleanup driver resources and shutdown hardware */
 826                         tavor_drv_fini(state);
 827                         cmn_err(CE_CONT, "Tavor driver successfully "
 828                             "detached\n");
 829                 }
 830 
 831                 tavor_drv_fini2(state);
 832                 ddi_soft_state_free(tavor_statep, instance);
 833 
 834                 TAVOR_TNF_EXIT(tavor_detach);
 835                 return (DDI_SUCCESS);
 836 
 837         case DDI_SUSPEND:
 838                 /* Add code here for DDI_SUSPEND XXX */
 839                 TAVOR_TNF_EXIT(tavor_detach);
 840                 return (DDI_FAILURE);
 841 
 842         default:
 843                 TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, "");
 844                 break;
 845         }
 846 
 847         TAVOR_TNF_EXIT(tavor_detach);
 848         return (DDI_FAILURE);
 849 }
 850 
 851 
 852 /*
 853  * tavor_drv_init()
 854  *    Context: Only called from attach() path context
 855  */
 856 static int
 857 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance)
 858 {
 859         int                     status;
 860 
 861         TAVOR_TNF_ENTER(tavor_drv_init);
 862 
 863         /* Save away devinfo and instance */
 864         state->ts_dip = dip;
 865         state->ts_instance = instance;
 866 
 867         /*
 868          * Check and set the operational mode of the device. If the driver is
 869          * bound to the Tavor device in "maintenance mode", then this generally
 870          * means that either the device has been specifically jumpered to
 871          * start in this mode or the firmware boot process has failed to
 872          * successfully load either the primary or the secondary firmware
 873          * image.
 874          */
 875         if (TAVOR_IS_HCA_MODE(state->ts_dip)) {
 876                 state->ts_operational_mode = TAVOR_HCA_MODE;
 877 
 878         } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) {
 879                 state->ts_operational_mode = TAVOR_COMPAT_MODE;
 880 
 881         } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) {
 882                 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
 883                 return (DDI_FAILURE);
 884 
 885         } else {
 886                 state->ts_operational_mode = 0;      /* invalid operational mode */
 887                 TAVOR_WARNING(state, "unexpected device type detected");
 888                 TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail,
 889                     TAVOR_TNF_ERROR, "");
 890                 TAVOR_TNF_EXIT(tavor_hw_init);
 891                 return (DDI_FAILURE);
 892         }
 893 
 894         /*
 895          * Initialize the Tavor hardware.
 896          * Note:  If this routine returns an error, it is often an reasonably
 897          * good indication that something Tavor firmware-related has caused
 898          * the failure.  In order to give the user an opportunity (if desired)
 899          * to update or reflash the Tavor firmware image, we set
 900          * "ts_operational_mode" flag (described above) to indicate that we
 901          * wish to enter maintenance mode.
 902          */
 903         status = tavor_hw_init(state);
 904         if (status != DDI_SUCCESS) {
 905                 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
 906                 cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance,
 907                     state->ts_attach_buf);
 908                 TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, "");
 909                 TAVOR_TNF_EXIT(tavor_drv_init);
 910                 return (DDI_FAILURE);
 911         }
 912 
 913         /* Setup Tavor interrupt handler */
 914         status = tavor_isr_init(state);
 915         if (status != DDI_SUCCESS) {
 916                 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
 917                 TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, "");
 918                 TAVOR_TNF_EXIT(tavor_drv_init);
 919                 return (DDI_FAILURE);
 920         }
 921 
 922         /* Initialize Tavor softstate */
 923         status = tavor_soft_state_init(state);
 924         if (status != DDI_SUCCESS) {
 925                 tavor_isr_fini(state);
 926                 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
 927                 TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, "");
 928                 TAVOR_TNF_EXIT(tavor_drv_init);
 929                 return (DDI_FAILURE);
 930         }
 931 
 932         TAVOR_TNF_EXIT(tavor_drv_init);
 933         return (DDI_SUCCESS);
 934 }
 935 
 936 
 937 /*
 938  * tavor_drv_fini()
 939  *    Context: Only called from attach() and/or detach() path contexts
 940  */
 941 static void
 942 tavor_drv_fini(tavor_state_t *state)
 943 {
 944         TAVOR_TNF_ENTER(tavor_drv_fini);
 945 
 946         /* Cleanup Tavor softstate */
 947         tavor_soft_state_fini(state);
 948 
 949         /* Teardown Tavor interrupts */
 950         tavor_isr_fini(state);
 951 
 952         /* Cleanup Tavor resources and shutdown hardware */
 953         tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
 954 
 955         TAVOR_TNF_EXIT(tavor_drv_fini);
 956 }
 957 
 958 /*
 959  * tavor_drv_fini2()
 960  *    Context: Only called from attach() and/or detach() path contexts
 961  */
 962 static void
 963 tavor_drv_fini2(tavor_state_t *state)
 964 {
 965         TAVOR_TNF_ENTER(tavor_drv_fini2);
 966 
 967         /* TAVOR_DRV_CLEANUP_LEVEL1 */
 968         if (state->ts_reg_cmdhdl) {
 969                 ddi_regs_map_free(&state->ts_reg_cmdhdl);
 970                 state->ts_reg_cmdhdl = NULL;
 971         }
 972 
 973         /* TAVOR_DRV_CLEANUP_LEVEL0 */
 974         if (state->ts_pci_cfghdl) {
 975                 pci_config_teardown(&state->ts_pci_cfghdl);
 976                 state->ts_pci_cfghdl = NULL;
 977         }
 978 
 979         TAVOR_TNF_EXIT(tavor_drv_fini2);
 980 }
 981 
 982 /*
 983  * tavor_isr_init()
 984  *    Context: Only called from attach() path context
 985  */
 986 static int
 987 tavor_isr_init(tavor_state_t *state)
 988 {
 989         int     status;
 990 
 991         TAVOR_TNF_ENTER(tavor_isr_init);
 992 
 993         /*
 994          * Add a handler for the interrupt or MSI
 995          */
 996         status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr,
 997             (caddr_t)state, NULL);
 998         if (status  != DDI_SUCCESS) {
 999                 TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, "");
1000                 TAVOR_TNF_EXIT(tavor_isr_init);
1001                 return (DDI_FAILURE);
1002         }
1003 
1004         /*
1005          * Enable the software interrupt.  Note: Even though we are only
1006          * using one (1) interrupt/MSI, depending on the value returned in
1007          * the capability flag, we have to call either ddi_intr_block_enable()
1008          * or ddi_intr_enable().
1009          */
1010         if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1011                 status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1);
1012                 if (status  != DDI_SUCCESS) {
1013                         TNF_PROBE_0(tavor_isr_init_blockenable_fail,
1014                             TAVOR_TNF_ERROR, "");
1015                         TAVOR_TNF_EXIT(tavor_isr_init);
1016                         return (DDI_FAILURE);
1017                 }
1018         } else {
1019                 status = ddi_intr_enable(state->ts_intrmsi_hdl);
1020                 if (status  != DDI_SUCCESS) {
1021                         TNF_PROBE_0(tavor_isr_init_intrenable_fail,
1022                             TAVOR_TNF_ERROR, "");
1023                         TAVOR_TNF_EXIT(tavor_isr_init);
1024                         return (DDI_FAILURE);
1025                 }
1026         }
1027 
1028         /*
1029          * Now that the ISR has been setup, arm all the EQs for event
1030          * generation.
1031          */
1032         tavor_eq_arm_all(state);
1033 
1034         TAVOR_TNF_EXIT(tavor_isr_init);
1035         return (DDI_SUCCESS);
1036 }
1037 
1038 
1039 /*
1040  * tavor_isr_fini()
1041  *    Context: Only called from attach() and/or detach() path contexts
1042  */
1043 static void
1044 tavor_isr_fini(tavor_state_t *state)
1045 {
1046         TAVOR_TNF_ENTER(tavor_isr_fini);
1047 
1048         /* Disable the software interrupt */
1049         if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1050                 (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1);
1051         } else {
1052                 (void) ddi_intr_disable(state->ts_intrmsi_hdl);
1053         }
1054 
1055         /*
1056          * Remove the software handler for the interrupt or MSI
1057          */
1058         (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl);
1059 
1060         TAVOR_TNF_EXIT(tavor_isr_fini);
1061 }
1062 
1063 
1064 /*
1065  * tavor_fix_error_buf()
1066  *      Context: Only called from attach().
1067  *
1068  * The error_buf_addr returned from QUERY_FW is a PCI address.
1069  * We need to convert it to an offset from the base address,
1070  * which is stored in the assigned-addresses property.
1071  */
1072 static int
1073 tavor_fix_error_buf(tavor_state_t *state)
1074 {
1075         int             assigned_addr_len;
1076         pci_regspec_t   *assigned_addr;
1077 
1078         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip,
1079             DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr,
1080             (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS)
1081                 return (DDI_FAILURE);
1082 
1083         state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low +
1084             ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32);
1085         ddi_prop_free(assigned_addr);
1086         return (DDI_SUCCESS);
1087 }
1088 
1089 /*
1090  * tavor_hw_init()
1091  *    Context: Only called from attach() path context
1092  */
1093 static int
1094 tavor_hw_init(tavor_state_t *state)
1095 {
1096         tavor_drv_cleanup_level_t       cleanup;
1097         sm_nodeinfo_t                   nodeinfo;
1098         uint64_t                        errorcode;
1099         off_t                           ddr_size;
1100         int                             status;
1101         int                             retries;
1102 
1103         TAVOR_TNF_ENTER(tavor_hw_init);
1104 
1105         /* This is where driver initialization begins */
1106         cleanup = TAVOR_DRV_CLEANUP_LEVEL0;
1107 
1108         /* Setup device access attributes */
1109         state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
1110         state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
1111         state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1112 
1113         /* Setup for PCI config read/write of HCA device  */
1114         status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl);
1115         if (status != DDI_SUCCESS) {
1116                 tavor_hw_fini(state, cleanup);
1117                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1118                     "hw_init_PCI_config_space_regmap_fail");
1119                 /* This case is not the degraded one */
1120                 return (DDI_FAILURE);
1121         }
1122 
1123         /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */
1124         status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR,
1125             &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr,
1126             &state->ts_reg_cmdhdl);
1127         if (status != DDI_SUCCESS) {
1128                 tavor_hw_fini(state, cleanup);
1129                 TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, "");
1130                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1131                     "hw_init_CMD_ddirms_fail");
1132                 TAVOR_TNF_EXIT(tavor_hw_init);
1133                 return (DDI_FAILURE);
1134         }
1135         cleanup = TAVOR_DRV_CLEANUP_LEVEL1;
1136 
1137         status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR,
1138             &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr,
1139             &state->ts_reg_uarhdl);
1140         if (status != DDI_SUCCESS) {
1141                 tavor_hw_fini(state, cleanup);
1142                 TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, "");
1143                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1144                     "hw_init_UAR_ddirms_fail");
1145                 TAVOR_TNF_EXIT(tavor_hw_init);
1146                 return (DDI_FAILURE);
1147         }
1148         cleanup = TAVOR_DRV_CLEANUP_LEVEL2;
1149 
1150         status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size);
1151         if (status != DDI_SUCCESS) {
1152                 cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed "
1153                     "(check HCA-attached DIMM memory?)\n");
1154                 tavor_hw_fini(state, cleanup);
1155                 TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail,
1156                     TAVOR_TNF_ERROR, "");
1157                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1158                     "hw_init_DDR_ddi_regsize_fail");
1159                 TAVOR_TNF_EXIT(tavor_hw_init);
1160                 return (DDI_FAILURE);
1161         }
1162 
1163 #if !defined(_ELF64) && !defined(__sparc)
1164         /*
1165          * For 32 bit x86/x64 kernels, where there is limited kernel virtual
1166          * memory available, define a minimal memory footprint. This is
1167          * specified in order to not take up too much resources, thus starving
1168          * out others. Only specified if the HCA DIMM is equal to or greater
1169          * than 256MB.
1170          *
1171          * Note: x86/x64 install and safemode boot are both 32bit.
1172          */
1173         ddr_size = TAVOR_DDR_SIZE_MIN;
1174 #endif  /* !(_ELF64) && !(__sparc) */
1175 
1176         state->ts_cfg_profile_setting = ddr_size;
1177 
1178         status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1179             &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr,
1180             &state->ts_reg_ddrhdl);
1181 
1182         /*
1183          * On 32-bit platform testing (primarily x86), it was seen that the
1184          * ddi_regs_map_setup() call would fail because there wasn't enough
1185          * kernel virtual address space available to map in the entire 256MB
1186          * DDR.  So we add this check in here, so that if the 256 (or other
1187          * larger value of DDR) map in fails, that we fallback to try the lower
1188          * size of 128MB.
1189          *
1190          * Note: If we only have 128MB of DDR in the system in the first place,
1191          * we don't try another ddi_regs_map_setup(), and just skip over this
1192          * check and return failures.
1193          */
1194         if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) {
1195                 /* Try falling back to 128MB DDR mapping */
1196                 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1197                     &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128,
1198                     &state->ts_reg_accattr, &state->ts_reg_ddrhdl);
1199 
1200                 /*
1201                  * 128MB DDR mapping worked.
1202                  * Set the updated config profile setting here.
1203                  */
1204                 if (status == DDI_SUCCESS) {
1205                         TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success,
1206                             TAVOR_TNF_TRACE, "");
1207                         state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128;
1208                 }
1209         }
1210 
1211         if (status != DDI_SUCCESS) {
1212                 if (status == DDI_ME_RNUMBER_RANGE) {
1213                         cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed "
1214                             "(check HCA-attached DIMM memory?)\n");
1215                 }
1216                 tavor_hw_fini(state, cleanup);
1217                 TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, "");
1218                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1219                     "hw_init_DDR_ddirms_fail");
1220                 TAVOR_TNF_EXIT(tavor_hw_init);
1221                 return (DDI_FAILURE);
1222         }
1223         cleanup = TAVOR_DRV_CLEANUP_LEVEL3;
1224 
1225         /* Setup Tavor Host Command Register (HCR) */
1226         state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *)
1227             ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET);
1228 
1229         /* Setup Tavor Event Cause Register (ecr and clr_ecr) */
1230         state->ts_cmd_regs.ecr = (uint64_t *)
1231             ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET);
1232         state->ts_cmd_regs.clr_ecr = (uint64_t *)
1233             ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET);
1234 
1235         /* Setup Tavor Software Reset register (sw_reset) */
1236         state->ts_cmd_regs.sw_reset = (uint32_t *)
1237             ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET);
1238 
1239         /* Setup Tavor Clear Interrupt register (clr_int) */
1240         state->ts_cmd_regs.clr_int = (uint64_t *)
1241             ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET);
1242 
1243         /* Initialize the Phase1 Tavor configuration profile */
1244         status = tavor_cfg_profile_init_phase1(state);
1245         if (status != DDI_SUCCESS) {
1246                 tavor_hw_fini(state, cleanup);
1247                 TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, "");
1248                 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail");
1249                 TAVOR_TNF_EXIT(tavor_hw_init);
1250                 return (DDI_FAILURE);
1251         }
1252         cleanup = TAVOR_DRV_CLEANUP_LEVEL4;
1253 
1254         /* Do a software reset of the Tavor HW to ensure proper state */
1255         status = tavor_sw_reset(state);
1256         if (status != TAVOR_CMD_SUCCESS) {
1257                 tavor_hw_fini(state, cleanup);
1258                 TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, "");
1259                 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail");
1260                 TAVOR_TNF_EXIT(tavor_hw_init);
1261                 return (DDI_FAILURE);
1262         }
1263 
1264         /* Post the SYS_EN command to start the hardware */
1265         status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL,
1266             &errorcode, TAVOR_CMD_NOSLEEP_SPIN);
1267         if (status != TAVOR_CMD_SUCCESS) {
1268                 if ((status == TAVOR_CMD_BAD_NVMEM) ||
1269                     (status == TAVOR_CMD_DDR_MEM_ERR)) {
1270                         cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1271                             "0x%" PRIx64 " (invalid firmware image?)\n",
1272                             status, errorcode);
1273                 } else {
1274                         cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1275                             "0x%" PRIx64 "\n", status, errorcode);
1276                 }
1277                 tavor_hw_fini(state, cleanup);
1278                 TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail,
1279                     TAVOR_TNF_ERROR, "");
1280                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1281                     "hw_init_sys_en_cmd_fail");
1282                 TAVOR_TNF_EXIT(tavor_hw_init);
1283                 return (DDI_FAILURE);
1284         }
1285         cleanup = TAVOR_DRV_CLEANUP_LEVEL5;
1286 
1287         /* First phase of init for Tavor configuration/resources */
1288         status = tavor_rsrc_init_phase1(state);
1289         if (status != DDI_SUCCESS) {
1290                 tavor_hw_fini(state, cleanup);
1291                 TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, "");
1292                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1293                     "hw_init_rsrcinit1_fail");
1294                 TAVOR_TNF_EXIT(tavor_hw_init);
1295                 return (DDI_FAILURE);
1296         }
1297         cleanup = TAVOR_DRV_CLEANUP_LEVEL6;
1298 
1299         /* Query the DDR properties (e.g. total DDR size) */
1300         status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0,
1301             &state->ts_ddr, sizeof (tavor_hw_queryddr_t),
1302             TAVOR_CMD_NOSLEEP_SPIN);
1303         if (status != TAVOR_CMD_SUCCESS) {
1304                 cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n",
1305                     status);
1306                 tavor_hw_fini(state, cleanup);
1307                 TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail,
1308                     TAVOR_TNF_ERROR, "");
1309                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1310                     "hw_init_query_ddr_cmd_fail");
1311                 TAVOR_TNF_EXIT(tavor_hw_init);
1312                 return (DDI_FAILURE);
1313         }
1314 
1315         /* Figure out how big the firmware image (in DDR) is */
1316         status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw,
1317             sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN);
1318         if (status != TAVOR_CMD_SUCCESS) {
1319                 cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n",
1320                     status);
1321                 tavor_hw_fini(state, cleanup);
1322                 TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail,
1323                     TAVOR_TNF_ERROR, "");
1324                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1325                     "hw_init_query_fw_cmd_fail");
1326                 TAVOR_TNF_EXIT(tavor_hw_init);
1327                 return (DDI_FAILURE);
1328         }
1329 
1330         if (tavor_fix_error_buf(state) != DDI_SUCCESS) {
1331                 tavor_hw_fini(state, cleanup);
1332                 TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail,
1333                     TAVOR_TNF_ERROR, "");
1334                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1335                     "hw_init_fixerrorbuf_fail");
1336                 TAVOR_TNF_EXIT(tavor_hw_init);
1337                 return (DDI_FAILURE);
1338         }
1339 
1340         /* Validate that the FW version is appropriate */
1341         status = tavor_fw_version_check(state);
1342         if (status != DDI_SUCCESS) {
1343                 if (state->ts_operational_mode == TAVOR_HCA_MODE) {
1344                         cmn_err(CE_CONT, "Unsupported Tavor FW version: "
1345                             "expected: %04d.%04d.%04d, "
1346                             "actual: %04d.%04d.%04d\n",
1347                             TAVOR_FW_VER_MAJOR,
1348                             TAVOR_FW_VER_MINOR,
1349                             TAVOR_FW_VER_SUBMINOR,
1350                             state->ts_fw.fw_rev_major,
1351                             state->ts_fw.fw_rev_minor,
1352                             state->ts_fw.fw_rev_subminor);
1353                 } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) {
1354                         cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: "
1355                             "expected: %04d.%04d.%04d, "
1356                             "actual: %04d.%04d.%04d\n",
1357                             TAVOR_COMPAT_FW_VER_MAJOR,
1358                             TAVOR_COMPAT_FW_VER_MINOR,
1359                             TAVOR_COMPAT_FW_VER_SUBMINOR,
1360                             state->ts_fw.fw_rev_major,
1361                             state->ts_fw.fw_rev_minor,
1362                             state->ts_fw.fw_rev_subminor);
1363                 } else {
1364                         cmn_err(CE_CONT, "Unsupported FW version: "
1365                             "%04d.%04d.%04d\n",
1366                             state->ts_fw.fw_rev_major,
1367                             state->ts_fw.fw_rev_minor,
1368                             state->ts_fw.fw_rev_subminor);
1369                 }
1370                 tavor_hw_fini(state, cleanup);
1371                 TNF_PROBE_0(tavor_hw_init_checkfwver_fail,
1372                     TAVOR_TNF_ERROR, "");
1373                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1374                     "hw_init_checkfwver_fail");
1375                 TAVOR_TNF_EXIT(tavor_hw_init);
1376                 return (DDI_FAILURE);
1377         }
1378 
1379         drv_usecwait(10);
1380         retries = 1000;         /* retry up to 1 second before giving up */
1381 retry:
1382         /* Call MOD_STAT_CFG to setup SRQ support (or disable) */
1383         status = tavor_mod_stat_cfg_cmd_post(state);
1384         if (status != DDI_SUCCESS) {
1385                 if (retries > 0) {
1386                         drv_usecwait(1000);
1387                         retries--;
1388                         goto retry;
1389                 }
1390                 cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n",
1391                     status);
1392                 tavor_hw_fini(state, cleanup);
1393                 TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail,
1394                     TAVOR_TNF_ERROR, "");
1395                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1396                     "hw_init_mod_stat_cfg_cmd_fail");
1397                 TAVOR_TNF_EXIT(tavor_hw_init);
1398                 return (DDI_FAILURE);
1399         }
1400 
1401         /* Figure out Tavor device limits */
1402         status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0,
1403             &state->ts_devlim, sizeof (tavor_hw_querydevlim_t),
1404             TAVOR_CMD_NOSLEEP_SPIN);
1405         if (status != TAVOR_CMD_SUCCESS) {
1406                 cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n",
1407                     status);
1408                 tavor_hw_fini(state, cleanup);
1409                 TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail,
1410                     TAVOR_TNF_ERROR, "");
1411                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1412                     "hw_init_query_devlim_cmd_fail");
1413                 TAVOR_TNF_EXIT(tavor_hw_init);
1414                 return (DDI_FAILURE);
1415         }
1416 
1417         /* Initialize the Phase2 Tavor configuration profile */
1418         status = tavor_cfg_profile_init_phase2(state);
1419         if (status != DDI_SUCCESS) {
1420                 tavor_hw_fini(state, cleanup);
1421                 TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, "");
1422                 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail");
1423                 TAVOR_TNF_EXIT(tavor_hw_init);
1424                 return (DDI_FAILURE);
1425         }
1426 
1427         /* Second phase of init for Tavor configuration/resources */
1428         status = tavor_rsrc_init_phase2(state);
1429         if (status != DDI_SUCCESS) {
1430                 tavor_hw_fini(state, cleanup);
1431                 TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, "");
1432                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1433                     "hw_init_rsrcinit2_fail");
1434                 TAVOR_TNF_EXIT(tavor_hw_init);
1435                 return (DDI_FAILURE);
1436         }
1437         cleanup = TAVOR_DRV_CLEANUP_LEVEL7;
1438 
1439         /* Miscellaneous query information */
1440         status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0,
1441             &state->ts_adapter, sizeof (tavor_hw_queryadapter_t),
1442             TAVOR_CMD_NOSLEEP_SPIN);
1443         if (status != TAVOR_CMD_SUCCESS) {
1444                 cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n",
1445                     status);
1446                 tavor_hw_fini(state, cleanup);
1447                 TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail,
1448                     TAVOR_TNF_ERROR, "");
1449                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1450                     "hw_init_query_adapter_cmd_fail");
1451                 TAVOR_TNF_EXIT(tavor_hw_init);
1452                 return (DDI_FAILURE);
1453         }
1454 
1455         /* Prepare configuration for Tavor INIT_HCA command */
1456         tavor_hca_config_setup(state, &state->ts_hcaparams);
1457 
1458         /* Post command to init Tavor HCA */
1459         status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams,
1460             TAVOR_CMD_NOSLEEP_SPIN);
1461         if (status != TAVOR_CMD_SUCCESS) {
1462                 cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n",
1463                     status);
1464                 tavor_hw_fini(state, cleanup);
1465                 TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail,
1466                     TAVOR_TNF_ERROR, "");
1467                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1468                     "hw_init_init_hca_cmd_fail");
1469                 TAVOR_TNF_EXIT(tavor_hw_init);
1470                 return (DDI_FAILURE);
1471         }
1472         cleanup = TAVOR_DRV_CLEANUP_LEVEL8;
1473 
1474         /* Allocate protection domain (PD) for Tavor internal use */
1475         status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP);
1476         if (status != DDI_SUCCESS) {
1477                 tavor_hw_fini(state, cleanup);
1478                 TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail,
1479                     TAVOR_TNF_ERROR, "");
1480                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1481                     "hw_init_internal_pd_alloc_fail");
1482                 TAVOR_TNF_EXIT(tavor_hw_init);
1483                 return (DDI_FAILURE);
1484         }
1485         cleanup = TAVOR_DRV_CLEANUP_LEVEL9;
1486 
1487         /* Setup Tavor internal UAR pages (0 and 1) */
1488         status = tavor_internal_uarpgs_init(state);
1489         if (status != DDI_SUCCESS) {
1490                 tavor_hw_fini(state, cleanup);
1491                 TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail,
1492                     TAVOR_TNF_ERROR, "");
1493                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1494                     "hw_init_internal_uarpgs_alloc_fail");
1495                 TAVOR_TNF_EXIT(tavor_hw_init);
1496                 return (DDI_FAILURE);
1497         }
1498         cleanup = TAVOR_DRV_CLEANUP_LEVEL10;
1499 
1500         /* Query and initialize the Tavor interrupt/MSI information */
1501         status = tavor_intr_or_msi_init(state);
1502         if (status != DDI_SUCCESS) {
1503                 tavor_hw_fini(state, cleanup);
1504                 TNF_PROBE_0(tavor_intr_or_msi_init_fail,
1505                     TAVOR_TNF_ERROR, "");
1506                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1507                     "intr_or_msi_init_fail");
1508                 TAVOR_TNF_EXIT(tavor_hw_init);
1509                 return (DDI_FAILURE);
1510         }
1511         cleanup = TAVOR_DRV_CLEANUP_LEVEL11;
1512 
1513         /* Setup all of the Tavor EQs */
1514         status = tavor_eq_init_all(state);
1515         if (status != DDI_SUCCESS) {
1516                 tavor_hw_fini(state, cleanup);
1517                 TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, "");
1518                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1519                     "hw_init_eqinitall_fail");
1520                 TAVOR_TNF_EXIT(tavor_hw_init);
1521                 return (DDI_FAILURE);
1522         }
1523         cleanup = TAVOR_DRV_CLEANUP_LEVEL12;
1524 
1525         /* Set aside contexts for QP0 and QP1 */
1526         status = tavor_special_qp_contexts_reserve(state);
1527         if (status != DDI_SUCCESS) {
1528                 tavor_hw_fini(state, cleanup);
1529                 TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail,
1530                     TAVOR_TNF_ERROR, "");
1531                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1532                     "hw_init_reserve_special_qp_fail");
1533                 TAVOR_TNF_EXIT(tavor_hw_init);
1534                 return (DDI_FAILURE);
1535         }
1536         cleanup = TAVOR_DRV_CLEANUP_LEVEL13;
1537 
1538         /* Initialize for multicast group handling */
1539         status = tavor_mcg_init(state);
1540         if (status != DDI_SUCCESS) {
1541                 tavor_hw_fini(state, cleanup);
1542                 TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, "");
1543                 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail");
1544                 TAVOR_TNF_EXIT(tavor_hw_init);
1545                 return (DDI_FAILURE);
1546         }
1547         cleanup = TAVOR_DRV_CLEANUP_LEVEL14;
1548 
1549         /* Initialize the Tavor IB port(s) */
1550         status = tavor_hca_port_init(state);
1551         if (status != DDI_SUCCESS) {
1552                 tavor_hw_fini(state, cleanup);
1553                 TNF_PROBE_0(tavor_hw_init_hca_port_init_fail,
1554                     TAVOR_TNF_ERROR, "");
1555                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1556                     "hw_init_hca_port_init_fail");
1557                 TAVOR_TNF_EXIT(tavor_hw_init);
1558                 return (DDI_FAILURE);
1559         }
1560         cleanup = TAVOR_DRV_CLEANUP_ALL;
1561 
1562         /* Determine NodeGUID and SystemImageGUID */
1563         status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1564             &nodeinfo);
1565         if (status != TAVOR_CMD_SUCCESS) {
1566                 cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n",
1567                     status);
1568                 tavor_hw_fini(state, cleanup);
1569                 TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail,
1570                     TAVOR_TNF_ERROR, "");
1571                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1572                     "hw_init_getnodeinfo_cmd_fail");
1573                 TAVOR_TNF_EXIT(tavor_hw_init);
1574                 return (DDI_FAILURE);
1575         }
1576 
1577         /*
1578          * If the NodeGUID value was set in OBP properties, then we use that
1579          * value.  But we still print a message if the value we queried from
1580          * firmware does not match this value.
1581          *
1582          * Otherwise if OBP value is not set then we use the value from
1583          * firmware unconditionally.
1584          */
1585         if (state->ts_cfg_profile->cp_nodeguid) {
1586                 state->ts_nodeguid   = state->ts_cfg_profile->cp_nodeguid;
1587         } else {
1588                 state->ts_nodeguid = nodeinfo.NodeGUID;
1589         }
1590 
1591         if (state->ts_nodeguid != nodeinfo.NodeGUID) {
1592                 cmn_err(CE_NOTE, "!NodeGUID value queried from firmware "
1593                     "does not match value set by device property");
1594         }
1595 
1596         /*
1597          * If the SystemImageGUID value was set in OBP properties, then we use
1598          * that value.  But we still print a message if the value we queried
1599          * from firmware does not match this value.
1600          *
1601          * Otherwise if OBP value is not set then we use the value from
1602          * firmware unconditionally.
1603          */
1604         if (state->ts_cfg_profile->cp_sysimgguid) {
1605                 state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid;
1606         } else {
1607                 state->ts_sysimgguid = nodeinfo.SystemImageGUID;
1608         }
1609 
1610         if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) {
1611                 cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware "
1612                     "does not match value set by device property");
1613         }
1614 
1615         /* Get NodeDescription */
1616         status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1617             (sm_nodedesc_t *)&state->ts_nodedesc);
1618         if (status != TAVOR_CMD_SUCCESS) {
1619                 cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n",
1620                     status);
1621                 tavor_hw_fini(state, cleanup);
1622                 TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail,
1623                     TAVOR_TNF_ERROR, "");
1624                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1625                     "hw_init_getnodedesc_cmd_fail");
1626                 TAVOR_TNF_EXIT(tavor_hw_init);
1627                 return (DDI_FAILURE);
1628         }
1629 
1630         TAVOR_TNF_EXIT(tavor_hw_init);
1631         return (DDI_SUCCESS);
1632 }
1633 
1634 
1635 /*
1636  * tavor_hw_fini()
1637  *    Context: Only called from attach() and/or detach() path contexts
1638  */
1639 static void
1640 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup)
1641 {
1642         uint_t          num_ports;
1643         int             status;
1644 
1645         TAVOR_TNF_ENTER(tavor_hw_fini);
1646 
1647         switch (cleanup) {
1648         /*
1649          * If we add more driver initialization steps that should be cleaned
1650          * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the
1651          * first entry (i.e. corresponds to the last init step).
1652          */
1653         case TAVOR_DRV_CLEANUP_ALL:
1654                 /* Shutdown the Tavor IB port(s) */
1655                 num_ports = state->ts_cfg_profile->cp_num_ports;
1656                 (void) tavor_hca_ports_shutdown(state, num_ports);
1657                 /* FALLTHROUGH */
1658 
1659         case TAVOR_DRV_CLEANUP_LEVEL14:
1660                 /* Teardown resources used for multicast group handling */
1661                 tavor_mcg_fini(state);
1662                 /* FALLTHROUGH */
1663 
1664         case TAVOR_DRV_CLEANUP_LEVEL13:
1665                 /* Unreserve the special QP contexts */
1666                 tavor_special_qp_contexts_unreserve(state);
1667                 /* FALLTHROUGH */
1668 
1669         case TAVOR_DRV_CLEANUP_LEVEL12:
1670                 /*
1671                  * Attempt to teardown all event queues (EQ).  If we fail
1672                  * here then print a warning message and return.  Something
1673                  * (either in HW or SW) has gone seriously wrong.
1674                  */
1675                 status = tavor_eq_fini_all(state);
1676                 if (status != DDI_SUCCESS) {
1677                         TAVOR_WARNING(state, "failed to teardown EQs");
1678                         TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail,
1679                             TAVOR_TNF_ERROR, "");
1680                         TAVOR_TNF_EXIT(tavor_hw_fini);
1681                         return;
1682                 }
1683                 /* FALLTHROUGH */
1684 
1685         case TAVOR_DRV_CLEANUP_LEVEL11:
1686                 status = tavor_intr_or_msi_fini(state);
1687                 if (status != DDI_SUCCESS) {
1688                         TAVOR_WARNING(state, "failed to free intr/MSI");
1689                         TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail,
1690                             TAVOR_TNF_ERROR, "");
1691                         TAVOR_TNF_EXIT(tavor_hw_fini);
1692                         return;
1693                 }
1694                 /* FALLTHROUGH */
1695 
1696         case TAVOR_DRV_CLEANUP_LEVEL10:
1697                 /* Free the resources for the Tavor internal UAR pages */
1698                 tavor_internal_uarpgs_fini(state);
1699                 /* FALLTHROUGH */
1700 
1701         case TAVOR_DRV_CLEANUP_LEVEL9:
1702                 /*
1703                  * Free the PD that was used internally by Tavor software.  If
1704                  * we fail here then print a warning and return.  Something
1705                  * (probably software-related, but perhaps HW) has gone wrong.
1706                  */
1707                 status = tavor_pd_free(state, &state->ts_pdhdl_internal);
1708                 if (status != DDI_SUCCESS) {
1709                         TAVOR_WARNING(state, "failed to free internal PD");
1710                         TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail,
1711                             TAVOR_TNF_ERROR, "");
1712                         TAVOR_TNF_EXIT(tavor_hw_fini);
1713                         return;
1714                 }
1715                 /* FALLTHROUGH */
1716 
1717         case TAVOR_DRV_CLEANUP_LEVEL8:
1718                 /*
1719                  * Post the CLOSE_HCA command to Tavor firmware.  If we fail
1720                  * here then print a warning and return.  Something (either in
1721                  * HW or SW) has gone seriously wrong.
1722                  */
1723                 status = tavor_close_hca_cmd_post(state,
1724                     TAVOR_CMD_NOSLEEP_SPIN);
1725                 if (status != TAVOR_CMD_SUCCESS) {
1726                         TAVOR_WARNING(state, "failed to shutdown HCA");
1727                         TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail,
1728                             TAVOR_TNF_ERROR, "");
1729                         TAVOR_TNF_EXIT(tavor_hw_fini);
1730                         return;
1731                 }
1732                 /* FALLTHROUGH */
1733 
1734         case TAVOR_DRV_CLEANUP_LEVEL7:
1735                 /* Cleanup all the phase2 resources first */
1736                 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL);
1737                 /* FALLTHROUGH */
1738 
1739         case TAVOR_DRV_CLEANUP_LEVEL6:
1740                 /* Then cleanup the phase1 resources */
1741                 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE);
1742                 /* FALLTHROUGH */
1743 
1744         case TAVOR_DRV_CLEANUP_LEVEL5:
1745                 /*
1746                  * Post the SYS_DIS command to Tavor firmware to shut
1747                  * everything down again.  If we fail here then print a
1748                  * warning and return.  Something (probably in HW, but maybe
1749                  * in SW) has gone seriously wrong.
1750                  */
1751                 status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN);
1752                 if (status != TAVOR_CMD_SUCCESS) {
1753                         TAVOR_WARNING(state, "failed to shutdown hardware");
1754                         TNF_PROBE_0(tavor_hw_fini_sys_dis_fail,
1755                             TAVOR_TNF_ERROR, "");
1756                         TAVOR_TNF_EXIT(tavor_hw_fini);
1757                         return;
1758                 }
1759                 /* FALLTHROUGH */
1760 
1761         case TAVOR_DRV_CLEANUP_LEVEL4:
1762                 /* Teardown any resources allocated for the config profile */
1763                 tavor_cfg_profile_fini(state);
1764                 /* FALLTHROUGH */
1765 
1766         case TAVOR_DRV_CLEANUP_LEVEL3:
1767                 ddi_regs_map_free(&state->ts_reg_ddrhdl);
1768                 /* FALLTHROUGH */
1769 
1770         case TAVOR_DRV_CLEANUP_LEVEL2:
1771                 ddi_regs_map_free(&state->ts_reg_uarhdl);
1772                 /* FALLTHROUGH */
1773 
1774         case TAVOR_DRV_CLEANUP_LEVEL1:
1775         case TAVOR_DRV_CLEANUP_LEVEL0:
1776                 /*
1777                  * LEVEL1 and LEVEL0 resources are freed in
1778                  * tavor_drv_fini2().
1779                  */
1780                 break;
1781 
1782         default:
1783                 TAVOR_WARNING(state, "unexpected driver cleanup level");
1784                 TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, "");
1785                 TAVOR_TNF_EXIT(tavor_hw_fini);
1786                 return;
1787         }
1788 
1789         TAVOR_TNF_EXIT(tavor_hw_fini);
1790 }
1791 
1792 
1793 /*
1794  * tavor_soft_state_init()
1795  *    Context: Only called from attach() path context
1796  */
1797 static int
1798 tavor_soft_state_init(tavor_state_t *state)
1799 {
1800         ibt_hca_attr_t          *hca_attr;
1801         uint64_t                maxval, val;
1802         ibt_hca_flags_t         caps = IBT_HCA_NO_FLAGS;
1803         int                     status;
1804 
1805         TAVOR_TNF_ENTER(tavor_soft_state_init);
1806 
1807         /*
1808          * The ibc_hca_info_t struct is passed to the IBTF.  This is the
1809          * routine where we initialize it.  Many of the init values come from
1810          * either configuration variables or successful queries of the Tavor
1811          * hardware abilities
1812          */
1813         state->ts_ibtfinfo.hca_ci_vers       = IBCI_V4;
1814         state->ts_ibtfinfo.hca_handle        = (ibc_hca_hdl_t)state;
1815         state->ts_ibtfinfo.hca_ops   = &tavor_ibc_ops;
1816 
1817         hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
1818         state->ts_ibtfinfo.hca_attr = hca_attr;
1819 
1820         hca_attr->hca_dip = state->ts_dip;
1821         hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major;
1822         hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor;
1823         hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor;
1824 
1825         /*
1826          * Determine HCA capabilities:
1827          * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST,
1828          *    IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE,
1829          *    or IBT_HCA_SHUTDOWN_PORT
1830          * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID,
1831          *    IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always
1832          *    supported
1833          * All other features are conditionally supported, depending on the
1834          *    status return by the Tavor HCA (in QUERY_DEV_LIM)
1835          */
1836         if (state->ts_devlim.ud_multi) {
1837                 caps |= IBT_HCA_UD_MULTICAST;
1838         }
1839         if (state->ts_devlim.atomic) {
1840                 caps |= IBT_HCA_ATOMICS_HCA;
1841         }
1842         if (state->ts_devlim.apm) {
1843                 caps |= IBT_HCA_AUTO_PATH_MIG;
1844         }
1845         if (state->ts_devlim.pkey_v) {
1846                 caps |= IBT_HCA_PKEY_CNTR;
1847         }
1848         if (state->ts_devlim.qkey_v) {
1849                 caps |= IBT_HCA_QKEY_CNTR;
1850         }
1851         if (state->ts_cfg_profile->cp_srq_enable) {
1852                 caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ;
1853         }
1854         caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT |
1855             IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE |
1856             IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE);
1857         hca_attr->hca_flags = caps;
1858         hca_attr->hca_flags2 = IBT_HCA2_DMA_MR;
1859 
1860         /* Determine VendorID, DeviceID, and revision ID */
1861         hca_attr->hca_vendor_id       = state->ts_adapter.vendor_id;
1862         hca_attr->hca_device_id       = state->ts_adapter.device_id;
1863         hca_attr->hca_version_id = state->ts_adapter.rev_id;
1864 
1865         /*
1866          * Determine number of available QPs and max QP size.  Number of
1867          * available QPs is determined by subtracting the number of
1868          * "reserved QPs" (i.e. reserved for firmware use) from the
1869          * total number configured.
1870          */
1871         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
1872         hca_attr->hca_max_qp = val - ((uint64_t)1 <<
1873             state->ts_devlim.log_rsvd_qp);
1874         maxval  = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz);
1875         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz);
1876         if (val > maxval) {
1877                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1878                 TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail,
1879                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size "
1880                     "exceeds device maximum", tnf_uint, maxsz, maxval);
1881                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1882                     "soft_state_init_maxqpsz_toobig_fail");
1883                 TAVOR_TNF_EXIT(tavor_soft_state_init);
1884                 return (DDI_FAILURE);
1885         }
1886         hca_attr->hca_max_qp_sz = val;
1887 
1888         /* Determine max scatter-gather size in WQEs */
1889         maxval  = state->ts_devlim.max_sg;
1890         val     = state->ts_cfg_profile->cp_wqe_max_sgl;
1891         if (val > maxval) {
1892                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1893                 TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail,
1894                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl "
1895                     "exceeds device maximum", tnf_uint, maxsgl, maxval);
1896                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1897                     "soft_state_init_toomanysgl_fail");
1898                 TAVOR_TNF_EXIT(tavor_soft_state_init);
1899                 return (DDI_FAILURE);
1900         }
1901         /* If the rounded value for max SGL is too large, cap it */
1902         if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) {
1903                 state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval;
1904                 val = maxval;
1905         } else {
1906                 val = state->ts_cfg_profile->cp_wqe_real_max_sgl;
1907         }
1908 
1909         hca_attr->hca_max_sgl         = val;
1910         hca_attr->hca_max_rd_sgl = 0;        /* zero because RD is unsupported */
1911 
1912         /*
1913          * Determine number of available CQs and max CQ size. Number of
1914          * available CQs is determined by subtracting the number of
1915          * "reserved CQs" (i.e. reserved for firmware use) from the
1916          * total number configured.
1917          */
1918         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq);
1919         hca_attr->hca_max_cq = val - ((uint64_t)1 <<
1920             state->ts_devlim.log_rsvd_cq);
1921         maxval  = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz);
1922         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1;
1923         if (val > maxval) {
1924                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1925                 TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail,
1926                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size "
1927                     "exceeds device maximum", tnf_uint, maxsz, maxval);
1928                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1929                     "soft_state_init_maxcqsz_toobig_fail");
1930                 TAVOR_TNF_EXIT(tavor_soft_state_init);
1931                 return (DDI_FAILURE);
1932         }
1933         hca_attr->hca_max_cq_sz = val;
1934 
1935         /*
1936          * Determine number of available SRQs and max SRQ size. Number of
1937          * available SRQs is determined by subtracting the number of
1938          * "reserved SRQs" (i.e. reserved for firmware use) from the
1939          * total number configured.
1940          */
1941         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq);
1942         hca_attr->hca_max_srqs = val - ((uint64_t)1 <<
1943             state->ts_devlim.log_rsvd_srq);
1944         maxval  = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz);
1945         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz);
1946 
1947         if (val > maxval) {
1948                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1949                 TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail,
1950                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size "
1951                     "exceeds device maximum", tnf_uint, maxsz, maxval);
1952                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1953                     "soft_state_init_maxsrqsz_toobig_fail");
1954                 TAVOR_TNF_EXIT(tavor_soft_state_init);
1955                 return (DDI_FAILURE);
1956         }
1957         hca_attr->hca_max_srqs_sz = val;
1958 
1959         val    = state->ts_cfg_profile->cp_srq_max_sgl;
1960         maxval  = state->ts_devlim.max_sg;
1961         if (val > maxval) {
1962                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1963                 TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail,
1964                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq "
1965                     "sgl exceeds device maximum", tnf_uint, maxsgl, maxval);
1966                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
1967                     "soft_state_init_toomanysrqsgl_fail");
1968                 TAVOR_TNF_EXIT(tavor_soft_state_init);
1969                 return (DDI_FAILURE);
1970         }
1971         hca_attr->hca_max_srq_sgl = val;
1972 
1973         /*
1974          * Determine supported HCA page sizes
1975          * XXX
1976          * For now we simply return the system pagesize as the only supported
1977          * pagesize
1978          */
1979         hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K :
1980             IBT_PAGE_4K);
1981 
1982         /*
1983          * Determine number of available MemReg, MemWin, and their max size.
1984          * Number of available MRs and MWs is determined by subtracting
1985          * the number of "reserved MPTs" (i.e. reserved for firmware use)
1986          * from the total number configured for each.
1987          */
1988         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt);
1989         hca_attr->hca_max_memr         = val - ((uint64_t)1 <<
1990             state->ts_devlim.log_rsvd_mpt);
1991         hca_attr->hca_max_mem_win = val - ((uint64_t)1 <<
1992             state->ts_devlim.log_rsvd_mpt);
1993         maxval  = state->ts_devlim.log_max_mrw_sz;
1994         val     = state->ts_cfg_profile->cp_log_max_mrw_sz;
1995         if (val > maxval) {
1996                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1997                 TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail,
1998                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size "
1999                     "exceeds device maximum", tnf_uint, maxsz, maxval);
2000                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2001                     "soft_state_init_maxmrwsz_toobig_fail");
2002                 TAVOR_TNF_EXIT(tavor_soft_state_init);
2003                 return (DDI_FAILURE);
2004         }
2005         hca_attr->hca_max_memr_len = ((uint64_t)1 << val);
2006 
2007         /* Determine RDMA/Atomic properties */
2008         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb);
2009         hca_attr->hca_max_rsc = val;
2010         val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp;
2011         hca_attr->hca_max_rdma_in_qp  = val;
2012         val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp;
2013         hca_attr->hca_max_rdma_out_qp = val;
2014         hca_attr->hca_max_rdma_in_ee  = 0;
2015         hca_attr->hca_max_rdma_out_ee = 0;
2016 
2017         /*
2018          * Determine maximum number of raw IPv6 and Ether QPs.  Set to 0
2019          * because neither type of raw QP is supported
2020          */
2021         hca_attr->hca_max_ipv6_qp  = 0;
2022         hca_attr->hca_max_ether_qp = 0;
2023 
2024         /* Determine max number of MCGs and max QP-per-MCG */
2025         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
2026         hca_attr->hca_max_mcg_qps   = val;
2027         val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg);
2028         hca_attr->hca_max_mcg            = val;
2029         val = state->ts_cfg_profile->cp_num_qp_per_mcg;
2030         hca_attr->hca_max_qp_per_mcg = val;
2031 
2032         /* Determine max number partitions (i.e. PKeys) */
2033         maxval  = ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2034         val     = ((uint64_t)state->ts_cfg_profile->cp_num_ports <<
2035             state->ts_cfg_profile->cp_log_max_pkeytbl);
2036 
2037         if (val > maxval) {
2038                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2039                 TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail,
2040                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys "
2041                     "exceeds device maximum", tnf_uint, maxpkey, maxval);
2042                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2043                     "soft_state_init_toomanypkey_fail");
2044                 TAVOR_TNF_EXIT(tavor_soft_state_init);
2045                 return (DDI_FAILURE);
2046         }
2047         hca_attr->hca_max_partitions = val;
2048 
2049         /* Determine number of ports */
2050         maxval = state->ts_devlim.num_ports;
2051         val = state->ts_cfg_profile->cp_num_ports;
2052         if ((val > maxval) || (val == 0)) {
2053                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2054                 TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail,
2055                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports "
2056                     "exceeds device maximum", tnf_uint, maxports, maxval);
2057                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2058                     "soft_state_init_toomanyports_fail");
2059                 TAVOR_TNF_EXIT(tavor_soft_state_init);
2060                 return (DDI_FAILURE);
2061         }
2062         hca_attr->hca_nports = val;
2063 
2064         /* Copy NodeGUID and SystemImageGUID from softstate */
2065         hca_attr->hca_node_guid = state->ts_nodeguid;
2066         hca_attr->hca_si_guid        = state->ts_sysimgguid;
2067 
2068         /*
2069          * Determine local ACK delay.  Use the value suggested by the Tavor
2070          * hardware (from the QUERY_DEV_LIM command)
2071          */
2072         hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay;
2073 
2074         /* Determine max SGID table and PKey table sizes */
2075         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl);
2076         hca_attr->hca_max_port_sgid_tbl_sz = val;
2077         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
2078         hca_attr->hca_max_port_pkey_tbl_sz = val;
2079 
2080         /* Determine max number of PDs */
2081         maxval  = ((uint64_t)1 << state->ts_devlim.log_max_pd);
2082         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd);
2083         if (val > maxval) {
2084                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2085                 TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail,
2086                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD "
2087                     "exceeds device maximum", tnf_uint, maxpd, maxval);
2088                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2089                     "soft_state_init_toomanypd_fail");
2090                 TAVOR_TNF_EXIT(tavor_soft_state_init);
2091                 return (DDI_FAILURE);
2092         }
2093         hca_attr->hca_max_pd = val;
2094 
2095         /* Determine max number of Address Handles */
2096         maxval  = ((uint64_t)1 << state->ts_devlim.log_max_av);
2097         val     = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah);
2098         if (val > maxval) {
2099                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2100                 TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail,
2101                     TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH "
2102                     "exceeds device maximum", tnf_uint, maxah, maxval);
2103                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2104                     "soft_state_init_toomanyah_fail");
2105                 TAVOR_TNF_EXIT(tavor_soft_state_init);
2106                 return (DDI_FAILURE);
2107         }
2108         hca_attr->hca_max_ah = val;
2109 
2110         /* No RDDs or EECs (since Reliable Datagram is not supported) */
2111         hca_attr->hca_max_rdd = 0;
2112         hca_attr->hca_max_eec = 0;
2113 
2114         /* Initialize lock for reserved UAR page access */
2115         mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER,
2116             DDI_INTR_PRI(state->ts_intrmsi_pri));
2117 
2118         /* Initialize the flash fields */
2119         state->ts_fw_flashstarted = 0;
2120         mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER,
2121             DDI_INTR_PRI(state->ts_intrmsi_pri));
2122 
2123         /* Initialize the lock for the info ioctl */
2124         mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER,
2125             DDI_INTR_PRI(state->ts_intrmsi_pri));
2126 
2127         /* Initialize the AVL tree for QP number support */
2128         tavor_qpn_avl_init(state);
2129 
2130         /* Initialize the kstat info structure */
2131         status = tavor_kstat_init(state);
2132         if (status != DDI_SUCCESS) {
2133                 tavor_qpn_avl_fini(state);
2134                 mutex_destroy(&state->ts_info_lock);
2135                 mutex_destroy(&state->ts_fw_flashlock);
2136                 mutex_destroy(&state->ts_uar_lock);
2137                 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2138                 TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail,
2139                     TAVOR_TNF_ERROR, "");
2140                 TAVOR_ATTACH_MSG(state->ts_attach_buf,
2141                     "soft_state_init_kstatinit_fail");
2142                 TAVOR_TNF_EXIT(tavor_soft_state_init);
2143                 return (DDI_FAILURE);
2144         }
2145 
2146         TAVOR_TNF_EXIT(tavor_soft_state_init);
2147         return (DDI_SUCCESS);
2148 }
2149 
2150 
2151 /*
2152  * tavor_soft_state_fini()
2153  *    Context: Called only from detach() path context
2154  */
2155 static void
2156 tavor_soft_state_fini(tavor_state_t *state)
2157 {
2158         TAVOR_TNF_ENTER(tavor_soft_state_fini);
2159 
2160         /* Teardown the kstat info */
2161         tavor_kstat_fini(state);
2162 
2163         /* Teardown the AVL tree for QP number support */
2164         tavor_qpn_avl_fini(state);
2165 
2166         /* Free up info ioctl mutex */
2167         mutex_destroy(&state->ts_info_lock);
2168 
2169         /* Free up flash mutex */
2170         mutex_destroy(&state->ts_fw_flashlock);
2171 
2172         /* Free up the UAR page access mutex */
2173         mutex_destroy(&state->ts_uar_lock);
2174 
2175         /* Free up the hca_attr struct */
2176         kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t));
2177 
2178         TAVOR_TNF_EXIT(tavor_soft_state_fini);
2179 }
2180 
2181 
2182 /*
2183  * tavor_hca_config_setup()
2184  *    Context: Only called from attach() path context
2185  */
2186 static void
2187 tavor_hca_config_setup(tavor_state_t *state,
2188     tavor_hw_initqueryhca_t *inithca)
2189 {
2190         tavor_rsrc_pool_info_t  *rsrc_pool;
2191         uint64_t                ddr_baseaddr, ddr_base_map_addr;
2192         uint64_t                offset, addr;
2193         uint_t                  mcg_size;
2194 
2195         TAVOR_TNF_ENTER(tavor_hca_config_setup);
2196 
2197         /* Set "host endianness".  Default is big endian */
2198 #ifdef  _LITTLE_ENDIAN
2199         inithca->big_endian  = 0;
2200 #else
2201         inithca->big_endian  = 1;
2202 #endif
2203         /* No Address Vector Protection, but Port Checking on by default */
2204         inithca->udav_chk    = TAVOR_UDAV_PROTECT_DISABLED;
2205         inithca->udav_port_chk       = TAVOR_UDAV_PORTCHK_ENABLED;
2206 
2207         ddr_baseaddr      = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr;
2208         ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr;
2209 
2210         /* Setup QPC table */
2211         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
2212         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2213         addr = ddr_base_map_addr + offset;
2214         inithca->context.qpc_baseaddr_h = (addr >> 32);
2215         inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7;
2216         inithca->context.log_num_qp  = state->ts_cfg_profile->cp_log_num_qp;
2217 
2218         /* Setup EEC table (initialize to zero - RD unsupported) */
2219         inithca->context.eec_baseaddr_h      = 0;
2220         inithca->context.eec_baseaddr_l      = 0;
2221         inithca->context.log_num_ee  = 0;
2222 
2223         /* Setup CQC table */
2224         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
2225         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2226         addr = ddr_base_map_addr + offset;
2227         inithca->context.cqc_baseaddr_h = (addr >> 32);
2228         inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2229         inithca->context.log_num_cq  = state->ts_cfg_profile->cp_log_num_cq;
2230 
2231         /* Setup SRQC table */
2232         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
2233         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2234         addr = ddr_base_map_addr + offset;
2235         inithca->context.srqc_baseaddr_h = (addr >> 32);
2236         inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2237         inithca->context.log_num_srq  =
2238             state->ts_cfg_profile->cp_log_num_srq;
2239 
2240         /* Setup EQPC table */
2241         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
2242         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2243         addr = ddr_base_map_addr + offset;
2244         inithca->context.eqpc_baseaddr       = addr;
2245 
2246         /* Setup EEEC table (initialize to zero - RD unsupported) */
2247         inithca->context.eeec_baseaddr       = 0;
2248 
2249         /* Setup EQC table */
2250         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
2251         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2252         addr = ddr_base_map_addr + offset;
2253         inithca->context.eqc_baseaddr_h = (addr >> 32);
2254         inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2255         inithca->context.log_num_eq  = TAVOR_NUM_EQ_SHIFT;
2256 
2257         /* Setup RDB table */
2258         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
2259         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2260         addr = ddr_base_map_addr + offset;
2261         inithca->context.rdb_baseaddr_h      = (addr >> 32);
2262         inithca->context.rdb_baseaddr_l = 0;
2263 
2264         /* Setup Multicast */
2265         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
2266         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2267         addr = ddr_base_map_addr + offset;
2268         inithca->multi.mc_baseaddr   = addr;
2269         mcg_size = TAVOR_MCGMEM_SZ(state);
2270         inithca->multi.log_mc_tbl_ent        = highbit(mcg_size) - 1;
2271         inithca->multi.mc_tbl_hash_sz        =
2272             (1 << state->ts_cfg_profile->cp_log_num_mcg_hash);
2273         inithca->multi.mc_hash_fn    = TAVOR_MCG_DEFAULT_HASH_FN;
2274         inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg;
2275 
2276 
2277         /* Setup TPT */
2278         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
2279         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2280         addr = ddr_base_map_addr + offset;
2281         inithca->tpt.mpt_baseaddr    = addr;
2282         inithca->tpt.mttseg_sz               = TAVOR_MTTSEG_SIZE_SHIFT;
2283         inithca->tpt.log_mpt_sz              = state->ts_cfg_profile->cp_log_num_mpt;
2284         inithca->tpt.mtt_version     = TAVOR_MTT_PG_WALK_VER;
2285 
2286         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
2287         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2288         addr = ddr_base_map_addr + offset;
2289         inithca->tpt.mtt_baseaddr    = addr;
2290 
2291         /* Setup UAR */
2292         rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
2293         offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2294         addr = ddr_base_map_addr + offset;
2295         inithca->uar.uarscr_baseaddr = addr;
2296 
2297         inithca->uar.uar_pg_sz = PAGESHIFT - 0xC;
2298 
2299         TAVOR_TNF_EXIT(tavor_hca_config_setup);
2300 }
2301 
2302 
2303 /*
2304  * tavor_hca_port_init()
2305  *    Context: Only called from attach() path context
2306  */
2307 static int
2308 tavor_hca_port_init(tavor_state_t *state)
2309 {
2310         tavor_hw_initib_t       *portinits, *initib;
2311         tavor_cfg_profile_t     *cfgprof;
2312         uint_t                  num_ports;
2313         int                     i, status;
2314         uint64_t                maxval, val;
2315         uint64_t                sysimgguid, nodeguid, portguid;
2316 
2317         TAVOR_TNF_ENTER(tavor_hca_port_init);
2318 
2319         cfgprof = state->ts_cfg_profile;
2320 
2321         /* Get number of HCA ports */
2322         num_ports = cfgprof->cp_num_ports;
2323 
2324         /* Allocate space for Tavor port init struct(s) */
2325         portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports *
2326             sizeof (tavor_hw_initib_t), KM_SLEEP);
2327 
2328         /* Post command to initialize Tavor HCA port */
2329         for (i = 0; i < num_ports; i++) {
2330                 initib = &portinits[i];
2331 
2332                 /*
2333                  * Determine whether we need to override the firmware's
2334                  * default SystemImageGUID setting.
2335                  */
2336                 sysimgguid = cfgprof->cp_sysimgguid;
2337                 if (sysimgguid != 0) {
2338                         initib->set_sysimg_guid      = 1;
2339                         initib->sysimg_guid  = sysimgguid;
2340                 }
2341 
2342                 /*
2343                  * Determine whether we need to override the firmware's
2344                  * default NodeGUID setting.
2345                  */
2346                 nodeguid = cfgprof->cp_nodeguid;
2347                 if (nodeguid != 0) {
2348                         initib->set_node_guid        = 1;
2349                         initib->node_guid    = nodeguid;
2350                 }
2351 
2352                 /*
2353                  * Determine whether we need to override the firmware's
2354                  * default PortGUID setting.
2355                  */
2356                 portguid = cfgprof->cp_portguid[i];
2357                 if (portguid != 0) {
2358                         initib->set_port_guid0       = 1;
2359                         initib->guid0                = portguid;
2360                 }
2361 
2362                 /* Validate max MTU size */
2363                 maxval  = state->ts_devlim.max_mtu;
2364                 val     = cfgprof->cp_max_mtu;
2365                 if (val > maxval) {
2366                         TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail,
2367                             TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2368                             "MTU size exceeds device maximum", tnf_uint,
2369                             maxmtu, maxval);
2370                         TAVOR_TNF_EXIT(tavor_hca_port_init);
2371                         goto init_ports_fail;
2372                 }
2373                 initib->mtu_cap = val;
2374 
2375                 /* Validate the max port width */
2376                 maxval  = state->ts_devlim.max_port_width;
2377                 val     = cfgprof->cp_max_port_width;
2378                 if (val > maxval) {
2379                         TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail,
2380                             TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2381                             "port width exceeds device maximum", tnf_uint,
2382                             maxportwidth, maxval);
2383                         TAVOR_TNF_EXIT(tavor_hca_port_init);
2384                         goto init_ports_fail;
2385                 }
2386                 initib->port_width_cap = val;
2387 
2388                 /* Validate max VL cap size */
2389                 maxval  = state->ts_devlim.max_vl;
2390                 val     = cfgprof->cp_max_vlcap;
2391                 if (val > maxval) {
2392                         TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail,
2393                             TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2394                             "VLcap size exceeds device maximum", tnf_uint,
2395                             maxvlcap, maxval);
2396                         TAVOR_TNF_EXIT(tavor_hca_port_init);
2397                         goto init_ports_fail;
2398                 }
2399                 initib->vl_cap = val;
2400 
2401                 /* Validate max GID table size */
2402                 maxval  = ((uint64_t)1 << state->ts_devlim.log_max_gid);
2403                 val     = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl);
2404                 if (val > maxval) {
2405                         TNF_PROBE_2(tavor_hca_port_init_gidtable_fail,
2406                             TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2407                             "GID table size exceeds device maximum", tnf_uint,
2408                             maxgidtbl, maxval);
2409                         TAVOR_TNF_EXIT(tavor_hca_port_init);
2410                         goto init_ports_fail;
2411                 }
2412                 initib->max_gid = val;
2413 
2414                 /* Validate max PKey table size */
2415                 maxval  = ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2416                 val     = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl);
2417                 if (val > maxval) {
2418                         TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail,
2419                             TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2420                             "PKey table size exceeds device maximum", tnf_uint,
2421                             maxpkeytbl, maxval);
2422                         TAVOR_TNF_EXIT(tavor_hca_port_init);
2423                         goto init_ports_fail;
2424                 }
2425                 initib->max_pkey = val;
2426 
2427                 /*
2428                  * Post the INIT_IB command to Tavor firmware.  When this
2429                  * command completes, the corresponding Tavor port will be
2430                  * physically "Up" and initialized.
2431                  */
2432                 status = tavor_init_ib_cmd_post(state, initib, i + 1,
2433                     TAVOR_CMD_NOSLEEP_SPIN);
2434                 if (status != TAVOR_CMD_SUCCESS) {
2435                         cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command "
2436                             "failed: %08x\n", i + 1, status);
2437                         TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail,
2438                             TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2439                             tnf_uint, port, i + 1);
2440                         TAVOR_TNF_EXIT(tavor_hca_port_init);
2441                         goto init_ports_fail;
2442                 }
2443         }
2444 
2445         /* Free up the memory for Tavor port init struct(s), return success */
2446         kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2447         TAVOR_TNF_EXIT(tavor_hca_port_init);
2448         return (DDI_SUCCESS);
2449 
2450 init_ports_fail:
2451         /*
2452          * Free up the memory for Tavor port init struct(s), shutdown any
2453          * successfully initialized ports, and return failure
2454          */
2455         kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2456         (void) tavor_hca_ports_shutdown(state, i);
2457 
2458         TAVOR_TNF_EXIT(tavor_hca_port_init);
2459         return (DDI_FAILURE);
2460 }
2461 
2462 
2463 /*
2464  * tavor_hca_ports_shutdown()
2465  *    Context: Only called from attach() and/or detach() path contexts
2466  */
2467 static int
2468 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init)
2469 {
2470         int     i, status;
2471 
2472         TAVOR_TNF_ENTER(tavor_hca_ports_shutdown);
2473 
2474         /*
2475          * Post commands to shutdown all init'd Tavor HCA ports.  Note: if
2476          * any of these commands fail for any reason, it would be entirely
2477          * unexpected and probably indicative a serious problem (HW or SW).
2478          * Although we do return void from this function, this type of failure
2479          * should not go unreported.  That is why we have the warning message
2480          * and the detailed TNF information.
2481          */
2482         for (i = 0; i < num_init; i++) {
2483                 status = tavor_close_ib_cmd_post(state, i + 1,
2484                     TAVOR_CMD_NOSLEEP_SPIN);
2485                 if (status != TAVOR_CMD_SUCCESS) {
2486                         TAVOR_WARNING(state, "failed to shutdown HCA port");
2487                         TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail,
2488                             TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2489                             tnf_uint, port, i + 1);
2490                         TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2491                         return (status);
2492                 }
2493         }
2494 
2495         TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2496 
2497         return (TAVOR_CMD_SUCCESS);
2498 }
2499 
2500 
2501 /*
2502  * tavor_internal_uarpgs_init
2503  *    Context: Only called from attach() path context
2504  */
2505 static int
2506 tavor_internal_uarpgs_init(tavor_state_t *state)
2507 {
2508         int     status;
2509 
2510         TAVOR_TNF_ENTER(tavor_internal_uarpgs_init);
2511 
2512         /*
2513          * Save away reserved Tavor UAR page #0.  This UAR page is not to
2514          * be used by software.
2515          */
2516         status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2517             &state->ts_uarpg0_rsrc_rsrvd);
2518         if (status != DDI_SUCCESS) {
2519                 TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2520                 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2521                 return (DDI_FAILURE);
2522         }
2523 
2524         /*
2525          * Save away Tavor UAR page #1 (for internal use).  This UAR page is
2526          * the privileged UAR page through which all kernel generated
2527          * doorbells will be rung.
2528          */
2529         status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2530             &state->ts_uarpg1_rsrc);
2531         if (status != DDI_SUCCESS) {
2532                 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2533                 TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2534                 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2535                 return (DDI_FAILURE);
2536         }
2537 
2538         /* Setup pointer to UAR page #1 doorbells */
2539         state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr;
2540 
2541         TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2542         return (DDI_SUCCESS);
2543 }
2544 
2545 
2546 /*
2547  * tavor_internal_uarpgs_fini
2548  *    Context: Only called from attach() and/or detach() path contexts
2549  */
2550 static void
2551 tavor_internal_uarpgs_fini(tavor_state_t *state)
2552 {
2553         TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini);
2554 
2555         /* Free up Tavor UAR page #1 (kernel driver doorbells) */
2556         tavor_rsrc_free(state, &state->ts_uarpg1_rsrc);
2557 
2558         /* Free up Tavor UAR page #0 (reserved) */
2559         tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2560 
2561         TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini);
2562 }
2563 
2564 
2565 /*
2566  * tavor_special_qp_contexts_reserve()
2567  *    Context: Only called from attach() path context
2568  */
2569 static int
2570 tavor_special_qp_contexts_reserve(tavor_state_t *state)
2571 {
2572         tavor_rsrc_t    *qp0_rsrc, *qp1_rsrc;
2573         int             status;
2574 
2575         TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve);
2576 
2577         /* Initialize the lock used for special QP rsrc management */
2578         mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER,
2579             DDI_INTR_PRI(state->ts_intrmsi_pri));
2580 
2581         /*
2582          * Reserve contexts for QP0.  These QP contexts will be setup to
2583          * act as aliases for the real QP0.  Note: We are required to grab
2584          * two QPs (one per port) even if we are operating in single-port
2585          * mode.
2586          */
2587         status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc);
2588         if (status != DDI_SUCCESS) {
2589                 mutex_destroy(&state->ts_spec_qplock);
2590                 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail,
2591                     TAVOR_TNF_ERROR, "");
2592                 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2593                 return (DDI_FAILURE);
2594         }
2595         state->ts_spec_qp0 = qp0_rsrc;
2596 
2597         /*
2598          * Reserve contexts for QP1.  These QP contexts will be setup to
2599          * act as aliases for the real QP1.  Note: We are required to grab
2600          * two QPs (one per port) even if we are operating in single-port
2601          * mode.
2602          */
2603         status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc);
2604         if (status != DDI_SUCCESS) {
2605                 tavor_rsrc_free(state, &qp0_rsrc);
2606                 mutex_destroy(&state->ts_spec_qplock);
2607                 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail,
2608                     TAVOR_TNF_ERROR, "");
2609                 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2610                 return (DDI_FAILURE);
2611         }
2612         state->ts_spec_qp1 = qp1_rsrc;
2613 
2614         TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2615         return (DDI_SUCCESS);
2616 }
2617 
2618 
2619 /*
2620  * tavor_special_qp_contexts_unreserve()
2621  *    Context: Only called from attach() and/or detach() path contexts
2622  */
2623 static void
2624 tavor_special_qp_contexts_unreserve(tavor_state_t *state)
2625 {
2626         TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve);
2627 
2628         /* Unreserve contexts for QP1 */
2629         tavor_rsrc_free(state, &state->ts_spec_qp1);
2630 
2631         /* Unreserve contexts for QP0 */
2632         tavor_rsrc_free(state, &state->ts_spec_qp0);
2633 
2634         /* Destroy the lock used for special QP rsrc management */
2635         mutex_destroy(&state->ts_spec_qplock);
2636 
2637         TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve);
2638 }
2639 
2640 
2641 /*
2642  * tavor_sw_reset()
2643  *    Context: Currently called only from attach() path context
2644  */
2645 static int
2646 tavor_sw_reset(tavor_state_t *state)
2647 {
2648         dev_info_t              *dip, *pdip;
2649         ddi_acc_handle_t        hdl = state->ts_pci_cfghdl, phdl;
2650         uint32_t                reset_delay;
2651         int                     status, i;
2652 
2653         TAVOR_TNF_ENTER(tavor_sw_reset);
2654 
2655         /*
2656          * If the configured software reset delay is set to zero, then we
2657          * will not attempt a software reset of the Tavor device.
2658          */
2659         reset_delay = state->ts_cfg_profile->cp_sw_reset_delay;
2660         if (reset_delay == 0) {
2661                 TAVOR_TNF_EXIT(tavor_sw_reset);
2662                 return (DDI_SUCCESS);
2663         }
2664 
2665         /*
2666          * Get dip for HCA device _and_ parent device as well.  Parent access
2667          * is necessary here because software reset of the Tavor hardware
2668          * will reinitialize both the config registers of the PCI bridge
2669          * (parent, if it exists) and the IB HCA (self)
2670          */
2671         dip  = state->ts_dip;
2672         pdip = ddi_get_parent(dip);
2673 
2674         /* Query the PCI capabilities of the HCA device */
2675         tavor_pci_capability_list(state, hdl);
2676 
2677         /*
2678          * Read all PCI config info (reg0...reg63).  Note: According to the
2679          * Tavor software reset application note, we should not read or
2680          * restore the values in reg22 and reg23.
2681          */
2682         for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2683                 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2684                     (i != TAVOR_SW_RESET_REG23_RSVD)) {
2685                         state->ts_cfg_data[i]  = pci_config_get32(hdl, i << 2);
2686                 }
2687         }
2688 
2689         if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2690                 /*
2691                  * Setup for PCI config read/write of bridge device
2692                  */
2693                 status = pci_config_setup(pdip, &phdl);
2694                 if (status != DDI_SUCCESS) {
2695                         TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail,
2696                             TAVOR_TNF_ERROR, "");
2697                         TAVOR_TNF_EXIT(tavor_sw_reset);
2698                         return (DDI_FAILURE);
2699                 }
2700 
2701                 /*
2702                  * Read all PCI config info (reg0...reg63).  Note: According to
2703                  * the Tavor software reset application note, we should not
2704                  * read or restore the values in reg22 and reg23.
2705                  */
2706                 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2707                         if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2708                             (i != TAVOR_SW_RESET_REG23_RSVD)) {
2709                                 state->ts_cfg_pdata[i] =
2710                                     pci_config_get32(phdl, i << 2);
2711                         }
2712                 }
2713         }
2714 
2715         /*
2716          * Perform the software reset (by writing 1 at offset 0xF0010)
2717          */
2718         ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset,
2719             TAVOR_SW_RESET_START);
2720 
2721         drv_usecwait(reset_delay);
2722 
2723         if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2724                 /*
2725                  * Bridge exists, so wait for the bridge to become ready.
2726                  *
2727                  * The above delay is necessary to avoid system panic from
2728                  * Master Abort.  If the device is accessed before this delay,
2729                  * device will not respond to config cycles and they will be
2730                  * terminate with a Master Abort which will panic the system.
2731                  * Below is the loop we use to poll status from the device to
2732                  * determine if it is OK to proceed.
2733                  */
2734                 i = 0;
2735                 while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2736                         drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2737                 }
2738 
2739                 /*
2740                  * Write all the PCI config registers back into each device
2741                  * (except for reg22 and reg23 - see above)
2742                  */
2743                 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2744                         if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2745                             (i != TAVOR_SW_RESET_REG23_RSVD)) {
2746                                 pci_config_put32(phdl, i << 2,
2747                                     state->ts_cfg_pdata[i]);
2748                         }
2749                 }
2750 
2751                 /*
2752                  * Tear down the config setup (for bridge device)
2753                  */
2754                 pci_config_teardown(&phdl);
2755 
2756         /* No Bridge Device */
2757         } else {
2758                 /*
2759                  * Bridge does not exist, so instead wait for the device itself
2760                  * to become ready.
2761                  *
2762                  * The above delay is necessary to avoid system panic from
2763                  * Master Abort.  If the device is accessed before this delay,
2764                  * device will not respond to config cycles and they will be
2765                  * terminate with a Master Abort which will panic the system.
2766                  * Below is the loop we use to poll status from the device to
2767                  * determine if it is OK to proceed.
2768                  */
2769                 i = 0;
2770                 while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2771                         drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2772                 }
2773         }
2774 
2775         for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2776                 if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2777                     (i != TAVOR_SW_RESET_REG23_RSVD)) {
2778                         pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]);
2779                 }
2780         }
2781 
2782         TAVOR_TNF_EXIT(tavor_sw_reset);
2783         return (DDI_SUCCESS);
2784 }
2785 
2786 
2787 /*
2788  * tavor_mcg_init()
2789  *    Context: Only called from attach() path context
2790  */
2791 static int
2792 tavor_mcg_init(tavor_state_t *state)
2793 {
2794         uint_t          mcg_tmp_sz;
2795 
2796         TAVOR_TNF_ENTER(tavor_mcg_init);
2797 
2798         /*
2799          * Allocate space for the MCG temporary copy buffer.  This is
2800          * used by the Attach/Detach Multicast Group code
2801          */
2802         mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2803         state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP);
2804 
2805         /*
2806          * Initialize the multicast group mutex.  This ensures atomic
2807          * access to add, modify, and remove entries in the multicast
2808          * group hash lists.
2809          */
2810         mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER,
2811             DDI_INTR_PRI(state->ts_intrmsi_pri));
2812 
2813         TAVOR_TNF_EXIT(tavor_mcg_init);
2814         return (DDI_SUCCESS);
2815 }
2816 
2817 
2818 /*
2819  * tavor_mcg_fini()
2820  *    Context: Only called from attach() and/or detach() path contexts
2821  */
2822 static void
2823 tavor_mcg_fini(tavor_state_t *state)
2824 {
2825         uint_t          mcg_tmp_sz;
2826 
2827         TAVOR_TNF_ENTER(tavor_mcg_fini);
2828 
2829         /* Free up the space used for the MCG temporary copy buffer */
2830         mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2831         kmem_free(state->ts_mcgtmp, mcg_tmp_sz);
2832 
2833         /* Destroy the multicast group mutex */
2834         mutex_destroy(&state->ts_mcglock);
2835 
2836         TAVOR_TNF_EXIT(tavor_mcg_fini);
2837 }
2838 
2839 
2840 /*
2841  * tavor_fw_version_check()
2842  *    Context: Only called from attach() path context
2843  */
2844 static int
2845 tavor_fw_version_check(tavor_state_t *state)
2846 {
2847         uint_t  tavor_fw_ver_major;
2848         uint_t  tavor_fw_ver_minor;
2849         uint_t  tavor_fw_ver_subminor;
2850 
2851         /*
2852          * Depending on which version of driver we have attached, the firmware
2853          * version checks will be different.  We set up the comparison values
2854          * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware
2855          * running in tavor mode).
2856          */
2857         switch (state->ts_operational_mode) {
2858         case TAVOR_HCA_MODE:
2859                 tavor_fw_ver_major = TAVOR_FW_VER_MAJOR;
2860                 tavor_fw_ver_minor = TAVOR_FW_VER_MINOR;
2861                 tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR;
2862                 break;
2863 
2864         case TAVOR_COMPAT_MODE:
2865                 tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR;
2866                 tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR;
2867                 tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR;
2868                 break;
2869 
2870         default:
2871                 return (DDI_FAILURE);
2872         }
2873 
2874         /*
2875          * If FW revision major number is less than acceptable,
2876          * return failure, else if greater return success.  If
2877          * the major numbers are equal than check the minor number
2878          */
2879         if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) {
2880                 return (DDI_FAILURE);
2881         } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) {
2882                 return (DDI_SUCCESS);
2883         }
2884         /*
2885          * Do the same check as above, except for minor revision numbers
2886          * If the minor numbers are equal than check the subminor number
2887          */
2888         if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) {
2889                 return (DDI_FAILURE);
2890         } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) {
2891                 return (DDI_SUCCESS);
2892         }
2893 
2894         /*
2895          * Once again we do the same check as above, except for the subminor
2896          * revision number.  If the subminor numbers are equal here, then
2897          * these are the same firmware version, return success
2898          */
2899         if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) {
2900                 return (DDI_FAILURE);
2901         } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) {
2902                 return (DDI_SUCCESS);
2903         }
2904 
2905         return (DDI_SUCCESS);
2906 }
2907 
2908 
2909 /*
2910  * tavor_device_info_report()
2911  *    Context: Only called from attach() path context
2912  */
2913 static void
2914 tavor_device_info_report(tavor_state_t *state)
2915 {
2916         cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, "
2917             "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major,
2918             state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor,
2919             state->ts_adapter.rev_id);
2920         cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n",
2921             state->ts_instance, state->ts_nodedesc, state->ts_nodeguid);
2922 }
2923 
2924 
2925 /*
2926  * tavor_pci_capability_list()
2927  *    Context: Only called from attach() path context
2928  */
2929 static void
2930 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl)
2931 {
2932         uint_t  offset, data;
2933 
2934         TAVOR_TNF_ENTER(tavor_pci_capability_list);
2935 
2936         /*
2937          * Check for the "PCI Capabilities" bit in the "Status Register".
2938          * Bit 4 in this register indicates the presence of a "PCI
2939          * Capabilities" list.
2940          */
2941         data = pci_config_get16(hdl, 0x6);
2942         if ((data & 0x10) == 0) {
2943                 TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, "");
2944                 TAVOR_TNF_EXIT(tavor_pci_capability_list);
2945                 return;
2946         }
2947 
2948         /*
2949          * Starting from offset 0x34 in PCI config space, find the
2950          * head of "PCI capabilities" list, and walk the list.  If
2951          * capabilities of a known type are encountered (e.g.
2952          * "PCI-X Capability"), then call the appropriate handler
2953          * function.
2954          */
2955         offset = pci_config_get8(hdl, 0x34);
2956         while (offset != 0x0) {
2957                 data = pci_config_get8(hdl, offset);
2958 
2959                 /*
2960                  * Check for known capability types.  Tavor has the
2961                  * following:
2962                  *    o VPD Capability   (0x03)
2963                  *    o PCI-X Capability (0x07)
2964                  *    o MSI Capability   (0x05)
2965                  *    o MSIX Capability  (0x11)
2966                  */
2967                 switch (data) {
2968                 case 0x03:
2969                         tavor_pci_capability_vpd(state, hdl, offset);
2970                         break;
2971                 case 0x07:
2972                         tavor_pci_capability_pcix(state, hdl, offset);
2973                         break;
2974                 case 0x05:
2975                         break;
2976                 default:
2977                         break;
2978                 }
2979 
2980                 /* Get offset of next entry in list */
2981                 offset = pci_config_get8(hdl, offset + 1);
2982         }
2983 
2984         TAVOR_TNF_EXIT(tavor_pci_capability_list);
2985 }
2986 
2987 /*
2988  * tavor_pci_read_vpd()
2989  *    Context: Only called from attach() path context
2990  *    utility routine for tavor_pci_capability_vpd()
2991  */
2992 static int
2993 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr,
2994     uint32_t *data)
2995 {
2996         int             retry = 4;  /* retry counter for EEPROM poll */
2997         uint32_t        val;
2998         int             vpd_addr = offset + 2;
2999         int             vpd_data = offset + 4;
3000 
3001         TAVOR_TNF_ENTER(tavor_pci_read_vpd);
3002 
3003         /*
3004          * In order to read a 32-bit value from VPD, we are to write down
3005          * the address (offset in the VPD itself) to the address register.
3006          * To signal the read, we also clear bit 31.  We then poll on bit 31
3007          * and when it is set, we can then read our 4 bytes from the data
3008          * register.
3009          */
3010         (void) pci_config_put32(hdl, offset, addr << 16);
3011         do {
3012                 drv_usecwait(1000);
3013                 val = pci_config_get16(hdl, vpd_addr);
3014                 if ((val >> 15) & 0x01) {
3015                         *data = pci_config_get32(hdl, vpd_data);
3016                         TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3017                         return (DDI_SUCCESS);
3018                 }
3019         } while (--retry);
3020 
3021         TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, "");
3022         TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3023         return (DDI_FAILURE);
3024 }
3025 
3026 
3027 /*
3028  * tavor_pci_capability_vpd()
3029  *    Context: Only called from attach() path context
3030  */
3031 static void
3032 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl,
3033     uint_t offset)
3034 {
3035         uint8_t                 name_length;
3036         uint8_t                 pn_length;
3037         int                     i, err = 0;
3038         int                     vpd_str_id = 0;
3039         int                     vpd_ro_desc;
3040         int                     vpd_ro_pn_desc;
3041 #ifndef _LITTLE_ENDIAN
3042         uint32_t                data32;
3043 #endif /* _LITTLE_ENDIAN */
3044         union {
3045                 uint32_t        vpd_int[TAVOR_VPD_HDR_DWSIZE];
3046                 uchar_t         vpd_char[TAVOR_VPD_HDR_BSIZE];
3047         } vpd;
3048 
3049         TAVOR_TNF_ENTER(tavor_pci_capability_vpd);
3050 
3051         /*
3052          * Read Vital Product Data (VPD) from PCI-X capability.
3053          */
3054         for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3055                 err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]);
3056                 if (err != DDI_SUCCESS) {
3057                         cmn_err(CE_NOTE, "!VPD read failed\n");
3058                         goto out;
3059                 }
3060         }
3061 
3062 #ifndef _LITTLE_ENDIAN
3063         /*
3064          * Need to swap bytes for big endian.
3065          */
3066         for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3067                 data32 = vpd.vpd_int[i];
3068                 vpd.vpd_char[(i << 2) + 3] =
3069                     (uchar_t)((data32 & 0xFF000000) >> 24);
3070                 vpd.vpd_char[(i << 2) + 2] =
3071                     (uchar_t)((data32 & 0x00FF0000) >> 16);
3072                 vpd.vpd_char[(i << 2) + 1] =
3073                     (uchar_t)((data32 & 0x0000FF00) >> 8);
3074                 vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF);
3075         }
3076 #endif  /* _LITTLE_ENDIAN */
3077 
3078         /* Check for VPD String ID Tag */
3079         if (vpd.vpd_char[vpd_str_id] == 0x82) {
3080                 /* get the product name */
3081                 name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1];
3082                 if (name_length > sizeof (state->ts_hca_name)) {
3083                         cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n",
3084                             name_length);
3085                         goto out;
3086                 }
3087                 (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3],
3088                     name_length);
3089                 state->ts_hca_name[name_length] = 0;
3090 
3091                 /* get the part number */
3092                 vpd_ro_desc = name_length + 3; /* read-only tag location */
3093                 vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */
3094                 /*
3095                  * Verify read-only tag and Part Number keyword.
3096                  */
3097                 if (vpd.vpd_char[vpd_ro_desc] != 0x90 ||
3098                     (vpd.vpd_char[vpd_ro_pn_desc] != 'P' &&
3099                     vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) {
3100                         cmn_err(CE_NOTE, "!VPD Part Number not found\n");
3101                         goto out;
3102                 }
3103 
3104                 pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2];
3105                 if (pn_length > sizeof (state->ts_hca_pn)) {
3106                         cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n",
3107                             name_length);
3108                         goto out;
3109                 }
3110                 (void) memcpy(state->ts_hca_pn,
3111                     &vpd.vpd_char[vpd_ro_pn_desc + 3],
3112                     pn_length);
3113                 state->ts_hca_pn[pn_length] = 0;
3114                 state->ts_hca_pn_len = pn_length;
3115         } else {
3116                 /* Wrong VPD String ID Tag */
3117                 cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n",
3118                     vpd.vpd_char[0]);
3119                 goto out;
3120         }
3121         TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3122         return;
3123 out:
3124         state->ts_hca_pn_len = 0;
3125         TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, "");
3126         TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3127 }
3128 
3129 /*
3130  * tavor_pci_capability_pcix()
3131  *    Context: Only called from attach() path context
3132  */
3133 static void
3134 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl,
3135     uint_t offset)
3136 {
3137         uint_t  command, status;
3138         int     max_out_splt_trans, max_mem_rd_byte_cnt;
3139         int     designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt;
3140 
3141         TAVOR_TNF_ENTER(tavor_pci_capability_pcix);
3142 
3143         /*
3144          * Query the current values for the PCI-X Command Register and
3145          * the PCI-X Status Register.
3146          */
3147         command = pci_config_get16(hdl, offset + 2);
3148         status  = pci_config_get32(hdl, offset + 4);
3149 
3150         /*
3151          * Check for config property specifying "maximum outstanding
3152          * split transactions".  If the property is defined and valid
3153          * (i.e. no larger than the so-called "designed maximum"),
3154          * then use the specified value to update the PCI-X Command Register.
3155          * Otherwise, extract the value from the Tavor config profile.
3156          */
3157         designed_max_out_splt_trans = ((status >> 23) & 7);
3158         max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3159             DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1);
3160         if ((max_out_splt_trans != -1) &&
3161             ((max_out_splt_trans < 0) ||
3162             (max_out_splt_trans > designed_max_out_splt_trans))) {
3163                 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-"
3164                     "split-trans\" (%d) invalid or exceeds device maximum"
3165                     " (%d), using default value (%d)\n", state->ts_instance,
3166                     max_out_splt_trans, designed_max_out_splt_trans,
3167                     state->ts_cfg_profile->cp_max_out_splt_trans);
3168                 max_out_splt_trans =
3169                     state->ts_cfg_profile->cp_max_out_splt_trans;
3170         } else if (max_out_splt_trans == -1) {
3171                 max_out_splt_trans =
3172                     state->ts_cfg_profile->cp_max_out_splt_trans;
3173         }
3174 
3175         /*
3176          * The config profile setting for max_out_splt_trans is determined
3177          * based on arch.  Check tavor_cfg.c for more information.  A value of
3178          * '-1' in the patchable variable means "do not change".  A value of
3179          * '0' means 1 outstanding splt trans and other values as defined by
3180          * PCI.  So we do one more check here, that if 'max_out_splt_trans' is
3181          * -1 (ie: < 0) we do not set the PCI command and leave it at the
3182          * default.
3183          */
3184         if (max_out_splt_trans >= 0) {
3185                 command = ((command & 0xFF8F) | max_out_splt_trans << 4);
3186         }
3187 
3188         /*
3189          * Check for config property specifying "maximum memory read
3190          * byte count.  If the property is defined and valid
3191          * (i.e. no larger than the so-called "designed maximum"),
3192          * then use the specified value to update the PCI-X Command Register.
3193          * Otherwise, extract the value from the Tavor config profile.
3194          */
3195         designed_max_mem_rd_byte_cnt = ((status >> 21) & 3);
3196         max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3197             DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1);
3198         if ((max_mem_rd_byte_cnt != -1) &&
3199             ((max_mem_rd_byte_cnt < 0) ||
3200             (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) {
3201                 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-"
3202                     "count\" (%d) invalid or exceeds device maximum"
3203                     " (%d), using default value (%d)\n", state->ts_instance,
3204                     max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt,
3205                     state->ts_cfg_profile->cp_max_mem_rd_byte_cnt);
3206                 max_mem_rd_byte_cnt =
3207                     state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3208         } else if (max_mem_rd_byte_cnt == -1) {
3209                 max_mem_rd_byte_cnt =
3210                     state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3211         }
3212 
3213         /*
3214          * The config profile setting for max_mem_rd_byte_cnt is determined
3215          * based on arch.  Check tavor_cfg.c for more information.  A value of
3216          * '-1' in the patchable variable means "do not change".  A value of
3217          * '0' means minimum (512B) read, and other values as defined by
3218          * PCI.  So we do one more check here, that if 'max_mem_rd_byte_cnt' is
3219          * -1 (ie: < 0) we do not set the PCI command and leave it at the
3220          * default.
3221          */
3222         if (max_mem_rd_byte_cnt >= 0) {
3223                 command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2);
3224         }
3225 
3226         /*
3227          * Update the PCI-X Command Register with the newly configured
3228          * values.
3229          */
3230         pci_config_put16(hdl, offset + 2, command);
3231 
3232         TAVOR_TNF_EXIT(tavor_pci_capability_pcix);
3233 }
3234 
3235 
3236 /*
3237  * tavor_intr_or_msi_init()
3238  *    Context: Only called from attach() path context
3239  */
3240 static int
3241 tavor_intr_or_msi_init(tavor_state_t *state)
3242 {
3243         int     status;
3244 
3245         TAVOR_TNF_ENTER(tavor_intr_or_msi_init);
3246 
3247         /* Query for the list of supported interrupt event types */
3248         status = ddi_intr_get_supported_types(state->ts_dip,
3249             &state->ts_intr_types_avail);
3250         if (status != DDI_SUCCESS) {
3251                 TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail,
3252                     TAVOR_TNF_ERROR, "");
3253                 TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3254                 return (DDI_FAILURE);
3255         }
3256 
3257         /*
3258          * If Tavor/Arbel supports MSI in this system (and, if it
3259          * hasn't been overridden by a configuration variable), then
3260          * the default behavior is to use a single MSI.  Otherwise,
3261          * fallback to using legacy interrupts.  Also, if MSI allocatis chosen,
3262          * but fails for whatever reasons, then fallback to using legacy
3263          * interrupts.
3264          */
3265         if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3266             (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3267                 status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI);
3268                 if (status == DDI_SUCCESS) {
3269                         state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI;
3270                         TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3271                         return (DDI_SUCCESS);
3272                 }
3273         }
3274 
3275         /*
3276          * MSI interrupt allocation failed, or was not available.  Fallback to
3277          * legacy interrupt support.
3278          */
3279         if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) {
3280                 status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED);
3281                 if (status == DDI_SUCCESS) {
3282                         state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED;
3283                         TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3284                         return (DDI_SUCCESS);
3285                 }
3286         }
3287 
3288         /*
3289          * Neither MSI or legacy interrupts were successful.  return failure.
3290          */
3291         TAVOR_TNF_EXIT(tavor_intr_or_msi_setup);
3292         return (DDI_FAILURE);
3293 }
3294 
3295 /*
3296  * tavor_add_intrs()
3297  *    Context: Only called from attach() patch context
3298  */
3299 static int
3300 tavor_add_intrs(tavor_state_t *state, int intr_type)
3301 {
3302         int status;
3303 
3304         TAVOR_TNF_ENTER(tavor_add_intrs);
3305 
3306         /* Get number of interrupts/MSI supported */
3307         status = ddi_intr_get_nintrs(state->ts_dip, intr_type,
3308             &state->ts_intrmsi_count);
3309         if (status != DDI_SUCCESS) {
3310                 TNF_PROBE_0(tavor_add_intrs_getnintrs_fail,
3311                     TAVOR_TNF_ERROR, "");
3312                 TAVOR_TNF_EXIT(tavor_add_intrs);
3313                 return (DDI_FAILURE);
3314         }
3315 
3316         /* Get number of available interrupts/MSI */
3317         status = ddi_intr_get_navail(state->ts_dip, intr_type,
3318             &state->ts_intrmsi_avail);
3319         if (status != DDI_SUCCESS) {
3320                 TNF_PROBE_0(tavor_add_intrs_getnavail_fail,
3321                     TAVOR_TNF_ERROR, "");
3322                 TAVOR_TNF_EXIT(tavor_add_intrs);
3323                 return (DDI_FAILURE);
3324         }
3325 
3326         /* Ensure that we have at least one (1) usable MSI or interrupt */
3327         if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) {
3328                 TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail,
3329                     TAVOR_TNF_ERROR, "");
3330                 TAVOR_TNF_EXIT(tavor_add_intrs);
3331                 return (DDI_FAILURE);
3332         }
3333 
3334         /* Attempt to allocate a single interrupt/MSI handle */
3335         status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl,
3336             intr_type, 0, 1, &state->ts_intrmsi_allocd,
3337             DDI_INTR_ALLOC_STRICT);
3338         if (status != DDI_SUCCESS) {
3339                 TNF_PROBE_0(tavor_add_intrs_intralloc_fail,
3340                     TAVOR_TNF_ERROR, "");
3341                 TAVOR_TNF_EXIT(tavor_add_intrs);
3342                 return (DDI_FAILURE);
3343         }
3344 
3345         /* Ensure that we have allocated at least one (1) MSI or interrupt */
3346         if (state->ts_intrmsi_allocd < 1) {
3347                 TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail,
3348                     TAVOR_TNF_ERROR, "");
3349                 TAVOR_TNF_EXIT(tavor_add_intrs);
3350                 return (DDI_FAILURE);
3351         }
3352 
3353         /*
3354          * Extract the priority for the allocated interrupt/MSI.  This
3355          * will be used later when initializing certain mutexes.
3356          */
3357         status = ddi_intr_get_pri(state->ts_intrmsi_hdl,
3358             &state->ts_intrmsi_pri);
3359         if (status != DDI_SUCCESS) {
3360                 /* Free the allocated interrupt/MSI handle */
3361                 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3362 
3363                 TNF_PROBE_0(tavor_add_intrs_getpri_fail,
3364                     TAVOR_TNF_ERROR, "");
3365                 TAVOR_TNF_EXIT(tavor_add_intrs);
3366                 return (DDI_FAILURE);
3367         }
3368 
3369         /* Make sure the interrupt/MSI priority is below 'high level' */
3370         if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) {
3371                 /* Free the allocated interrupt/MSI handle */
3372                 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3373 
3374                 TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail,
3375                     TAVOR_TNF_ERROR, "");
3376                 TAVOR_TNF_EXIT(tavor_add_intrs);
3377                 return (DDI_FAILURE);
3378         }
3379 
3380         /* Get add'l capability information regarding interrupt/MSI */
3381         status = ddi_intr_get_cap(state->ts_intrmsi_hdl,
3382             &state->ts_intrmsi_cap);
3383         if (status != DDI_SUCCESS) {
3384                 /* Free the allocated interrupt/MSI handle */
3385                 (void) ddi_intr_free(state->ts_intrmsi_hdl);
3386 
3387                 TNF_PROBE_0(tavor_add_intrs_getcap_fail,
3388                     TAVOR_TNF_ERROR, "");
3389                 TAVOR_TNF_EXIT(tavor_add_intrs);
3390                 return (DDI_FAILURE);
3391         }
3392 
3393         TAVOR_TNF_EXIT(tavor_add_intrs);
3394         return (DDI_SUCCESS);
3395 }
3396 
3397 
3398 /*
3399  * tavor_intr_or_msi_fini()
3400  *    Context: Only called from attach() and/or detach() path contexts
3401  */
3402 static int
3403 tavor_intr_or_msi_fini(tavor_state_t *state)
3404 {
3405         int     status;
3406 
3407         TAVOR_TNF_ENTER(tavor_intr_or_msi_fini);
3408 
3409         /* Free the allocated interrupt/MSI handle */
3410         status = ddi_intr_free(state->ts_intrmsi_hdl);
3411         if (status != DDI_SUCCESS) {
3412                 TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail,
3413                     TAVOR_TNF_ERROR, "");
3414                 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3415                 return (DDI_FAILURE);
3416         }
3417 
3418         TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3419         return (DDI_SUCCESS);
3420 }
3421 
3422 
3423 /* Disable Tavor interrupts */
3424 static int
3425 tavor_intr_disable(tavor_state_t *state)
3426 {
3427         ushort_t msi_ctrl = 0, caps_ctrl = 0;
3428         ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl;
3429         ASSERT(pci_cfg_hdl != NULL);
3430         ASSERT(state->ts_intr_types_avail &
3431             (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI));
3432 
3433         /*
3434          * Check if MSI interrupts are used. If so, disable MSI interupts.
3435          * If not, since Tavor doesn't support MSI-X interrupts, assuming the
3436          * legacy interrupt is used instead, disable the legacy interrupt.
3437          */
3438         if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3439             (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3440 
3441                 if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI,
3442                     &caps_ctrl) == DDI_SUCCESS)) {
3443                         if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL,
3444                             caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16)
3445                                 return (DDI_FAILURE);
3446                 }
3447                 ASSERT(msi_ctrl != 0);
3448 
3449                 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
3450                         return (DDI_SUCCESS);
3451 
3452                 if (msi_ctrl &  PCI_MSI_PVM_MASK) {
3453                         int offset = (msi_ctrl &  PCI_MSI_64BIT_MASK) ?
3454                             PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK;
3455 
3456                         /* Clear all inums in MSI */
3457                         PCI_CAP_PUT32(pci_cfg_hdl, NULL, caps_ctrl,
3458                             offset, 0x0);
3459                 }
3460 
3461                 /* Disable MSI interrupts */
3462                 msi_ctrl &= ~PCI_MSI_ENABLE_BIT;
3463                 PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSI_CTRL,
3464                     msi_ctrl);
3465 
3466         } else {
3467                 uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM);
3468                 ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED);
3469 
3470                 /* Disable the legacy interrupts */
3471                 cmdreg |= PCI_COMM_INTX_DISABLE;
3472                 pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg);
3473         }
3474 
3475         return (DDI_SUCCESS);
3476 }
3477 
3478 /* Tavor quiesce(9F) entry */
3479 static int
3480 tavor_quiesce(dev_info_t *dip)
3481 {
3482         tavor_state_t *state = ddi_get_soft_state(tavor_statep,
3483             DEVI(dip)->devi_instance);
3484         ASSERT(state != NULL);
3485 
3486         /* start fastreboot */
3487         state->ts_quiescing = B_TRUE;
3488 
3489         /* If it's in maintenance mode, do nothing but return with SUCCESS */
3490         if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
3491                 return (DDI_SUCCESS);
3492         }
3493 
3494         /* Shutdown HCA ports */
3495         if (tavor_hca_ports_shutdown(state,
3496             state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) {
3497                 state->ts_quiescing = B_FALSE;
3498                 return (DDI_FAILURE);
3499         }
3500 
3501         /* Close HCA */
3502         if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3503             TAVOR_CMD_SUCCESS) {
3504                 state->ts_quiescing = B_FALSE;
3505                 return (DDI_FAILURE);
3506         }
3507 
3508         /* Shutdown FW */
3509         if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3510             TAVOR_CMD_SUCCESS) {
3511                 state->ts_quiescing = B_FALSE;
3512                 return (DDI_FAILURE);
3513         }
3514 
3515         /* Disable interrupts */
3516         if (tavor_intr_disable(state) != DDI_SUCCESS) {
3517                 state->ts_quiescing = B_FALSE;
3518                 return (DDI_FAILURE);
3519         }
3520 
3521         /* SW-reset */
3522         if (tavor_sw_reset(state) != DDI_SUCCESS) {
3523                 state->ts_quiescing = B_FALSE;
3524                 return (DDI_FAILURE);
3525         }
3526 
3527         return (DDI_SUCCESS);
3528 }