1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * tavor_cmd.c
  29  *    Tavor Firmware Command Routines
  30  *
  31  *    Implements all the routines necessary for allocating, posting, and
  32  *    freeing commands for the Tavor firmware.  These routines manage a
  33  *    preallocated list of command mailboxes and provide interfaces to post
  34  *    each of the several dozen commands to the Tavor firmware.
  35  */
  36 
  37 #include <sys/types.h>
  38 #include <sys/conf.h>
  39 #include <sys/ddi.h>
  40 #include <sys/sunddi.h>
  41 #include <sys/modctl.h>
  42 
  43 #include <sys/ib/adapters/tavor/tavor.h>
  44 
  45 static int tavor_impl_mbox_alloc(tavor_state_t *state, tavor_mboxlist_t *mblist,
  46     tavor_mbox_t **mb, uint_t mbox_wait);
  47 static void tavor_impl_mbox_free(tavor_mboxlist_t *mblist, tavor_mbox_t **mb);
  48 static int tavor_impl_mboxlist_init(tavor_state_t *state,
  49     tavor_mboxlist_t *mblist, uint_t num_mbox, tavor_rsrc_type_t type);
  50 static void tavor_impl_mboxlist_fini(tavor_state_t *state,
  51     tavor_mboxlist_t *mblist);
  52 static int tavor_outstanding_cmd_alloc(tavor_state_t *state,
  53     tavor_cmd_t **cmd_ptr, uint_t cmd_wait);
  54 static void tavor_outstanding_cmd_free(tavor_state_t *state,
  55     tavor_cmd_t **cmd_ptr);
  56 static int tavor_write_hcr(tavor_state_t *state, tavor_cmd_post_t *cmdpost,
  57     uint16_t token);
  58 static void tavor_mbox_sync(tavor_mbox_t *mbox, uint_t offset,
  59     uint_t length, uint_t flag);
  60 
  61 /*
  62  * tavor_cmd_post()
  63  *    Context: Can be called from interrupt or base context.
  64  *
  65  *    The "cp_flags" field in cmdpost
  66  *    is used to determine whether to wait for an available
  67  *    outstanding command (if necessary) or to return error.
  68  */
  69 int
  70 tavor_cmd_post(tavor_state_t *state, tavor_cmd_post_t *cmdpost)
  71 {
  72         tavor_cmd_t     *cmdptr;
  73         int             status;
  74         uint16_t        token;
  75 
  76         TAVOR_TNF_ENTER(tavor_cmd_post);
  77 
  78         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdpost))
  79 
  80         /* Determine if we are going to spin until completion */
  81         if (cmdpost->cp_flags == TAVOR_CMD_NOSLEEP_SPIN) {
  82 
  83                 TNF_PROBE_0_DEBUG(tavor_cmd_post_spin, TAVOR_TNF_TRACE, "");
  84 
  85                 /* Write the command to the HCR */
  86                 status = tavor_write_hcr(state, cmdpost, 0);
  87                 if (status != TAVOR_CMD_SUCCESS) {
  88                         TNF_PROBE_0(tavor_cmd_post_fail,
  89                             TAVOR_TNF_ERROR, "");
  90                         TAVOR_TNF_EXIT(tavor_cmd_post);
  91                         return (status);
  92                 }
  93 
  94                 TAVOR_TNF_EXIT(tavor_cmd_post);
  95                 return (TAVOR_CMD_SUCCESS);
  96 
  97         } else {  /* "TAVOR_CMD_SLEEP_NOSPIN" */
  98 
  99                 TNF_PROBE_0_DEBUG(tavor_cmd_post_nospin, TAVOR_TNF_TRACE, "");
 100 
 101                 ASSERT(TAVOR_SLEEPFLAG_FOR_CONTEXT() != TAVOR_NOSLEEP);
 102 
 103                 /* NOTE: Expect threads to be waiting in here */
 104                 status = tavor_outstanding_cmd_alloc(state, &cmdptr,
 105                     cmdpost->cp_flags);
 106                 if (status != TAVOR_CMD_SUCCESS) {
 107                         TNF_PROBE_0(tavor_cmd_alloc_fail, TAVOR_TNF_ERROR, "");
 108                         TAVOR_TNF_EXIT(tavor_cmd_post);
 109                         return (status);
 110                 }
 111                 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdptr))
 112 
 113                 /*
 114                  * Set status to "TAVOR_CMD_INVALID_STATUS".  It is
 115                  * appropriate to do this here without the "cmd_comp_lock"
 116                  * because this register is overloaded.  Later it will be
 117                  * used to indicate - through a change from this invalid
 118                  * value to some other value - that the condition variable
 119                  * has been signaled.  Once it has, status will then contain
 120                  * the _real_ completion status
 121                  */
 122                 cmdptr->cmd_status = TAVOR_CMD_INVALID_STATUS;
 123 
 124                 /* Write the command to the HCR */
 125                 token = (uint16_t)cmdptr->cmd_indx;
 126                 status = tavor_write_hcr(state, cmdpost, token);
 127                 if (status != TAVOR_CMD_SUCCESS) {
 128                         tavor_outstanding_cmd_free(state, &cmdptr);
 129                         TNF_PROBE_0(tavor_cmd_post_fail, TAVOR_TNF_ERROR, "");
 130                         TAVOR_TNF_EXIT(tavor_cmd_post);
 131                         return (status);
 132                 }
 133 
 134                 /*
 135                  * cv_wait() on the "command_complete" condition variable.
 136                  * Note: We have the "__lock_lint" here to workaround warlock.
 137                  * Since warlock doesn't know that other parts of the Tavor
 138                  * may occasionally call this routine while holding their own
 139                  * locks, it complains about this cv_wait.  In reality,
 140                  * however, the rest of the driver never calls this routine
 141                  * with a lock held unless they pass TAVOR_CMD_NOSLEEP.
 142                  */
 143                 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*cmdptr))
 144                 mutex_enter(&cmdptr->cmd_comp_lock);
 145                 while (cmdptr->cmd_status == TAVOR_CMD_INVALID_STATUS) {
 146 #ifndef __lock_lint
 147                         cv_wait(&cmdptr->cmd_comp_cv, &cmdptr->cmd_comp_lock);
 148                         /* NOTE: EXPECT SEVERAL THREADS TO BE WAITING HERE */
 149 #endif
 150                 }
 151                 mutex_exit(&cmdptr->cmd_comp_lock);
 152                 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdptr))
 153 
 154                 /*
 155                  * Wake up after command completes (cv_signal).  Read status
 156                  * from the command (success, fail, etc.).  It is appropriate
 157                  * here (as above) to read the status field without the
 158                  * "cmd_comp_lock" because it is no longer being used to
 159                  * indicate whether the condition variable has been signaled
 160                  * (i.e. at this point we are certain that it already has).
 161                  */
 162                 status = cmdptr->cmd_status;
 163 
 164                 /* Save the "outparam" values into the cmdpost struct */
 165                 cmdpost->cp_outparm = cmdptr->cmd_outparm;
 166 
 167                 /*
 168                  * Add the command back to the "outstanding commands list".
 169                  * Signal the "cmd_list" condition variable, if necessary.
 170                  */
 171                 tavor_outstanding_cmd_free(state, &cmdptr);
 172 
 173                 if (status != TAVOR_CMD_SUCCESS) {
 174                         TNF_PROBE_0(tavor_cmd_post_fail, TAVOR_TNF_ERROR, "");
 175                         TAVOR_TNF_EXIT(tavor_cmd_post);
 176                         return (status);
 177                 }
 178 
 179                 TAVOR_TNF_EXIT(tavor_cmd_post);
 180                 return (TAVOR_CMD_SUCCESS);
 181         }
 182 }
 183 
 184 
 185 /*
 186  * tavor_mbox_alloc()
 187  *    Context: Can be called from interrupt or base context.
 188  *
 189  *    The "mbox_wait" parameter is used to determine whether to
 190  *    wait for a mailbox to become available or not.
 191  */
 192 int
 193 tavor_mbox_alloc(tavor_state_t *state, tavor_mbox_info_t *mbox_info,
 194     uint_t mbox_wait)
 195 {
 196         int                     status;
 197         uint_t                  sleep_context;
 198 
 199         TAVOR_TNF_ENTER(tavor_mbox_alloc);
 200 
 201         sleep_context = TAVOR_SLEEPFLAG_FOR_CONTEXT();
 202 
 203         /* Allocate an "In" mailbox */
 204         if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
 205                 /* Determine correct mboxlist based on calling context */
 206                 if (sleep_context == TAVOR_NOSLEEP) {
 207                         status = tavor_impl_mbox_alloc(state,
 208                             &state->ts_in_intr_mblist,
 209                             &mbox_info->mbi_in, mbox_wait);
 210 
 211                         ASSERT(status == TAVOR_CMD_SUCCESS);
 212                 } else {
 213                         /* NOTE: Expect threads to be waiting in here */
 214                         status = tavor_impl_mbox_alloc(state,
 215                             &state->ts_in_mblist, &mbox_info->mbi_in,
 216                             mbox_wait);
 217                         if (status != TAVOR_CMD_SUCCESS) {
 218                                 TAVOR_TNF_EXIT(tavor_mbox_alloc);
 219                                 return (status);
 220                         }
 221                 }
 222 
 223         }
 224 
 225         /* Allocate an "Out" mailbox */
 226         if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
 227                 /* Determine correct mboxlist based on calling context */
 228                 if (sleep_context == TAVOR_NOSLEEP) {
 229                         status = tavor_impl_mbox_alloc(state,
 230                             &state->ts_out_intr_mblist,
 231                             &mbox_info->mbi_out, mbox_wait);
 232 
 233                         ASSERT(status == TAVOR_CMD_SUCCESS);
 234                 } else {
 235                         /* NOTE: Expect threads to be waiting in here */
 236                         status = tavor_impl_mbox_alloc(state,
 237                             &state->ts_out_mblist, &mbox_info->mbi_out,
 238                             mbox_wait);
 239                         if (status != TAVOR_CMD_SUCCESS) {
 240                                 /* If we allocated an "In" mailbox, free it */
 241                                 if (mbox_info->mbi_alloc_flags &
 242                                     TAVOR_ALLOC_INMBOX) {
 243                                         tavor_impl_mbox_free(
 244                                             &state->ts_in_mblist,
 245                                             &mbox_info->mbi_in);
 246                                 }
 247                                 TAVOR_TNF_EXIT(tavor_mbox_alloc);
 248                                 return (status);
 249                         }
 250                 }
 251         }
 252 
 253         /* Store appropriate context in mbox_info */
 254         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mbox_info->mbi_sleep_context))
 255         mbox_info->mbi_sleep_context = sleep_context;
 256 
 257         TAVOR_TNF_EXIT(tavor_mbox_alloc);
 258         return (TAVOR_CMD_SUCCESS);
 259 }
 260 
 261 
 262 /*
 263  * tavor_mbox_free()
 264  *    Context: Can be called from interrupt or base context.
 265  */
 266 void
 267 tavor_mbox_free(tavor_state_t *state, tavor_mbox_info_t *mbox_info)
 268 {
 269         TAVOR_TNF_ENTER(tavor_mbox_free);
 270 
 271         /*
 272          * The mailbox has to be freed in the same context from which it was
 273          * allocated.  The context is stored in the mbox_info at
 274          * tavor_mbox_alloc() time.  We check the stored context against the
 275          * current context here.
 276          */
 277         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mbox_info->mbi_sleep_context))
 278         ASSERT(mbox_info->mbi_sleep_context == TAVOR_SLEEPFLAG_FOR_CONTEXT());
 279 
 280         /* Determine correct mboxlist based on calling context */
 281         if (mbox_info->mbi_sleep_context == TAVOR_NOSLEEP) {
 282                 /* Free the intr "In" mailbox */
 283                 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
 284                         tavor_impl_mbox_free(&state->ts_in_intr_mblist,
 285                             &mbox_info->mbi_in);
 286                 }
 287 
 288                 /* Free the intr "Out" mailbox */
 289                 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
 290                         tavor_impl_mbox_free(&state->ts_out_intr_mblist,
 291                             &mbox_info->mbi_out);
 292                 }
 293         } else {
 294                 /* Free the "In" mailbox */
 295                 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
 296                         tavor_impl_mbox_free(&state->ts_in_mblist,
 297                             &mbox_info->mbi_in);
 298                 }
 299 
 300                 /* Free the "Out" mailbox */
 301                 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
 302                         tavor_impl_mbox_free(&state->ts_out_mblist,
 303                             &mbox_info->mbi_out);
 304                 }
 305         }
 306 
 307         TAVOR_TNF_EXIT(tavor_mbox_free);
 308 }
 309 
 310 
 311 /*
 312  * tavor_cmd_complete_handler()
 313  *    Context: Called only from interrupt context.
 314  */
 315 int
 316 tavor_cmd_complete_handler(tavor_state_t *state, tavor_eqhdl_t eq,
 317     tavor_hw_eqe_t *eqe)
 318 {
 319         tavor_cmd_t             *cmdp;
 320         uint_t                  eqe_evttype;
 321 
 322         TAVOR_TNF_ENTER(tavor_cmd_complete_handler);
 323 
 324         eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
 325 
 326         ASSERT(eqe_evttype == TAVOR_EVT_COMMAND_INTF_COMP ||
 327             eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
 328 
 329         if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
 330                 TNF_PROBE_0(tavor_cmd_complete_overflow_condition,
 331                     TAVOR_TNF_ERROR, "");
 332                 tavor_eq_overflow_handler(state, eq, eqe);
 333 
 334                 TAVOR_TNF_EXIT(tavor_cmd_complete_handler);
 335                 return (DDI_FAILURE);
 336         }
 337 
 338         /*
 339          * Find the outstanding command pointer based on value returned
 340          * in "token"
 341          */
 342         cmdp = &state->ts_cmd_list.cml_cmd[TAVOR_EQE_CMDTOKEN_GET(eq, eqe)];
 343 
 344         /* Signal the waiting thread */
 345         mutex_enter(&cmdp->cmd_comp_lock);
 346         cmdp->cmd_outparm = ((uint64_t)TAVOR_EQE_CMDOUTP0_GET(eq, eqe) << 32) |
 347             TAVOR_EQE_CMDOUTP1_GET(eq, eqe);
 348         cmdp->cmd_status = TAVOR_EQE_CMDSTATUS_GET(eq, eqe);
 349 
 350         cv_signal(&cmdp->cmd_comp_cv);
 351         mutex_exit(&cmdp->cmd_comp_lock);
 352 
 353         TAVOR_TNF_EXIT(tavor_cmd_complete_handler);
 354         return (DDI_SUCCESS);
 355 }
 356 
 357 
 358 /*
 359  * tavor_inmbox_list_init()
 360  *    Context: Only called from attach() path context
 361  */
 362 int
 363 tavor_inmbox_list_init(tavor_state_t *state)
 364 {
 365         int             status;
 366         uint_t          num_inmbox;
 367 
 368         TAVOR_TNF_ENTER(tavor_inmbox_list_init);
 369 
 370         /* Initialize the "In" mailbox list */
 371         num_inmbox  =  (1 << state->ts_cfg_profile->cp_log_num_inmbox);
 372         status = tavor_impl_mboxlist_init(state, &state->ts_in_mblist,
 373             num_inmbox, TAVOR_IN_MBOX);
 374         if (status != DDI_SUCCESS) {
 375                 TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
 376                     TAVOR_TNF_ERROR, "");
 377                 TAVOR_TNF_EXIT(tavor_inmbox_list_init);
 378                 return (DDI_FAILURE);
 379         }
 380 
 381         TAVOR_TNF_EXIT(tavor_inmbox_list_init);
 382         return (DDI_SUCCESS);
 383 }
 384 
 385 
 386 /*
 387  * tavor_intr_inmbox_list_init()
 388  *    Context: Only called from attach() path context
 389  */
 390 int
 391 tavor_intr_inmbox_list_init(tavor_state_t *state)
 392 {
 393         int             status;
 394         uint_t          num_inmbox;
 395 
 396         TAVOR_TNF_ENTER(tavor_intr_inmbox_list_init);
 397 
 398         /* Initialize the interrupt "In" mailbox list */
 399         num_inmbox  =  (1 << state->ts_cfg_profile->cp_log_num_intr_inmbox);
 400         status = tavor_impl_mboxlist_init(state, &state->ts_in_intr_mblist,
 401             num_inmbox, TAVOR_INTR_IN_MBOX);
 402         if (status != DDI_SUCCESS) {
 403                 TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
 404                     TAVOR_TNF_ERROR, "");
 405                 TAVOR_TNF_EXIT(tavor_intr_inmbox_list_init);
 406                 return (DDI_FAILURE);
 407         }
 408 
 409         TAVOR_TNF_EXIT(tavor_intr_inmbox_list_init);
 410         return (DDI_SUCCESS);
 411 }
 412 
 413 
 414 /*
 415  * tavor_outmbox_list_init()
 416  *    Context: Only called from attach() path context
 417  */
 418 int
 419 tavor_outmbox_list_init(tavor_state_t *state)
 420 {
 421         int             status;
 422         uint_t          num_outmbox;
 423 
 424         TAVOR_TNF_ENTER(tavor_outmbox_list_init);
 425 
 426         /* Initialize the "Out" mailbox list */
 427         num_outmbox  =  (1 << state->ts_cfg_profile->cp_log_num_outmbox);
 428         status = tavor_impl_mboxlist_init(state, &state->ts_out_mblist,
 429             num_outmbox, TAVOR_OUT_MBOX);
 430         if (status != DDI_SUCCESS) {
 431                 TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
 432                     TAVOR_TNF_ERROR, "");
 433                 TAVOR_TNF_EXIT(tavor_outmbox_list_init);
 434                 return (DDI_FAILURE);
 435         }
 436 
 437         TAVOR_TNF_EXIT(tavor_outmbox_list_init);
 438         return (DDI_SUCCESS);
 439 }
 440 
 441 
 442 /*
 443  * tavor_intr_outmbox_list_init()
 444  *    Context: Only called from attach() path context
 445  */
 446 int
 447 tavor_intr_outmbox_list_init(tavor_state_t *state)
 448 {
 449         int             status;
 450         uint_t          num_outmbox;
 451 
 452         TAVOR_TNF_ENTER(tavor_intr_outmbox_list_init);
 453 
 454         /* Initialize the interrupts "Out" mailbox list */
 455         num_outmbox  =  (1 << state->ts_cfg_profile->cp_log_num_intr_outmbox);
 456         status = tavor_impl_mboxlist_init(state, &state->ts_out_intr_mblist,
 457             num_outmbox, TAVOR_INTR_OUT_MBOX);
 458         if (status != DDI_SUCCESS) {
 459                 TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
 460                     TAVOR_TNF_ERROR, "");
 461                 TAVOR_TNF_EXIT(tavor_intr_outmbox_list_init);
 462                 return (DDI_FAILURE);
 463         }
 464 
 465         TAVOR_TNF_EXIT(tavor_intr_outmbox_list_init);
 466         return (DDI_SUCCESS);
 467 }
 468 
 469 
 470 /*
 471  * tavor_inmbox_list_fini()
 472  *    Context: Only called from attach() and/or detach() path contexts
 473  */
 474 void
 475 tavor_inmbox_list_fini(tavor_state_t *state)
 476 {
 477         TAVOR_TNF_ENTER(tavor_inmbox_list_fini);
 478 
 479         /* Free up the "In" mailbox list */
 480         tavor_impl_mboxlist_fini(state, &state->ts_in_mblist);
 481 
 482         TAVOR_TNF_EXIT(tavor_inmbox_list_fini);
 483 }
 484 
 485 
 486 /*
 487  * tavor_intr_inmbox_list_fini()
 488  *    Context: Only called from attach() and/or detach() path contexts
 489  */
 490 void
 491 tavor_intr_inmbox_list_fini(tavor_state_t *state)
 492 {
 493         TAVOR_TNF_ENTER(tavor_intr_inmbox_list_fini);
 494 
 495         /* Free up the interupts "In" mailbox list */
 496         tavor_impl_mboxlist_fini(state, &state->ts_in_intr_mblist);
 497 
 498         TAVOR_TNF_EXIT(tavor_intr_inmbox_list_fini);
 499 }
 500 
 501 
 502 /*
 503  * tavor_outmbox_list_fini()
 504  *    Context: Only called from attach() and/or detach() path contexts
 505  */
 506 void
 507 tavor_outmbox_list_fini(tavor_state_t *state)
 508 {
 509         TAVOR_TNF_ENTER(tavor_outmbox_list_fini);
 510 
 511         /* Free up the "Out" mailbox list */
 512         tavor_impl_mboxlist_fini(state, &state->ts_out_mblist);
 513 
 514         TAVOR_TNF_EXIT(tavor_outmbox_list_fini);
 515 }
 516 
 517 
 518 /*
 519  * tavor_intr_outmbox_list_fini()
 520  *    Context: Only called from attach() and/or detach() path contexts
 521  */
 522 void
 523 tavor_intr_outmbox_list_fini(tavor_state_t *state)
 524 {
 525         TAVOR_TNF_ENTER(tavor_intr_outmbox_list_fini);
 526 
 527         /* Free up the interrupt "Out" mailbox list */
 528         tavor_impl_mboxlist_fini(state, &state->ts_out_intr_mblist);
 529 
 530         TAVOR_TNF_EXIT(tavor_intr_outmbox_list_fini);
 531 }
 532 
 533 
 534 /*
 535  * tavor_impl_mbox_alloc()
 536  *    Context: Can be called from interrupt or base context.
 537  */
 538 static int
 539 tavor_impl_mbox_alloc(tavor_state_t *state, tavor_mboxlist_t *mblist,
 540     tavor_mbox_t **mb, uint_t mbox_wait)
 541 {
 542         tavor_mbox_t    *mbox_ptr;
 543         uint_t          index, next, prev;
 544         uint_t          count, countmax;
 545 
 546         TAVOR_TNF_ENTER(tavor_impl_mbox_alloc);
 547 
 548         /*
 549          * If the mailbox list is empty, then wait (if appropriate in the
 550          * current context).  Otherwise, grab the next available mailbox.
 551          */
 552         if (mbox_wait == TAVOR_NOSLEEP) {
 553                 count    = 0;
 554                 countmax = state->ts_cfg_profile->cp_cmd_poll_max;
 555 
 556                 mutex_enter(&mblist->mbl_lock);
 557                 mblist->mbl_pollers++;
 558                 while (mblist->mbl_entries_free == 0) {
 559                         mutex_exit(&mblist->mbl_lock);
 560                         /* Delay loop polling for an available mbox */
 561                         if (++count > countmax) {
 562                                 TNF_PROBE_0(tavor_impl_mbox_alloc_fail,
 563                                     TAVOR_TNF_ERROR, "");
 564                                 TAVOR_TNF_EXIT(tavor_impl_mbox_alloc);
 565                                 return (TAVOR_CMD_INSUFF_RSRC);
 566                         }
 567 
 568                         /* Delay before polling for mailbox again */
 569                         drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
 570                         mutex_enter(&mblist->mbl_lock);
 571                 }
 572                 mblist->mbl_pollers--;
 573 
 574         /* TAVOR_SLEEP */
 575         } else {
 576                 /*
 577                  * Grab lock here as we prepare to cv_wait if needed.
 578                  */
 579                 mutex_enter(&mblist->mbl_lock);
 580                 while (mblist->mbl_entries_free == 0) {
 581                         /*
 582                          * Wait (on cv) for a mailbox to become free.  Note:
 583                          * Just as we do above in tavor_cmd_post(), we also
 584                          * have the "__lock_lint" here to workaround warlock.
 585                          * Warlock doesn't know that other parts of the Tavor
 586                          * may occasionally call this routine while holding
 587                          * their own locks, so it complains about this cv_wait.
 588                          * In reality, however, the rest of the driver never
 589                          * calls this routine with a lock held unless they pass
 590                          * TAVOR_CMD_NOSLEEP.
 591                          */
 592                         mblist->mbl_waiters++;
 593 #ifndef __lock_lint
 594                         cv_wait(&mblist->mbl_cv, &mblist->mbl_lock);
 595 #endif
 596                 }
 597         }
 598 
 599         /* Grab the next available mailbox from list */
 600         mbox_ptr = mblist->mbl_mbox;
 601         index    = mblist->mbl_head_indx;
 602         next     = mbox_ptr[index].mb_next;
 603         prev     = mbox_ptr[index].mb_prev;
 604 
 605         /* Remove it from the mailbox list */
 606         mblist->mbl_mbox[next].mb_prev       = prev;
 607         mblist->mbl_mbox[prev].mb_next       = next;
 608         mblist->mbl_head_indx                = next;
 609 
 610         /* Update the "free" count and return the mailbox pointer */
 611         mblist->mbl_entries_free--;
 612         *mb = &mbox_ptr[index];
 613 
 614         mutex_exit(&mblist->mbl_lock);
 615 
 616         TAVOR_TNF_EXIT(tavor_impl_mbox_alloc);
 617         return (TAVOR_CMD_SUCCESS);
 618 }
 619 
 620 
 621 /*
 622  * tavor_impl_mbox_free()
 623  *    Context: Can be called from interrupt or base context.
 624  */
 625 static void
 626 tavor_impl_mbox_free(tavor_mboxlist_t *mblist, tavor_mbox_t **mb)
 627 {
 628         uint_t          mbox_indx;
 629 
 630         TAVOR_TNF_ENTER(tavor_impl_mbox_free);
 631 
 632         mutex_enter(&mblist->mbl_lock);
 633 
 634         /* Pull the "index" from mailbox entry */
 635         mbox_indx = (*mb)->mb_indx;
 636 
 637         /*
 638          * If mailbox list is not empty, then insert the entry.  Otherwise,
 639          * this is the only entry.  So update the pointers appropriately.
 640          */
 641         if (mblist->mbl_entries_free++ != 0) {
 642                 /* Update the current mailbox */
 643                 (*mb)->mb_next = mblist->mbl_head_indx;
 644                 (*mb)->mb_prev = mblist->mbl_tail_indx;
 645 
 646                 /* Update head and tail mailboxes */
 647                 mblist->mbl_mbox[mblist->mbl_head_indx].mb_prev = mbox_indx;
 648                 mblist->mbl_mbox[mblist->mbl_tail_indx].mb_next = mbox_indx;
 649 
 650                 /* Update tail index */
 651                 mblist->mbl_tail_indx = mbox_indx;
 652 
 653         } else {
 654                 /* Update the current mailbox */
 655                 (*mb)->mb_next = mbox_indx;
 656                 (*mb)->mb_prev = mbox_indx;
 657 
 658                 /* Update head and tail indexes */
 659                 mblist->mbl_tail_indx = mbox_indx;
 660                 mblist->mbl_head_indx = mbox_indx;
 661         }
 662 
 663         /*
 664          * Because we can have both waiters (SLEEP treads waiting for a
 665          * cv_signal to continue processing) and pollers (NOSLEEP treads
 666          * polling for a mailbox to become available), we try to share CPU time
 667          * between them.  We do this by signalling the waiters only every other
 668          * call to mbox_free.  This gives the pollers a chance to get some CPU
 669          * time to do their command.  If we signalled every time, the pollers
 670          * would have a much harder time getting CPU time.
 671          *
 672          * If there are waiters and no pollers, then we signal always.
 673          *
 674          * Otherwise, if there are either no waiters, there may in fact be
 675          * pollers, so we do not signal in that case.
 676          */
 677         if (mblist->mbl_pollers > 0 && mblist->mbl_waiters > 0) {
 678                 /* flip the signal value */
 679                 mblist->mbl_signal = (++mblist->mbl_signal) % 2;
 680         } else if (mblist->mbl_waiters > 0) {
 681                 mblist->mbl_signal = 1;
 682         } else {
 683                 mblist->mbl_signal = 0;
 684         }
 685 
 686         /*
 687          * Depending on the conditions in the previous check, we signal only if
 688          * we are supposed to.
 689          */
 690         if (mblist->mbl_signal) {
 691                 mblist->mbl_waiters--;
 692                 cv_signal(&mblist->mbl_cv);
 693         }
 694 
 695         /* Clear out the mailbox entry pointer */
 696         *mb = NULL;
 697 
 698         mutex_exit(&mblist->mbl_lock);
 699 
 700         TAVOR_TNF_EXIT(tavor_impl_mbox_free);
 701 }
 702 
 703 
 704 /*
 705  * tavor_impl_mboxlist_init()
 706  *    Context: Only called from attach() path context
 707  */
 708 static int
 709 tavor_impl_mboxlist_init(tavor_state_t *state, tavor_mboxlist_t *mblist,
 710     uint_t num_mbox, tavor_rsrc_type_t type)
 711 {
 712         tavor_rsrc_t            *rsrc;
 713         ddi_dma_cookie_t        dma_cookie;
 714         uint_t                  dma_cookiecnt, flag, sync;
 715         int                     status, i;
 716 
 717         TAVOR_TNF_ENTER(tavor_impl_mboxlist_init);
 718 
 719         /* Allocate the memory for the mailbox entries list */
 720         mblist->mbl_list_sz = num_mbox;
 721         mblist->mbl_mbox = kmem_zalloc(mblist->mbl_list_sz *
 722             sizeof (tavor_mbox_t), KM_SLEEP);
 723 
 724         /* Initialize the mailbox entries list */
 725         mblist->mbl_head_indx         = 0;
 726         mblist->mbl_tail_indx         = mblist->mbl_list_sz - 1;
 727         mblist->mbl_entries_free = mblist->mbl_list_sz;
 728         mblist->mbl_waiters   = 0;
 729         mblist->mbl_num_alloc         = 0;
 730 
 731         /* Set up the mailbox list's cv and mutex */
 732         mutex_init(&mblist->mbl_lock, NULL, MUTEX_DRIVER,
 733             DDI_INTR_PRI(state->ts_intrmsi_pri));
 734         cv_init(&mblist->mbl_cv, NULL, CV_DRIVER, NULL);
 735 
 736         /* Determine if syncs will be necessary */
 737         sync = TAVOR_MBOX_IS_SYNC_REQ(state, type);
 738 
 739         /* Determine whether to map DDI_DMA_STREAMING or DDI_DMA_CONSISTENT */
 740         flag = state->ts_cfg_profile->cp_streaming_consistent;
 741 
 742         /* Initialize the mailbox list entries */
 743         for (i = 0; i < mblist->mbl_list_sz; i++) {
 744                 /* Allocate resources for the mailbox */
 745                 status = tavor_rsrc_alloc(state, type, 1, TAVOR_SLEEP,
 746                     &rsrc);
 747                 if (status != DDI_SUCCESS) {
 748                         /* Jump to cleanup and return error */
 749                         TNF_PROBE_0(tavor_impl_mbox_init_rsrcalloc_fail,
 750                             TAVOR_TNF_ERROR, "");
 751                         goto mboxlist_init_fail;
 752                 }
 753 
 754                 /* Save away the mailbox resource info */
 755                 mblist->mbl_mbox[i].mb_rsrcptr       = rsrc;
 756                 mblist->mbl_mbox[i].mb_addr  = rsrc->tr_addr;
 757                 mblist->mbl_mbox[i].mb_acchdl        = rsrc->tr_acchdl;
 758 
 759                 /*
 760                  * Get a PCI mapped address for each mailbox.  Note: this
 761                  * uses the ddi_dma_handle return from the resource
 762                  * allocation routine
 763                  */
 764                 status = ddi_dma_addr_bind_handle(rsrc->tr_dmahdl, NULL,
 765                     rsrc->tr_addr, rsrc->tr_len, (DDI_DMA_RDWR | flag),
 766                     DDI_DMA_SLEEP, NULL, &dma_cookie, &dma_cookiecnt);
 767                 if (status != DDI_SUCCESS) {
 768                         /* Jump to cleanup and return error */
 769                         tavor_rsrc_free(state, &rsrc);
 770                         TNF_PROBE_0(tavor_impl_mbox_init_dmabind_fail,
 771                             TAVOR_TNF_ERROR, "");
 772                         goto mboxlist_init_fail;
 773                 }
 774 
 775                 /* Save away the mapped address for the mailbox */
 776                 mblist->mbl_mbox[i].mb_mapaddr       = dma_cookie.dmac_laddress;
 777 
 778                 /* Set sync flag appropriately */
 779                 mblist->mbl_mbox[i].mb_sync  = sync;
 780 
 781                 /* Make each entry point to the "next" and "prev" entries */
 782                 mblist->mbl_mbox[i].mb_next  = i+1;
 783                 mblist->mbl_mbox[i].mb_prev  = i-1;
 784                 mblist->mbl_mbox[i].mb_indx  = i;
 785                 mblist->mbl_num_alloc                = i + 1;
 786         }
 787 
 788         /* Make the "head" and "tail" entries point to each other */
 789         mblist->mbl_mbox[mblist->mbl_head_indx].mb_prev =
 790             mblist->mbl_tail_indx;
 791         mblist->mbl_mbox[mblist->mbl_tail_indx].mb_next =
 792             mblist->mbl_head_indx;
 793 
 794         TAVOR_TNF_EXIT(tavor_impl_mboxlist_init);
 795         return (DDI_SUCCESS);
 796 
 797 mboxlist_init_fail:
 798         tavor_impl_mboxlist_fini(state, mblist);
 799 
 800         TAVOR_TNF_EXIT(tavor_impl_mboxlist_init);
 801         return (DDI_FAILURE);
 802 }
 803 
 804 
 805 /*
 806  * tavor_impl_mboxlist_fini()
 807  *    Context: Only called from attach() and/or detach() path contexts
 808  */
 809 static void
 810 tavor_impl_mboxlist_fini(tavor_state_t *state, tavor_mboxlist_t *mblist)
 811 {
 812         tavor_rsrc_t    *rsrc;
 813         int             i, status;
 814 
 815         TAVOR_TNF_ENTER(tavor_impl_mboxlist_fini);
 816 
 817         /* Release the resources for each of the mailbox list entries */
 818         for (i = 0; i < mblist->mbl_num_alloc; i++) {
 819                 rsrc = mblist->mbl_mbox[i].mb_rsrcptr;
 820 
 821                 /*
 822                  * First, unbind the DMA memory for the mailbox
 823                  *
 824                  * Note: The only way ddi_dma_unbind_handle() currently
 825                  * can return an error is if the handle passed in is invalid.
 826                  * Since this should never happen, we choose to return void
 827                  * from this function!  If this does return an error,
 828                  * however, then we print a warning message to the console.
 829                  */
 830                 status = ddi_dma_unbind_handle(rsrc->tr_dmahdl);
 831                 if (status != DDI_SUCCESS) {
 832                         TAVOR_WARNING(state, "failed to unbind DMA mapping");
 833                         TNF_PROBE_0(tavor_impl_mboxlist_fini_dmaunbind_fail,
 834                             TAVOR_TNF_ERROR, "");
 835                         TAVOR_TNF_EXIT(tavor_impl_mboxlist_fini);
 836                         return;
 837                 }
 838 
 839                 /* Next, free the mailbox resource */
 840                 tavor_rsrc_free(state, &rsrc);
 841         }
 842 
 843         /* Destroy the mailbox list mutex and cv */
 844         mutex_destroy(&mblist->mbl_lock);
 845         cv_destroy(&mblist->mbl_cv);
 846 
 847         /* Free up the memory for tracking the mailbox list */
 848         kmem_free(mblist->mbl_mbox, mblist->mbl_list_sz *
 849             sizeof (tavor_mbox_t));
 850 
 851         TAVOR_TNF_EXIT(tavor_impl_mboxlist_fini);
 852 }
 853 
 854 
 855 /*
 856  * tavor_outstanding_cmd_alloc()
 857  *    Context: Can be called only from base context.
 858  */
 859 static int
 860 tavor_outstanding_cmd_alloc(tavor_state_t *state, tavor_cmd_t **cmd_ptr,
 861     uint_t cmd_wait)
 862 {
 863         tavor_cmdlist_t *cmd_list;
 864         uint_t          next, prev, head;
 865 
 866         TAVOR_TNF_ENTER(tavor_outstanding_cmd_alloc);
 867 
 868         cmd_list = &state->ts_cmd_list;
 869         mutex_enter(&cmd_list->cml_lock);
 870 
 871         /* Ensure that outstanding commands are supported */
 872         ASSERT(cmd_list->cml_num_alloc != 0);
 873 
 874         /*
 875          * If the outstanding command list is empty, then wait (if
 876          * appropriate in the current context).  Otherwise, grab the
 877          * next available command.
 878          */
 879         while (cmd_list->cml_entries_free == 0) {
 880                 /* No free commands */
 881                 if (cmd_wait == TAVOR_NOSLEEP) {
 882                         mutex_exit(&cmd_list->cml_lock);
 883                         TNF_PROBE_0(tavor_outstanding_cmd_alloc_fail,
 884                             TAVOR_TNF_ERROR, "");
 885                         TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
 886                         return (TAVOR_CMD_INSUFF_RSRC);
 887                 }
 888 
 889                 /*
 890                  * Wait (on cv) for a command to become free.  Note: Just
 891                  * as we do above in tavor_cmd_post(), we also have the
 892                  * "__lock_lint" here to workaround warlock.  Warlock doesn't
 893                  * know that other parts of the Tavor may occasionally call
 894                  * this routine while holding their own locks, so it complains
 895                  * about this cv_wait.  In reality, however, the rest of the
 896                  * driver never calls this routine with a lock held unless
 897                  * they pass TAVOR_CMD_NOSLEEP.
 898                  */
 899                 cmd_list->cml_waiters++;
 900 #ifndef __lock_lint
 901                 cv_wait(&cmd_list->cml_cv, &cmd_list->cml_lock);
 902 #endif
 903         }
 904 
 905         /* Grab the next available command from the list */
 906         head = cmd_list->cml_head_indx;
 907         *cmd_ptr = &cmd_list->cml_cmd[head];
 908         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(**cmd_ptr))
 909         next = (*cmd_ptr)->cmd_next;
 910         prev = (*cmd_ptr)->cmd_prev;
 911         (*cmd_ptr)->cmd_status = TAVOR_CMD_INVALID_STATUS;
 912 
 913         /* Remove it from the command list */
 914         cmd_list->cml_cmd[next].cmd_prev = prev;
 915         cmd_list->cml_cmd[prev].cmd_next = next;
 916         cmd_list->cml_head_indx               = next;
 917 
 918         /* Update the "free" count and return */
 919         cmd_list->cml_entries_free--;
 920 
 921         mutex_exit(&cmd_list->cml_lock);
 922 
 923         TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
 924         return (TAVOR_CMD_SUCCESS);
 925 }
 926 
 927 
 928 /*
 929  * tavor_outstanding_cmd_free()
 930  *    Context: Can be called only from base context.
 931  */
 932 static void
 933 tavor_outstanding_cmd_free(tavor_state_t *state, tavor_cmd_t **cmd_ptr)
 934 {
 935         tavor_cmdlist_t *cmd_list;
 936         uint_t          cmd_indx;
 937 
 938         TAVOR_TNF_ENTER(tavor_outstanding_cmd_free);
 939 
 940         cmd_list = &state->ts_cmd_list;
 941         mutex_enter(&cmd_list->cml_lock);
 942 
 943         /* Pull the "index" from command entry */
 944         cmd_indx = (*cmd_ptr)->cmd_indx;
 945 
 946         /*
 947          * If outstanding command list is not empty, then insert the entry.
 948          * Otherwise, this is the only entry.  So update the pointers
 949          * appropriately.
 950          */
 951         if (cmd_list->cml_entries_free++ != 0) {
 952                 /* Update the current command */
 953                 (*cmd_ptr)->cmd_next = cmd_list->cml_head_indx;
 954                 (*cmd_ptr)->cmd_prev = cmd_list->cml_tail_indx;
 955 
 956                 /* Update head and tail commands */
 957                 cmd_list->cml_cmd[cmd_list->cml_head_indx].cmd_prev = cmd_indx;
 958                 cmd_list->cml_cmd[cmd_list->cml_tail_indx].cmd_next = cmd_indx;
 959 
 960                 /* Update tail index */
 961                 cmd_list->cml_tail_indx = cmd_indx;
 962 
 963         } else {
 964                 /* Update the current command */
 965                 (*cmd_ptr)->cmd_next = cmd_indx;
 966                 (*cmd_ptr)->cmd_prev = cmd_indx;
 967 
 968                 /* Update head and tail indexes */
 969                 cmd_list->cml_head_indx = cmd_indx;
 970                 cmd_list->cml_tail_indx = cmd_indx;
 971         }
 972 
 973         /* If there are threads waiting, signal one of them */
 974         if (cmd_list->cml_waiters > 0) {
 975                 cmd_list->cml_waiters--;
 976                 cv_signal(&cmd_list->cml_cv);
 977         }
 978 
 979         /* Clear out the command entry pointer */
 980         *cmd_ptr = NULL;
 981 
 982         mutex_exit(&cmd_list->cml_lock);
 983 
 984         TAVOR_TNF_EXIT(tavor_outstanding_cmd_free);
 985 }
 986 
 987 
 988 /*
 989  * tavor_write_hcr()
 990  *    Context: Can be called from interrupt or base context.
 991  */
 992 static int
 993 tavor_write_hcr(tavor_state_t *state, tavor_cmd_post_t *cmdpost,
 994     uint16_t token)
 995 {
 996         tavor_hw_hcr_t  *hcr;
 997         uint_t          status, count, countmax;
 998         uint64_t        hcrreg;
 999 
1000         TAVOR_TNF_ENTER(tavor_write_hcr);
1001 
1002         /*
1003          * Grab the "HCR access" lock if the driver is not in
1004          * fastreboot. In fastreboot, this function is called
1005          * with the single thread but in high interrupt context
1006          * (so that this mutex lock cannot be used).
1007          */
1008 #ifdef __lock_lint
1009         mutex_enter(&state->ts_cmd_regs.hcr_lock);
1010 #else
1011         if (!TAVOR_IN_FASTREBOOT(state)) {
1012                 mutex_enter(&state->ts_cmd_regs.hcr_lock);
1013         }
1014 #endif
1015 
1016         hcr = state->ts_cmd_regs.hcr;
1017 
1018         /*
1019          * First, check the "go" bit to see if the previous hcr usage is
1020          * complete.  As long as it is set then we must continue to poll.
1021          */
1022         count    = 0;
1023         countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1024         for (;;) {
1025                 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1026 
1027                 /* If "go" bit is clear, then done */
1028                 if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1029                         TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1030                             TAVOR_TNF_ERROR, "", tnf_uint, nospinloopcount,
1031                             count);
1032                         break;
1033                 }
1034                 /* Delay before polling the "go" bit again */
1035                 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1036 
1037                 /*
1038                  * If we poll more than the maximum number of times, then
1039                  * return a "timeout" error.
1040                  */
1041                 if (++count > countmax) {
1042 #ifdef __lock_lint
1043                         mutex_exit(&state->ts_cmd_regs.hcr_lock);
1044 #else
1045                         if (!TAVOR_IN_FASTREBOOT(state)) {
1046                                 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1047                         }
1048 #endif
1049                         TNF_PROBE_0(tavor_write_hcr_timeout1, TAVOR_TNF_ERROR,
1050                             "");
1051                         TAVOR_TNF_EXIT(tavor_write_hcr);
1052                         return (TAVOR_CMD_TIMEOUT);
1053                 }
1054         }
1055 
1056         /* Write "inparam" as a 64-bit quantity */
1057         ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->in_param0,
1058             cmdpost->cp_inparm);
1059 
1060         /* Write "inmod" and 32-bits of "outparam" as 64-bit */
1061         hcrreg = ((uint64_t)cmdpost->cp_inmod << 32);
1062         hcrreg = hcrreg | (cmdpost->cp_outparm >> 32);
1063         ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->input_modifier,
1064             hcrreg);
1065 
1066         /* Write the other 32-bits of "outparam" and "token" as 64-bit */
1067         hcrreg = (cmdpost->cp_outparm << 32);
1068         hcrreg = hcrreg | ((uint32_t)token << TAVOR_HCR_TOKEN_SHIFT);
1069         ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->out_param1,
1070             hcrreg);
1071 
1072         /* Then setup the final hcrreg to hit doorbell (i.e. "go" bit) */
1073         hcrreg = TAVOR_HCR_CMD_GO_MASK;
1074         if (cmdpost->cp_flags == TAVOR_CMD_SLEEP_NOSPIN)
1075                 hcrreg = hcrreg | TAVOR_HCR_CMD_E_MASK;
1076         hcrreg = hcrreg | (cmdpost->cp_opmod << TAVOR_HCR_CMD_OPMOD_SHFT);
1077         hcrreg = hcrreg | (cmdpost->cp_opcode);
1078 
1079         /* Write the doorbell to the HCR */
1080         ddi_put32(state->ts_reg_cmdhdl, &hcr->cmd, hcrreg);
1081 
1082         /*
1083          * In the SPIN case we read the HCR and check the "go" bit.  For the
1084          * NOSPIN case we do not have to poll, we simply release the HCR lock
1085          * and return.
1086          */
1087         if (cmdpost->cp_flags == TAVOR_CMD_NOSLEEP_SPIN) {
1088                 count    = 0;
1089                 countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1090 
1091                 for (;;) {
1092                         hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1093 
1094                         /* If "go" bit is clear, then done */
1095                         if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1096                                 TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1097                                     TAVOR_TNF_ERROR, "", tnf_uint,
1098                                     spinloopcount, count);
1099                                 break;
1100                         }
1101                         /* Delay before polling the "go" bit again */
1102                         drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1103 
1104                         /*
1105                          * If we poll more than the maximum number of times,
1106                          * then return a "timeout" error.
1107                          */
1108                         if (++count > countmax) {
1109 #ifdef __lock_lint
1110                                 mutex_exit(&state-> ts_cmd_regs.hcr_lock);
1111 #else
1112                                 if (!TAVOR_IN_FASTREBOOT(state)) {
1113                                         mutex_exit(&state->
1114                                             ts_cmd_regs.hcr_lock);
1115                                 }
1116 #endif
1117                                 TNF_PROBE_0(tavor_write_hcr_timeout2,
1118                                     TAVOR_TNF_ERROR, "");
1119                                 TAVOR_TNF_EXIT(tavor_write_hcr);
1120                                 return (TAVOR_CMD_TIMEOUT);
1121                         }
1122                 }
1123 
1124                 /* Pull out the "status" bits from the HCR */
1125                 status = (hcrreg >> TAVOR_HCR_CMD_STATUS_SHFT);
1126 
1127                 /*
1128                  * Read the "outparam" value.  Note: we have to read "outparam"
1129                  * as two separate 32-bit reads because the field in the HCR is
1130                  * not 64-bit aligned.
1131                  */
1132                 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param0);
1133                 cmdpost->cp_outparm = hcrreg << 32;
1134                 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param1);
1135                 cmdpost->cp_outparm |= hcrreg;
1136 
1137         /* NOSPIN */
1138         } else {
1139                 status = TAVOR_CMD_SUCCESS;
1140         }
1141 
1142         /* Drop the "HCR access" lock */
1143 #ifdef __lock_lint
1144         mutex_exit(&state->ts_cmd_regs.hcr_lock);
1145 #else
1146         if (!TAVOR_IN_FASTREBOOT(state)) {
1147                 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1148         }
1149 #endif
1150 
1151         TAVOR_TNF_EXIT(tavor_write_hcr);
1152         return (status);
1153 }
1154 
1155 
1156 /*
1157  * tavor_outstanding_cmdlist_init()
1158  *    Context: Only called from attach() path context
1159  */
1160 int
1161 tavor_outstanding_cmdlist_init(tavor_state_t *state)
1162 {
1163         uint_t          num_outstanding_cmds, head, tail;
1164         int             i;
1165 
1166         TAVOR_TNF_ENTER(tavor_outstanding_cmdlist_init);
1167 
1168         /*
1169          * Determine the number of the outstanding commands supported
1170          * by the Tavor device (obtained from the QUERY_FW command).  Note:
1171          * Because we handle both SLEEP and NOSLEEP cases around the tavor HCR,
1172          * we know that when an interrupt comes in it will be next on the
1173          * command register, and will at most have to wait one commands time.
1174          * We do not have to reserve an outstanding command here for
1175          * interrupts.
1176          */
1177         num_outstanding_cmds = (1 << state->ts_fw.log_max_cmd);
1178 
1179         /* Initialize the outstanding command list */
1180         state->ts_cmd_list.cml_list_sz        = num_outstanding_cmds;
1181         state->ts_cmd_list.cml_head_indx = 0;
1182         state->ts_cmd_list.cml_tail_indx = state->ts_cmd_list.cml_list_sz - 1;
1183         state->ts_cmd_list.cml_entries_free = state->ts_cmd_list.cml_list_sz;
1184         state->ts_cmd_list.cml_waiters        = 0;
1185         state->ts_cmd_list.cml_num_alloc = 0;
1186 
1187         /* Allocate the memory for the outstanding command list */
1188         if (num_outstanding_cmds) {
1189                 state->ts_cmd_list.cml_cmd =
1190                     kmem_zalloc(state->ts_cmd_list.cml_list_sz *
1191                     sizeof (tavor_cmd_t), KM_SLEEP);
1192         }
1193         mutex_init(&state->ts_cmd_list.cml_lock, NULL, MUTEX_DRIVER,
1194             DDI_INTR_PRI(state->ts_intrmsi_pri));
1195         cv_init(&state->ts_cmd_list.cml_cv, NULL, CV_DRIVER, NULL);
1196 
1197         /* Initialize the individual outstanding command list entries */
1198         for (i = 0; i < state->ts_cmd_list.cml_list_sz; i++) {
1199                 mutex_init(&state->ts_cmd_list.cml_cmd[i].cmd_comp_lock,
1200                     NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri));
1201                 cv_init(&state->ts_cmd_list.cml_cmd[i].cmd_comp_cv, NULL,
1202                     CV_DRIVER, NULL);
1203 
1204                 state->ts_cmd_list.cml_cmd[i].cmd_next       = i+1;
1205                 state->ts_cmd_list.cml_cmd[i].cmd_prev       = i-1;
1206                 state->ts_cmd_list.cml_cmd[i].cmd_indx       = i;
1207                 state->ts_cmd_list.cml_num_alloc     = i + 1;
1208         }
1209         if (num_outstanding_cmds) {
1210                 head = state->ts_cmd_list.cml_head_indx;
1211                 tail = state->ts_cmd_list.cml_tail_indx;
1212                 state->ts_cmd_list.cml_cmd[head].cmd_prev =
1213                     state->ts_cmd_list.cml_tail_indx;
1214                 state->ts_cmd_list.cml_cmd[tail].cmd_next =
1215                     state->ts_cmd_list.cml_head_indx;
1216         }
1217 
1218         TAVOR_TNF_EXIT(tavor_outstanding_cmdlist_init);
1219         return (DDI_SUCCESS);
1220 }
1221 
1222 
1223 /*
1224  * tavor_outstanding_cmdlist_fini()
1225  *    Context: Only called from attach() and/or detach() path contexts
1226  */
1227 void
1228 tavor_outstanding_cmdlist_fini(tavor_state_t *state)
1229 {
1230         int             i;
1231 
1232         TAVOR_TNF_ENTER(tavor_outstanding_cmdlist_fini);
1233 
1234         /* Destroy the outstanding command list entries */
1235         for (i = 0; i < state->ts_cmd_list.cml_num_alloc; i++) {
1236                 mutex_destroy(&state->ts_cmd_list.cml_cmd[i].cmd_comp_lock);
1237                 cv_destroy(&state->ts_cmd_list.cml_cmd[i].cmd_comp_cv);
1238         }
1239 
1240         /* Destroy the lock (and cv) and free up memory for list */
1241         mutex_destroy(&state->ts_cmd_list.cml_lock);
1242         cv_destroy(&state->ts_cmd_list.cml_cv);
1243         if (state->ts_cmd_list.cml_num_alloc) {
1244                 kmem_free(state->ts_cmd_list.cml_cmd,
1245                     state->ts_cmd_list.cml_list_sz * sizeof (tavor_cmd_t));
1246         }
1247 
1248         TAVOR_TNF_EXIT(tavor_outstanding_cmdlist_fini);
1249 }
1250 
1251 
1252 /*
1253  * tavor_mbox_sync()
1254  */
1255 static void
1256 tavor_mbox_sync(tavor_mbox_t *mbox, uint_t offset, uint_t length,
1257     uint_t flag)
1258 {
1259         ddi_dma_handle_t        dmahdl;
1260         int                     status;
1261 
1262         TAVOR_TNF_ENTER(tavor_mbox_sync);
1263 
1264         /* Determine if mailbox needs to be synced or not */
1265         if (mbox->mb_sync == 0) {
1266                 TAVOR_TNF_EXIT(tavor_mbox_sync);
1267                 return;
1268         }
1269 
1270         /* Get the DMA handle from mailbox */
1271         dmahdl = mbox->mb_rsrcptr->tr_dmahdl;
1272 
1273         /* Calculate offset into mailbox */
1274         status = ddi_dma_sync(dmahdl, (off_t)offset, (size_t)length, flag);
1275         if (status != DDI_SUCCESS) {
1276                 TNF_PROBE_0(tavor_mbox_sync_fail, TAVOR_TNF_ERROR, "");
1277                 TAVOR_TNF_EXIT(tavor_mbox_sync);
1278                 return;
1279         }
1280 
1281         TAVOR_TNF_EXIT(tavor_mbox_sync);
1282 }
1283 
1284 
1285 /*
1286  * tavor_sys_en_cmd_post()
1287  *    Context: Can be called from interrupt or base context.
1288  *    (Currently called only from attach() path context)
1289  */
1290 int
1291 tavor_sys_en_cmd_post(tavor_state_t *state, uint_t flags,
1292     uint64_t *errorcode, uint_t sleepflag)
1293 {
1294         tavor_cmd_post_t        cmd;
1295         int                     status;
1296 
1297         TAVOR_TNF_ENTER(tavor_sys_en_cmd_post);
1298 
1299         /* Make sure we are called with the correct flag */
1300         ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1301 
1302         /* Setup and post the Tavor "SYS_EN" command */
1303         cmd.cp_inparm   = 0;
1304         cmd.cp_outparm  = 0;
1305         cmd.cp_inmod    = 0;
1306         cmd.cp_opcode   = SYS_EN;
1307         cmd.cp_opmod    = flags;
1308         cmd.cp_flags    = sleepflag;
1309         status = tavor_cmd_post(state, &cmd);
1310         if (status != TAVOR_CMD_SUCCESS) {
1311                 TNF_PROBE_0(tavor_sys_en_cmd_post_fail, TAVOR_TNF_ERROR, "");
1312                 /*
1313                  * When the SYS_EN command fails, the "outparam" field may
1314                  * contain more detailed information about what caused the
1315                  * failure.
1316                  */
1317                 *errorcode = cmd.cp_outparm;
1318         }
1319 
1320         TAVOR_TNF_EXIT(tavor_sys_en_cmd_post);
1321         return (status);
1322 }
1323 
1324 
1325 /*
1326  * tavor_sys_dis_cmd_post()
1327  *    Context: Can be called from interrupt or base context.
1328  *    (Currently called only from attach() and/or detach() path contexts)
1329  */
1330 int
1331 tavor_sys_dis_cmd_post(tavor_state_t *state, uint_t sleepflag)
1332 {
1333         tavor_cmd_post_t        cmd;
1334         int                     status;
1335 
1336         TAVOR_TNF_ENTER(tavor_sys_dis_cmd_post);
1337 
1338         /* Make sure we are called with the correct flag */
1339         ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1340 
1341         /* Setup and post the Tavor "SYS_DIS" command */
1342         cmd.cp_inparm   = 0;
1343         cmd.cp_outparm  = 0;
1344         cmd.cp_inmod    = 0;
1345         cmd.cp_opcode   = SYS_DIS;
1346         cmd.cp_opmod    = 0;
1347         cmd.cp_flags    = sleepflag;
1348         status = tavor_cmd_post(state, &cmd);
1349         if (status != TAVOR_CMD_SUCCESS) {
1350                 TNF_PROBE_0(tavor_sys_dis_cmd_post_fail,
1351                     TAVOR_TNF_ERROR, "");
1352         }
1353 
1354         TAVOR_TNF_EXIT(tavor_sys_dis_cmd_post);
1355         return (status);
1356 }
1357 
1358 
1359 /*
1360  * tavor_init_hca_cmd_post()
1361  *    Context: Can be called from interrupt or base context.
1362  *    (Currently called only from attach() path context)
1363  */
1364 int
1365 tavor_init_hca_cmd_post(tavor_state_t *state,
1366     tavor_hw_initqueryhca_t *inithca, uint_t sleepflag)
1367 {
1368         tavor_mbox_info_t       mbox_info;
1369         tavor_cmd_post_t        cmd;
1370         uint64_t                data;
1371         uint_t                  size;
1372         int                     status, i;
1373 
1374         TAVOR_TNF_ENTER(tavor_init_hca_cmd_post);
1375 
1376         /* Make sure we are called with the correct flag */
1377         ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1378 
1379         /* Get an "In" mailbox for the command */
1380         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
1381         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1382         if (status != TAVOR_CMD_SUCCESS) {
1383                 TNF_PROBE_0(tavor_init_hca_mbox_fail, TAVOR_TNF_ERROR, "");
1384                 TAVOR_TNF_EXIT(tavor_init_hca_cmd_post);
1385                 return (status);
1386         }
1387 
1388         /* Copy the Tavor "INIT_HCA" command into the mailbox */
1389         size = sizeof (tavor_hw_initqueryhca_t);
1390         for (i = 0; i < (size >> 3); i++) {
1391                 data = ((uint64_t *)inithca)[i];
1392                 ddi_put64(mbox_info.mbi_in->mb_acchdl,
1393                     ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
1394         }
1395 
1396         /* Sync the mailbox for the device to read */
1397         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1398 
1399         /* Setup and post the Tavor "INIT_HCA" command */
1400         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1401         cmd.cp_outparm  = 0;
1402         cmd.cp_inmod    = 0;
1403         cmd.cp_opcode   = INIT_HCA;
1404         cmd.cp_opmod    = 0;
1405         cmd.cp_flags    = sleepflag;
1406         status = tavor_cmd_post(state, &cmd);
1407         if (status != TAVOR_CMD_SUCCESS) {
1408                 TNF_PROBE_0(tavor_init_hca_cmd_post_fail,
1409                     TAVOR_TNF_ERROR, "");
1410         }
1411 
1412         /* Free the mailbox */
1413         tavor_mbox_free(state, &mbox_info);
1414 
1415         TAVOR_TNF_EXIT(tavor_init_hca_cmd_post);
1416         return (status);
1417 }
1418 
1419 
1420 /*
1421  * tavor_close_hca_cmd_post()
1422  *    Context: Can be called from interrupt or base context.
1423  *    (Currently called only from attach() and/or detach() path contexts)
1424  */
1425 int
1426 tavor_close_hca_cmd_post(tavor_state_t *state, uint_t sleepflag)
1427 {
1428         tavor_cmd_post_t        cmd;
1429         int                     status;
1430 
1431         TAVOR_TNF_ENTER(tavor_close_hca_cmd_post);
1432 
1433         /* Make sure we are called with the correct flag */
1434         ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1435 
1436         /* Setup and post the Tavor "CLOSE_HCA" command */
1437         cmd.cp_inparm   = 0;
1438         cmd.cp_outparm  = 0;
1439         cmd.cp_inmod    = 0;
1440         cmd.cp_opcode   = CLOSE_HCA;
1441         cmd.cp_opmod    = 0;
1442         cmd.cp_flags    = sleepflag;
1443         status = tavor_cmd_post(state, &cmd);
1444         if (status != TAVOR_CMD_SUCCESS) {
1445                 TNF_PROBE_0(tavor_close_hca_cmd_post_fail,
1446                     TAVOR_TNF_ERROR, "");
1447         }
1448 
1449         TAVOR_TNF_EXIT(tavor_close_hca_cmd_post);
1450         return (status);
1451 }
1452 
1453 
1454 /*
1455  * tavor_init_ib_cmd_post()
1456  *    Context: Can be called from interrupt or base context.
1457  *    (Currently called only from attach() path context)
1458  */
1459 int
1460 tavor_init_ib_cmd_post(tavor_state_t *state, tavor_hw_initib_t *initib,
1461     uint_t port, uint_t sleepflag)
1462 {
1463         tavor_mbox_info_t       mbox_info;
1464         tavor_cmd_post_t        cmd;
1465         uint64_t                data;
1466         uint_t                  size;
1467         int                     status, i;
1468 
1469         TAVOR_TNF_ENTER(tavor_init_ib_cmd_post);
1470 
1471         /* Make sure we are called with the correct flag */
1472         ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1473 
1474         /* Get an "In" mailbox for the command */
1475         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
1476         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1477         if (status != TAVOR_CMD_SUCCESS) {
1478                 TNF_PROBE_0(tavor_init_ib_mbox_fail, TAVOR_TNF_ERROR, "");
1479                 TAVOR_TNF_EXIT(tavor_init_ib_cmd_post);
1480                 return (status);
1481         }
1482 
1483         /* Copy the Tavor "INIT_IB" command into the mailbox */
1484         size = sizeof (tavor_hw_initib_t);
1485         for (i = 0; i < (size >> 3); i++) {
1486                 data = ((uint64_t *)initib)[i];
1487                 ddi_put64(mbox_info.mbi_in->mb_acchdl,
1488                     ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
1489         }
1490 
1491         /* Sync the mailbox for the device to read */
1492         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1493 
1494         /* Setup and post the Tavor "INIT_IB" command */
1495         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1496         cmd.cp_outparm  = 0;
1497         cmd.cp_inmod    = port;
1498         cmd.cp_opcode   = INIT_IB;
1499         cmd.cp_opmod    = 0;
1500         cmd.cp_flags    = sleepflag;
1501         status = tavor_cmd_post(state, &cmd);
1502         if (status != TAVOR_CMD_SUCCESS) {
1503                 TNF_PROBE_0(tavor_init_ib_cmd_post_fail,
1504                     TAVOR_TNF_ERROR, "");
1505         }
1506 
1507         /* Free the mailbox */
1508         tavor_mbox_free(state, &mbox_info);
1509 
1510         TAVOR_TNF_EXIT(tavor_init_ib_cmd_post);
1511         return (status);
1512 }
1513 
1514 
1515 /*
1516  * tavor_close_ib_cmd_post()
1517  *    Context: Can be called from interrupt or base context.
1518  *    (Currently called only from attach() and/or detach() path contexts)
1519  */
1520 int
1521 tavor_close_ib_cmd_post(tavor_state_t *state, uint_t port, uint_t sleepflag)
1522 {
1523         tavor_cmd_post_t        cmd;
1524         int                     status;
1525 
1526         TAVOR_TNF_ENTER(tavor_close_ib_cmd_post);
1527 
1528         /* Setup and post the Tavor "CLOSE_IB" command */
1529         cmd.cp_inparm   = 0;
1530         cmd.cp_outparm  = 0;
1531         cmd.cp_inmod    = port;
1532         cmd.cp_opcode   = CLOSE_IB;
1533         cmd.cp_opmod    = 0;
1534         cmd.cp_flags    = sleepflag;
1535         status = tavor_cmd_post(state, &cmd);
1536         if (status != TAVOR_CMD_SUCCESS) {
1537                 TNF_PROBE_0(tavor_close_ib_cmd_post_fail, TAVOR_TNF_ERROR, "");
1538         }
1539 
1540         TAVOR_TNF_EXIT(tavor_close_ib_cmd_post);
1541         return (status);
1542 }
1543 
1544 
1545 /*
1546  * tavor_set_ib_cmd_post()
1547  *    Context: Can be called from interrupt or base context.
1548  */
1549 int
1550 tavor_set_ib_cmd_post(tavor_state_t *state, uint32_t capmask, uint_t port,
1551     uint_t reset_qkey, uint_t sleepflag)
1552 {
1553         tavor_mbox_info_t       mbox_info;
1554         tavor_cmd_post_t        cmd;
1555         int                     status;
1556 
1557         TAVOR_TNF_ENTER(tavor_set_ib_cmd_post);
1558 
1559         /* Get an "In" mailbox for the command */
1560         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
1561         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1562         if (status != TAVOR_CMD_SUCCESS) {
1563                 TNF_PROBE_0(tavor_set_ib_mbox_fail, TAVOR_TNF_ERROR, "");
1564                 TAVOR_TNF_EXIT(tavor_set_ib_cmd_post);
1565                 return (status);
1566         }
1567 
1568         /* Copy the Tavor "SET_IB" command into mailbox */
1569         ddi_put32(mbox_info.mbi_in->mb_acchdl,
1570             ((uint32_t *)mbox_info.mbi_in->mb_addr + 0), reset_qkey);
1571         ddi_put32(mbox_info.mbi_in->mb_acchdl,
1572             ((uint32_t *)mbox_info.mbi_in->mb_addr + 1), capmask);
1573 
1574         /* Sync the mailbox for the device to read */
1575         tavor_mbox_sync(mbox_info.mbi_in, 0, TAVOR_CMD_SETIB_SZ,
1576             DDI_DMA_SYNC_FORDEV);
1577 
1578         /* Setup and post the Tavor "SET_IB" command */
1579         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1580         cmd.cp_outparm  = 0;
1581         cmd.cp_inmod    = port;
1582         cmd.cp_opcode   = SET_IB;
1583         cmd.cp_opmod    = 0;
1584         cmd.cp_flags    = sleepflag;
1585         status = tavor_cmd_post(state, &cmd);
1586         if (status != TAVOR_CMD_SUCCESS) {
1587                 TNF_PROBE_0(tavor_set_ib_cmd_post_fail, TAVOR_TNF_ERROR, "");
1588         }
1589 
1590         /* Free the mailbox */
1591         tavor_mbox_free(state, &mbox_info);
1592 
1593         TAVOR_TNF_EXIT(tavor_set_ib_cmd_post);
1594         return (status);
1595 }
1596 
1597 
1598 /*
1599  * tavor_mod_stat_cfg_cmd_post()
1600  *    Context: Can be called only from attach() path
1601  */
1602 int
1603 tavor_mod_stat_cfg_cmd_post(tavor_state_t *state)
1604 {
1605         tavor_mbox_info_t       mbox_info;
1606         tavor_cmd_post_t        cmd;
1607         tavor_hw_mod_stat_cfg_t *mod;
1608         uint64_t                data;
1609         uint_t                  size;
1610         int                     status, i;
1611 
1612         TAVOR_TNF_ENTER(tavor_mod_stat_cfg_cmd_post);
1613 
1614         /*
1615          * "MOD_STAT_CFG" needs an INMBOX parameter, to specify what operations
1616          * to do.  However, at the point in time that we call this command, the
1617          * DDR has not yet been initialized, and all INMBOX'es are located in
1618          * DDR.  Because we want to call MOD_STAT_CFG before QUERY_DEVLIM is
1619          * called, and thus call it before DDR is setup, we simply use an
1620          * OUTMBOX memory location here as our INMBOX parameter.
1621          */
1622         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
1623         status = tavor_mbox_alloc(state, &mbox_info, TAVOR_NOSLEEP);
1624         if (status != TAVOR_CMD_SUCCESS) {
1625                 TNF_PROBE_0(tavor_mod_stat_cfg_mbox_fail, TAVOR_TNF_ERROR, "");
1626                 TAVOR_TNF_EXIT(tavor_mod_stat_cfg_cmd_post);
1627                 return (status);
1628         }
1629 
1630         /*
1631          * Allocate on the heap our 'mod_stat_cfg' structure.  We want to
1632          * ideally move all of this on to the stack in the future, but this
1633          * works well for now.
1634          */
1635         mod = (tavor_hw_mod_stat_cfg_t *)kmem_zalloc(
1636             sizeof (tavor_hw_mod_stat_cfg_t), KM_SLEEP);
1637 
1638         /* Setup "MOD_STAT_CFG" settings */
1639         mod->srq_m   = 1;
1640         mod->srq     = state->ts_cfg_profile->cp_srq_enable;
1641 
1642         if (mod->srq) {
1643                 mod->log_max_srq = state->ts_cfg_profile->cp_log_num_srq;
1644         } else {
1645                 mod->log_max_srq = 0;
1646         }
1647 
1648         /* Copy the "MOD_STAT_CFG" command into the "In" mailbox */
1649         size = sizeof (tavor_hw_mod_stat_cfg_t);
1650         for (i = 0; i < (size >> 3); i++) {
1651                 data = ((uint64_t *)mod)[i];
1652                 ddi_put64(mbox_info.mbi_out->mb_acchdl,
1653                     ((uint64_t *)mbox_info.mbi_out->mb_addr + i), data);
1654         }
1655 
1656         /* Sync the mailbox for the device to read */
1657         tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORDEV);
1658 
1659         /* Setup and post the Tavor "MOD_STAT_CFG" command */
1660         cmd.cp_inparm   = mbox_info.mbi_out->mb_mapaddr;
1661         cmd.cp_outparm  = 0;
1662         cmd.cp_inmod    = 0;
1663         cmd.cp_opcode   = MOD_STAT_CFG;
1664         cmd.cp_opmod    = 0;
1665         cmd.cp_flags    = TAVOR_CMD_NOSLEEP_SPIN;
1666         status = tavor_cmd_post(state, &cmd);
1667         if (status != TAVOR_CMD_SUCCESS) {
1668                 TNF_PROBE_0(tavor_mod_stat_cfg_cmd_post_fail, TAVOR_TNF_ERROR,
1669                     "");
1670         }
1671 
1672         /* Free "MOD_STAT_CFG" struct */
1673         kmem_free(mod, sizeof (tavor_hw_mod_stat_cfg_t));
1674 
1675         /* Free the mailbox */
1676         tavor_mbox_free(state, &mbox_info);
1677 
1678         TAVOR_TNF_EXIT(tavor_mod_stat_cfg_cmd_post);
1679         return (status);
1680 }
1681 
1682 
1683 /*
1684  * tavor_mad_ifc_cmd_post()
1685  *    Context: Can be called from interrupt or base context.
1686  */
1687 int
1688 tavor_mad_ifc_cmd_post(tavor_state_t *state, uint_t port,
1689     uint_t sleepflag, uint32_t *mad, uint32_t *resp)
1690 {
1691         tavor_mbox_info_t       mbox_info;
1692         tavor_cmd_post_t        cmd;
1693         uint_t                  size;
1694         int                     status;
1695 
1696         TAVOR_TNF_ENTER(tavor_mad_ifc_cmd_post);
1697 
1698         /* Get "In" and "Out" mailboxes for the command */
1699         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1700         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1701         if (status != TAVOR_CMD_SUCCESS) {
1702                 TNF_PROBE_0(tavor_mad_ifc_mbox_fail, TAVOR_TNF_ERROR, "");
1703                 TAVOR_TNF_EXIT(tavor_mad_ifc_cmd_post);
1704                 return (status);
1705         }
1706 
1707         /* Copy the request MAD into the "In" mailbox */
1708         size = TAVOR_CMD_MAD_IFC_SIZE;
1709         bcopy(mad, mbox_info.mbi_in->mb_addr, size);
1710 
1711         /* Sync the mailbox for the device to read */
1712         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1713 
1714         /* Setup the Tavor "MAD_IFC" command */
1715         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1716         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
1717         cmd.cp_inmod    = port;
1718         cmd.cp_opcode   = MAD_IFC;
1719         cmd.cp_opmod    = TAVOR_CMD_MKEY_CHECK;  /* Enable MKey checking */
1720         cmd.cp_flags    = sleepflag;
1721         status = tavor_cmd_post(state, &cmd);
1722         if (status != TAVOR_CMD_SUCCESS) {
1723                 TNF_PROBE_0(tavor_mad_ifc_cmd_post_fail,
1724                     TAVOR_TNF_ERROR, "");
1725                 goto mad_ifc_fail;
1726         }
1727 
1728         /* Sync the mailbox to read the results */
1729         tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
1730 
1731         /* Copy the response MAD into "resp" */
1732         bcopy(mbox_info.mbi_out->mb_addr, resp, size);
1733 
1734 mad_ifc_fail:
1735         /* Free the mailbox */
1736         tavor_mbox_free(state, &mbox_info);
1737 
1738         TAVOR_TNF_EXIT(tavor_mad_ifc_cmd_post);
1739         return (status);
1740 }
1741 
1742 
1743 /*
1744  * tavor_getportinfo_cmd_post()
1745  *    Context: Can be called from interrupt or base context.
1746  */
1747 int
1748 tavor_getportinfo_cmd_post(tavor_state_t *state, uint_t port,
1749     uint_t sleepflag, sm_portinfo_t *portinfo)
1750 {
1751         tavor_mbox_info_t       mbox_info;
1752         tavor_cmd_post_t        cmd;
1753         uint32_t                *mbox;
1754         uint_t                  size;
1755         int                     status, i;
1756 
1757         TAVOR_TNF_ENTER(tavor_getportinfo_cmd_post);
1758 
1759         /* Get "In" and "Out" mailboxes for the command */
1760         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1761         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1762         if (status != TAVOR_CMD_SUCCESS) {
1763                 TNF_PROBE_0(tavor_getportinfo_mbox_fail,
1764                     TAVOR_TNF_ERROR, "");
1765                 TAVOR_TNF_EXIT(tavor_getportinfo_cmd_post);
1766                 return (status);
1767         }
1768 
1769         /* Build the GetPortInfo request MAD in the "In" mailbox */
1770         size = TAVOR_CMD_MAD_IFC_SIZE;
1771         mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
1772         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
1773         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
1774         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
1775         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
1776         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_PORTINFO);
1777         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], port);
1778         for (i = 6; i < (size >> 2); i++) {
1779                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
1780         }
1781 
1782         /* Sync the mailbox for the device to read */
1783         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1784 
1785         /* Setup the Tavor "MAD_IFC" command */
1786         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1787         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
1788         cmd.cp_inmod    = port;
1789         cmd.cp_opcode   = MAD_IFC;
1790         cmd.cp_opmod    = TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
1791         cmd.cp_flags    = sleepflag;
1792         status = tavor_cmd_post(state, &cmd);
1793         if (status != TAVOR_CMD_SUCCESS) {
1794                 TNF_PROBE_0(tavor_getportinfo_cmd_post_fail,
1795                     TAVOR_TNF_ERROR, "");
1796                 goto getportinfo_fail;
1797         }
1798 
1799         /* Sync the mailbox to read the results */
1800         size = sizeof (sm_portinfo_t);
1801         tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1802             size, DDI_DMA_SYNC_FORCPU);
1803 
1804         /*
1805          * Copy GetPortInfo response MAD into "portinfo".  Do any endian
1806          * swapping that may be necessary to flip any of the "portinfo"
1807          * fields
1808          */
1809         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*portinfo))
1810         bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1811             TAVOR_CMD_MADDATA_OFFSET), portinfo, size);
1812         TAVOR_GETPORTINFO_SWAP(portinfo);
1813         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*portinfo))
1814 
1815 getportinfo_fail:
1816         /* Free the mailbox */
1817         tavor_mbox_free(state, &mbox_info);
1818 
1819         TAVOR_TNF_EXIT(tavor_getportinfo_cmd_post);
1820         return (status);
1821 }
1822 
1823 
1824 /*
1825  * tavor_getnodeinfo_cmd_post()
1826  *    Context: Can be called from interrupt or base context.
1827  *    (Currently called only from attach() and detach() path contexts)
1828  */
1829 int
1830 tavor_getnodeinfo_cmd_post(tavor_state_t *state, uint_t sleepflag,
1831     sm_nodeinfo_t *nodeinfo)
1832 {
1833         tavor_mbox_info_t       mbox_info;
1834         tavor_cmd_post_t        cmd;
1835         uint32_t                *mbox;
1836         uint_t                  size;
1837         int                     status, i;
1838 
1839         TAVOR_TNF_ENTER(tavor_getnodeinfo_cmd_post);
1840 
1841         /* Make sure we are called with the correct flag */
1842         ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1843 
1844         /* Get "In" and "Out" mailboxes for the command */
1845         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1846         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1847         if (status != TAVOR_CMD_SUCCESS) {
1848                 TNF_PROBE_0(tavor_getnodeinfo_mbox_fail,
1849                     TAVOR_TNF_ERROR, "");
1850                 TAVOR_TNF_EXIT(tavor_getnodeinfo_cmd_post);
1851                 return (status);
1852         }
1853 
1854         /* Build the GetNodeInfo request MAD into the "In" mailbox */
1855         size = TAVOR_CMD_MAD_IFC_SIZE;
1856         mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
1857         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
1858         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
1859         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
1860         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
1861         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_NODEINFO);
1862         for (i = 5; i < (size >> 2); i++) {
1863                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
1864         }
1865 
1866         /* Sync the mailbox for the device to read */
1867         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1868 
1869         /* Setup the Tavor "MAD_IFC" command */
1870         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1871         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
1872         cmd.cp_inmod    = 1;  /* Get NodeInfo from port #1 */
1873         cmd.cp_opcode   = MAD_IFC;
1874         cmd.cp_opmod    = TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
1875         cmd.cp_flags    = sleepflag;
1876         status = tavor_cmd_post(state, &cmd);
1877         if (status != TAVOR_CMD_SUCCESS) {
1878                 TNF_PROBE_0(tavor_getnodeinfo_cmd_post_fail,
1879                     TAVOR_TNF_ERROR, "");
1880                 goto getnodeinfo_fail;
1881         }
1882 
1883         /* Sync the mailbox to read the results */
1884         size = sizeof (sm_nodeinfo_t);
1885         tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1886             size, DDI_DMA_SYNC_FORCPU);
1887 
1888         /*
1889          * Copy GetNodeInfo response MAD into "nodeinfo".  Do any endian
1890          * swapping that may be necessary to flip any of the "nodeinfo"
1891          * fields
1892          */
1893         bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1894             TAVOR_CMD_MADDATA_OFFSET), nodeinfo, size);
1895         TAVOR_GETNODEINFO_SWAP(nodeinfo);
1896 
1897 getnodeinfo_fail:
1898         /* Free the mailbox */
1899         tavor_mbox_free(state, &mbox_info);
1900 
1901         TAVOR_TNF_EXIT(tavor_getnodeinfo_cmd_post);
1902         return (status);
1903 }
1904 
1905 
1906 /*
1907  * tavor_getnodedesc_cmd_post()
1908  *    Context: Can be called from interrupt or base context.
1909  *    (Currently called only from attach() and detach() path contexts)
1910  */
1911 int
1912 tavor_getnodedesc_cmd_post(tavor_state_t *state, uint_t sleepflag,
1913     sm_nodedesc_t *nodedesc)
1914 {
1915         tavor_mbox_info_t       mbox_info;
1916         tavor_cmd_post_t        cmd;
1917         uint32_t                *mbox;
1918         uint_t                  size;
1919         int                     status, i;
1920 
1921         TAVOR_TNF_ENTER(tavor_getnodedesc_cmd_post);
1922 
1923         /* Get "In" and "Out" mailboxes for the command */
1924         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1925         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1926         if (status != TAVOR_CMD_SUCCESS) {
1927                 TNF_PROBE_0(tavor_getnodedesc_mbox_fail, TAVOR_TNF_ERROR, "");
1928                 TAVOR_TNF_EXIT(tavor_getnodedesc_cmd_post);
1929                 return (status);
1930         }
1931 
1932         /* Build the GetNodeDesc request MAD into the "In" mailbox */
1933         size = TAVOR_CMD_MAD_IFC_SIZE;
1934         mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
1935         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
1936         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
1937         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
1938         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
1939         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_NODEDESC);
1940         for (i = 5; i < (size >> 2); i++) {
1941                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
1942         }
1943 
1944         /* Sync the mailbox for the device to read */
1945         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1946 
1947         /* Setup the Tavor "MAD_IFC" command */
1948         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
1949         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
1950         cmd.cp_inmod    = 1;  /* Get NodeDesc from port #1 */
1951         cmd.cp_opcode   = MAD_IFC;
1952         cmd.cp_opmod    = TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
1953         cmd.cp_flags    = sleepflag;
1954         status = tavor_cmd_post(state, &cmd);
1955         if (status != TAVOR_CMD_SUCCESS) {
1956                 TNF_PROBE_0(tavor_getnodedesc_cmd_post_fail,
1957                     TAVOR_TNF_ERROR, "");
1958                 goto getnodedesc_fail;
1959         }
1960 
1961         /* Sync the mailbox to read the results */
1962         size = sizeof (sm_nodedesc_t);
1963         tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1964             size, DDI_DMA_SYNC_FORCPU);
1965 
1966         /* Copy GetNodeDesc response MAD into "nodedesc" */
1967         bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1968             TAVOR_CMD_MADDATA_OFFSET), nodedesc, size);
1969 
1970 getnodedesc_fail:
1971         /* Free the mailbox */
1972         tavor_mbox_free(state, &mbox_info);
1973 
1974         TAVOR_TNF_EXIT(tavor_getnodedesc_cmd_post);
1975         return (status);
1976 }
1977 
1978 
1979 /*
1980  * tavor_getguidinfo_cmd_post()
1981  *    Context: Can be called from interrupt or base context.
1982  */
1983 int
1984 tavor_getguidinfo_cmd_post(tavor_state_t *state, uint_t port,
1985     uint_t guidblock, uint_t sleepflag, sm_guidinfo_t *guidinfo)
1986 {
1987         tavor_mbox_info_t       mbox_info;
1988         tavor_cmd_post_t        cmd;
1989         uint32_t                *mbox;
1990         uint_t                  size;
1991         int                     status, i;
1992 
1993         TAVOR_TNF_ENTER(tavor_getguidinfo_cmd_post);
1994 
1995         /* Get "In" and "Out" mailboxes for the command */
1996         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1997         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1998         if (status != TAVOR_CMD_SUCCESS) {
1999                 TNF_PROBE_0(tavor_getguidinfo_mbox_fail, TAVOR_TNF_ERROR, "");
2000                 TAVOR_TNF_EXIT(tavor_getguidinfo_cmd_post);
2001                 return (status);
2002         }
2003 
2004         /* Build the GetGUIDInfo request MAD into the "In" mailbox */
2005         size = TAVOR_CMD_MAD_IFC_SIZE;
2006         mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
2007         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
2008         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
2009         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
2010         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
2011         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_GUIDINFO);
2012         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], guidblock);
2013         for (i = 6; i < (size >> 2); i++) {
2014                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
2015         }
2016 
2017         /* Sync the mailbox for the device to read */
2018         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2019 
2020         /* Setup the Tavor "MAD_IFC" command */
2021         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
2022         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
2023         cmd.cp_inmod    = port;
2024         cmd.cp_opcode   = MAD_IFC;
2025         cmd.cp_opmod    = TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
2026         cmd.cp_flags    = sleepflag;
2027         status = tavor_cmd_post(state, &cmd);
2028         if (status != TAVOR_CMD_SUCCESS) {
2029                 TNF_PROBE_0(tavor_getguidinfo_cmd_post_fail,
2030                     TAVOR_TNF_ERROR, "");
2031                 goto getguidinfo_fail;
2032         }
2033 
2034         /* Sync the mailbox to read the results */
2035         size = sizeof (sm_guidinfo_t);
2036         tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2037             size, DDI_DMA_SYNC_FORCPU);
2038 
2039         /*
2040          * Copy GetGUIDInfo response MAD into "guidinfo".  Do any endian
2041          * swapping that may be necessary to flip the "guidinfo" fields
2042          */
2043         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*guidinfo))
2044         bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2045             TAVOR_CMD_MADDATA_OFFSET), guidinfo, size);
2046         TAVOR_GETGUIDINFO_SWAP(guidinfo);
2047         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*guidinfo))
2048 
2049 getguidinfo_fail:
2050         /* Free the mailbox */
2051         tavor_mbox_free(state, &mbox_info);
2052 
2053         TAVOR_TNF_EXIT(tavor_getguidinfo_cmd_post);
2054         return (status);
2055 }
2056 
2057 
2058 /*
2059  * tavor_getpkeytable_cmd_post()
2060  *    Context: Can be called from interrupt or base context.
2061  */
2062 int
2063 tavor_getpkeytable_cmd_post(tavor_state_t *state, uint_t port,
2064     uint_t pkeyblock, uint_t sleepflag, sm_pkey_table_t *pkeytable)
2065 {
2066         tavor_mbox_info_t       mbox_info;
2067         tavor_cmd_post_t        cmd;
2068         uint32_t                *mbox;
2069         uint_t                  size;
2070         int                     status, i;
2071 
2072         TAVOR_TNF_ENTER(tavor_getpkeytable_cmd_post);
2073 
2074         /* Get "In" and "Out" mailboxes for the command */
2075         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
2076         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2077         if (status != TAVOR_CMD_SUCCESS) {
2078                 TNF_PROBE_0(tavor_getpkeytable_mbox_fail, TAVOR_TNF_ERROR, "");
2079                 TAVOR_TNF_EXIT(tavor_getpkeytable_cmd_post);
2080                 return (status);
2081         }
2082 
2083         /* Build the GetPkeyTable request MAD into the "In" mailbox */
2084         size = TAVOR_CMD_MAD_IFC_SIZE;
2085         mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
2086         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
2087         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
2088         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
2089         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
2090         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_PKEYTBLE);
2091         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], pkeyblock);
2092         for (i = 6; i < (size >> 2); i++) {
2093                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
2094         }
2095 
2096         /* Sync the mailbox for the device to read */
2097         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2098 
2099         /* Setup the Tavor "MAD_IFC" command */
2100         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
2101         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
2102         cmd.cp_inmod    = port;
2103         cmd.cp_opcode   = MAD_IFC;
2104         cmd.cp_opmod    = TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
2105         cmd.cp_flags    = sleepflag;
2106         status = tavor_cmd_post(state, &cmd);
2107         if (status != TAVOR_CMD_SUCCESS) {
2108                 TNF_PROBE_0(tavor_getpkeytable_cmd_post_fail,
2109                     TAVOR_TNF_ERROR, "");
2110                 goto getpkeytable_fail;
2111         }
2112 
2113         /* Sync the mailbox to read the results */
2114         size = sizeof (sm_pkey_table_t);
2115         tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2116             size, DDI_DMA_SYNC_FORCPU);
2117 
2118         /*
2119          * Copy GetPKeyTable response MAD into "pkeytable".  Do any endian
2120          * swapping that may be necessary to flip the "pkeytable" fields
2121          */
2122         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pkeytable))
2123         bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2124             TAVOR_CMD_MADDATA_OFFSET), pkeytable, size);
2125         TAVOR_GETPKEYTABLE_SWAP(pkeytable);
2126         _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*pkeytable))
2127 
2128 getpkeytable_fail:
2129         /* Free the mailbox */
2130         tavor_mbox_free(state, &mbox_info);
2131 
2132         TAVOR_TNF_EXIT(tavor_getpkeytable_cmd_post);
2133         return (status);
2134 }
2135 
2136 
2137 /*
2138  * tavor_write_mtt_cmd_post()
2139  *    Context: Can be called from interrupt or base context.
2140  */
2141 int
2142 tavor_write_mtt_cmd_post(tavor_state_t *state, tavor_mbox_info_t *mbox_info,
2143     uint_t num_mtt, uint_t sleepflag)
2144 {
2145         tavor_cmd_post_t        cmd;
2146         uint_t                  size;
2147         int                     status;
2148 
2149         TAVOR_TNF_ENTER(tavor_write_mtt_cmd_post);
2150 
2151         /*
2152          * The WRITE_MTT command is unlike the other commands we use, in that
2153          * we have intentionally separated the mailbox allocation step from
2154          * the rest of the command posting steps.  At this point (when this
2155          * function is called) the "In" mailbox already contains all the MTT
2156          * entries to be copied into the Tavor tables (starting at offset
2157          * 0x10) _and_ the 64-bit address of the destination for the first
2158          * MTT entry in the MTT table.
2159          */
2160 
2161         /* Sync the mailbox for the device to read */
2162         size = (num_mtt << TAVOR_MTT_SIZE_SHIFT) + TAVOR_CMD_WRITEMTT_RSVD_SZ;
2163         tavor_mbox_sync(mbox_info->mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2164 
2165         /* Setup and post Tavor "WRITE_MTT" command */
2166         cmd.cp_inparm   = mbox_info->mbi_in->mb_mapaddr;
2167         cmd.cp_outparm  = 0;
2168         cmd.cp_inmod    = num_mtt;
2169         cmd.cp_opcode   = WRITE_MTT;
2170         cmd.cp_opmod    = 0;
2171         cmd.cp_flags    = sleepflag;
2172         status = tavor_cmd_post(state, &cmd);
2173         if (status != TAVOR_CMD_SUCCESS) {
2174                 TNF_PROBE_0(tavor_write_mtt_cmd_fail, TAVOR_TNF_ERROR, "");
2175         }
2176 
2177         TAVOR_TNF_EXIT(tavor_write_mtt_cmd_post);
2178         return (status);
2179 }
2180 
2181 
2182 /*
2183  * tavor_sync_tpt_cmd_post()
2184  *    Context: Can be called from interrupt or base context.
2185  */
2186 int
2187 tavor_sync_tpt_cmd_post(tavor_state_t *state, uint_t sleepflag)
2188 {
2189         tavor_cmd_post_t        cmd;
2190         int                     status;
2191 
2192         TAVOR_TNF_ENTER(tavor_sync_tpt_cmd_post);
2193 
2194         /* Setup and post the Tavor "SYNC_TPT" command */
2195         cmd.cp_inparm   = 0;
2196         cmd.cp_outparm  = 0;
2197         cmd.cp_inmod    = 0;
2198         cmd.cp_opcode   = SYNC_TPT;
2199         cmd.cp_opmod    = 0;
2200         cmd.cp_flags    = sleepflag;
2201         status = tavor_cmd_post(state, &cmd);
2202         if (status != TAVOR_CMD_SUCCESS) {
2203                 TNF_PROBE_0(tavor_sync_tpt_cmd_post_fail, TAVOR_TNF_ERROR, "");
2204         }
2205 
2206         TAVOR_TNF_EXIT(tavor_sync_tpt_cmd_post);
2207         return (status);
2208 }
2209 
2210 /*
2211  * tavor_map_eq_cmd_post()
2212  *    Context: Can be called from interrupt or base context.
2213  *    (Currently called only from attach() and/or detach() path contexts)
2214  */
2215 int
2216 tavor_map_eq_cmd_post(tavor_state_t *state, uint_t map, uint_t eqcindx,
2217     uint64_t eqmapmask, uint_t sleepflag)
2218 {
2219         tavor_cmd_post_t        cmd;
2220         int                     status;
2221 
2222         TAVOR_TNF_ENTER(tavor_map_eq_cmd_post);
2223 
2224         /* Setup and post Tavor "MAP_EQ" command */
2225         cmd.cp_inparm   = eqmapmask;
2226         cmd.cp_outparm  = 0;
2227         cmd.cp_inmod    = eqcindx;
2228         if (map != TAVOR_CMD_MAP_EQ_EVT_MAP) {
2229                 cmd.cp_inmod |= TAVOR_CMD_UNMAP_EQ_MASK;
2230         }
2231         cmd.cp_opcode   = MAP_EQ;
2232         cmd.cp_opmod    = 0;
2233         cmd.cp_flags    = sleepflag;
2234         status = tavor_cmd_post(state, &cmd);
2235         if (status != TAVOR_CMD_SUCCESS) {
2236                 TNF_PROBE_0(tavor_map_eq_cmd_post_fail, TAVOR_TNF_ERROR, "");
2237         }
2238 
2239         TAVOR_TNF_EXIT(tavor_map_eq_cmd_post);
2240         return (status);
2241 }
2242 
2243 
2244 /*
2245  * tavor_resize_cq_cmd_post()
2246  *    Context: Can be called from interrupt or base context.
2247  */
2248 int
2249 tavor_resize_cq_cmd_post(tavor_state_t *state, tavor_hw_cqc_t *cqc,
2250     uint_t cqcindx, uint32_t *prod_indx, uint_t sleepflag)
2251 {
2252         tavor_mbox_info_t       mbox_info;
2253         tavor_cmd_post_t        cmd;
2254         uint64_t                data;
2255         uint_t                  size;
2256         int                     status, i;
2257 
2258         TAVOR_TNF_ENTER(tavor_resize_cq_cmd_post);
2259 
2260         /* Get an "In" mailbox for the command */
2261         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2262         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2263         if (status != TAVOR_CMD_SUCCESS) {
2264                 TNF_PROBE_0(tavor_resize_cq_mbox_fail, TAVOR_TNF_ERROR, "");
2265                 TAVOR_TNF_EXIT(tavor_resize_cq_cmd_post);
2266                 return (status);
2267         }
2268 
2269         /* Copy the Tavor "RESIZE_CQ" command into mailbox */
2270         size = sizeof (tavor_hw_cqc_t);
2271         for (i = 0; i < (size >> 3); i++) {
2272                 data = ((uint64_t *)cqc)[i];
2273                 ddi_put64(mbox_info.mbi_in->mb_acchdl,
2274                     ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
2275         }
2276 
2277         /* Sync the mailbox for the device to read */
2278         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2279 
2280         /* Setup and post Tavor "RESIZE_CQ" command */
2281         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
2282         cmd.cp_outparm  = 0;
2283         cmd.cp_inmod    = cqcindx;
2284         cmd.cp_opcode   = RESIZE_CQ;
2285         cmd.cp_opmod    = 0;
2286         cmd.cp_flags    = sleepflag;
2287         status = tavor_cmd_post(state, &cmd);
2288         if (status != TAVOR_CMD_SUCCESS) {
2289                 TNF_PROBE_0(tavor_resize_cq_cmd_post_fail, TAVOR_TNF_ERROR, "");
2290         }
2291 
2292         /*
2293          * New "producer index" is returned in the upper 32 bits of
2294          * command "outparam"
2295          */
2296         *prod_indx = (cmd.cp_outparm >> 32);
2297 
2298         /* Free the mailbox */
2299         tavor_mbox_free(state, &mbox_info);
2300 
2301         TAVOR_TNF_EXIT(tavor_resize_cq_cmd_post);
2302         return (status);
2303 }
2304 
2305 
2306 /*
2307  * tavor_cmn_qp_cmd_post()
2308  *    Context: Can be called from interrupt or base context.
2309  *
2310  *    This is the common function for posting all the various types of
2311  *    QP state transition related Tavor commands.  Since some of the
2312  *    commands differ from the others in the number (and type) of arguments
2313  *    that each require, this routine does checks based on opcode type
2314  *    (explained in more detail below).
2315  *
2316  * Note: This common function should be used only with the following
2317  *    opcodes: RTS2SQD_QP, TOERR_QP, TORST_QP, RST2INIT_QP, INIT2INIT_QP,
2318  *    INIT2RTR_QP, RTR2RTS_QP, RTS2RTS_QP, SQD2RTS_QP, and SQERR2RTS_QP.
2319  */
2320 int
2321 tavor_cmn_qp_cmd_post(tavor_state_t *state, uint_t opcode,
2322     tavor_hw_qpc_t *qp, uint_t qpindx, uint32_t opmask,
2323     uint_t sleepflag)
2324 {
2325         tavor_mbox_info_t       mbox_info;
2326         tavor_cmd_post_t        cmd;
2327         uint64_t                data, in_mapaddr, out_mapaddr;
2328         uint_t                  size, flags, opmod;
2329         int                     status, i;
2330 
2331         TAVOR_TNF_ENTER(tavor_cmn_qp_cmd_post);
2332 
2333         /*
2334          * Use the specified opcode type to set the appropriate parameters.
2335          * Specifically, we need to set in_mapaddr, out_mapaddr, flags, and
2336          * opmod (as necessary).  Setting these parameters may also require
2337          * us to allocate an "In" or "Out" mailbox depending on the command
2338          * type.
2339          */
2340         if (opcode == RTS2SQD_QP) {
2341                 /*
2342                  * Note: For RTS-to-SendQueueDrain state transitions we
2343                  * always want to request the event generation from the
2344                  * hardware.  Though we may not notify the consumer of the
2345                  * drained event, the decision to forward (or not) is made
2346                  * later in the SQD event handler.
2347                  */
2348                 flags = TAVOR_CMD_REQ_SQD_EVENT;
2349 
2350                 /*
2351                  * The RTS2SQD_QP command uses no "In" or "Out" mailboxes (and
2352                  * has no special opcode modifiers).
2353                  */
2354                 in_mapaddr  = 0;
2355                 out_mapaddr = 0;
2356                 opmod = 0;
2357 
2358         } else if (opcode == TOERR_QP) {
2359                 /*
2360                  * The TOERR_QP command uses no "In" or "Out" mailboxes, has no
2361                  * special opcode modifiers, and takes no special flags.
2362                  */
2363                 in_mapaddr  = 0;
2364                 out_mapaddr = 0;
2365                 opmod = 0;
2366                 flags = 0;
2367 
2368         } else if (opcode == TORST_QP) {
2369                 /*
2370                  * The TORST_QP command could take an "Out" mailbox, but we do
2371                  * not require it here.  It also does not takes any special
2372                  * flags.  It does however, take a TAVOR_CMD_DIRECT_TO_RESET
2373                  * opcode modifier, which indicates that the transition to
2374                  * reset should happen without first moving the QP through the
2375                  * Error state (and, hence, without generating any unnecessary
2376                  * "flushed-in-error" completions).
2377                  */
2378                 in_mapaddr  = 0;
2379                 out_mapaddr = 0;
2380                 opmod = TAVOR_CMD_DIRECT_TO_RESET | TAVOR_CMD_NO_OUTMBOX;
2381                 flags = 0;
2382 
2383         } else {
2384                 /*
2385                  * All the other QP state transition commands (RST2INIT_QP,
2386                  * INIT2INIT_QP, INIT2RTR_QP, RTR2RTS_QP, RTS2RTS_QP,
2387                  * SQD2RTS_QP, and SQERR2RTS_QP) require an "In" mailbox.
2388                  * None of these require any special flags or opcode modifiers.
2389                  */
2390                 mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2391                 status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2392                 if (status != TAVOR_CMD_SUCCESS) {
2393                         TNF_PROBE_0(tavor_cmn_qp_mbox_fail,
2394                             TAVOR_TNF_ERROR, "");
2395                         TAVOR_TNF_EXIT(tavor_cmn_qp_cmd_post);
2396                         return (status);
2397                 }
2398                 in_mapaddr  = mbox_info.mbi_in->mb_mapaddr;
2399                 out_mapaddr = 0;
2400                 flags = 0;
2401                 opmod = 0;
2402 
2403                 /* Copy the Tavor command into the "In" mailbox */
2404                 size = sizeof (tavor_hw_qpc_t);
2405                 for (i = 0; i < (size >> 3); i++) {
2406                         data = ((uint64_t *)qp)[i];
2407                         ddi_put64(mbox_info.mbi_in->mb_acchdl,
2408                             ((uint64_t *)mbox_info.mbi_in->mb_addr + i + 1),
2409                             data);
2410                 }
2411                 ddi_put32(mbox_info.mbi_in->mb_acchdl,
2412                     ((uint32_t *)mbox_info.mbi_in->mb_addr), opmask);
2413 
2414                 /*
2415                  * Sync the mailbox for the device to read.  We have to add
2416                  * eight bytes here to account for "opt_param_mask" and
2417                  * proper alignment.
2418                  */
2419                 tavor_mbox_sync(mbox_info.mbi_in, 0, size + 8,
2420                     DDI_DMA_SYNC_FORDEV);
2421         }
2422 
2423         /* Setup and post Tavor QP state transition command */
2424         cmd.cp_inparm   = in_mapaddr;
2425         cmd.cp_outparm  = out_mapaddr;
2426         cmd.cp_inmod    = qpindx | flags;
2427         cmd.cp_opcode   = opcode;
2428         cmd.cp_opmod    = opmod;
2429         cmd.cp_flags    = sleepflag;
2430         status = tavor_cmd_post(state, &cmd);
2431         if (status != TAVOR_CMD_SUCCESS) {
2432                 TNF_PROBE_0(tavor_cmn_qp_cmd_post_fail, TAVOR_TNF_ERROR, "");
2433         }
2434 
2435         /*
2436          * If we allocated a mailbox (either an "In" or an "Out") above,
2437          * then free it now before returning.
2438          */
2439         if ((opcode != RTS2SQD_QP) && (opcode != TOERR_QP) &&
2440             (opcode != TORST_QP)) {
2441                 /* Free the mailbox */
2442                 tavor_mbox_free(state, &mbox_info);
2443         }
2444 
2445         TAVOR_TNF_EXIT(tavor_cmn_qp_cmd_post);
2446         return (status);
2447 }
2448 
2449 
2450 /*
2451  * tavor_cmn_query_cmd_post()
2452  *    Context: Can be called from interrupt or base context.
2453  *
2454  *    This is the common function for posting all the various types of
2455  *    Tavor query commands.  All Tavor query commands require an "Out"
2456  *    mailbox to be allocated for the resulting queried data.
2457  *
2458  * Note: This common function should be used only with the following
2459  *    opcodes: QUERY_DEV_LIM, QUERY_FW, QUERY_DDR, QUERY_ADAPTER,
2460  *     QUERY_HCA, QUERY_MPT, QUERY_EQ, QUERY_CQ, and QUERY_QP.
2461  */
2462 int
2463 tavor_cmn_query_cmd_post(tavor_state_t *state, uint_t opcode,
2464     uint_t queryindx, void *query, uint_t size, uint_t sleepflag)
2465 {
2466         tavor_mbox_info_t       mbox_info;
2467         tavor_cmd_post_t        cmd;
2468         uint64_t                data;
2469         uint_t                  offset;
2470         int                     status, i;
2471 
2472         TAVOR_TNF_ENTER(tavor_cmn_query_cmd_post);
2473 
2474         /* Get an "Out" mailbox for the command */
2475         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
2476         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2477         if (status != TAVOR_CMD_SUCCESS) {
2478                 TNF_PROBE_0(tavor_cmn_query_mbox_fail, TAVOR_TNF_ERROR, "");
2479                 TAVOR_TNF_EXIT(tavor_cmn_query_cmd_post);
2480                 return (status);
2481         }
2482 
2483         /* Setup and post the Tavor query command */
2484         cmd.cp_inparm   = 0;
2485         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
2486         cmd.cp_inmod    = queryindx;
2487         cmd.cp_opcode   = opcode;
2488         cmd.cp_opmod    = 0;
2489         cmd.cp_flags    = sleepflag;
2490         status = tavor_cmd_post(state, &cmd);
2491         if (status != TAVOR_CMD_SUCCESS) {
2492                 TNF_PROBE_0(tavor_cmn_query_cmd_post_fail, TAVOR_TNF_ERROR, "");
2493                 goto cmn_query_fail;
2494         }
2495 
2496         /* Sync the mailbox to read the results */
2497         tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
2498 
2499         /*
2500          * QUERY_QP is handled somewhat differently than the other query
2501          * commands.  For QUERY_QP, the actual queried data is offset into
2502          * the mailbox (by one 64-bit word).
2503          */
2504         offset = (opcode == QUERY_QP) ? 1 : 0;
2505 
2506         /* Copy query command results into "query" */
2507         for (i = 0; i < (size >> 3); i++) {
2508                 data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
2509                     ((uint64_t *)mbox_info.mbi_out->mb_addr + i + offset));
2510                 ((uint64_t *)query)[i] = data;
2511         }
2512 
2513 cmn_query_fail:
2514         /* Free the mailbox */
2515         tavor_mbox_free(state, &mbox_info);
2516 
2517         TAVOR_TNF_EXIT(tavor_cmn_query_cmd_post);
2518         return (status);
2519 }
2520 
2521 
2522 /*
2523  * tavor_cmn_ownership_cmd_post()
2524  *    Context: Can be called from interrupt or base context.
2525  *
2526  *    This is the common function for posting all the various types of
2527  *    Tavor HW/SW resource ownership commands.  Since some of the commands
2528  *    differ from the others in the direction of ownership change (i.e.
2529  *    from HW ownership to SW, or vice versa), they differ in the type of
2530  *    mailbox and specific handling that each requires.  This routine does
2531  *    certain checks based on opcode type to determine the direction of
2532  *    the transition and to correctly handle the request.
2533  *
2534  * Note: This common function should be used only with the following
2535  *    opcodes: HW2SW_MPT, HW2SW_EQ, HW2SW_CQ, SW2HW_MPT, SW2HW_EQ, and
2536  *    SW2HW_CQ
2537  */
2538 int
2539 tavor_cmn_ownership_cmd_post(tavor_state_t *state, uint_t opcode,
2540     void *hwrsrc, uint_t size, uint_t hwrsrcindx, uint_t sleepflag)
2541 {
2542         tavor_mbox_info_t       mbox_info;
2543         tavor_cmd_post_t        cmd;
2544         uint64_t                data, in_mapaddr, out_mapaddr;
2545         uint_t                  direction, opmod;
2546         int                     status, i;
2547 
2548         TAVOR_TNF_ENTER(tavor_cmn_ownership_cmd_post);
2549 
2550         /*
2551          * Determine the direction of the ownership transfer based on the
2552          * provided opcode
2553          */
2554         if ((opcode == HW2SW_MPT) || (opcode == HW2SW_EQ) ||
2555             (opcode == HW2SW_CQ) || (opcode == HW2SW_SRQ)) {
2556                 direction = TAVOR_CMD_RSRC_HW2SW;
2557 
2558         } else if ((opcode == SW2HW_MPT) || (opcode == SW2HW_EQ) ||
2559             (opcode == SW2HW_CQ) || (opcode == SW2HW_SRQ)) {
2560                 direction = TAVOR_CMD_RSRC_SW2HW;
2561 
2562         } else {
2563                 TNF_PROBE_0(tavor_cmn_ownership_dir_fail,
2564                     TAVOR_TNF_ERROR, "");
2565                 TAVOR_TNF_EXIT(tavor_cmn_ownership_cmd_post);
2566                 return (TAVOR_CMD_INVALID_STATUS);
2567         }
2568 
2569         /*
2570          * If hwrsrc is NULL then we do not allocate a mailbox.  This is used
2571          * in the case of memory deregister where the out mailbox is not
2572          * needed.  In the case of re-register, we do use the hwrsrc.
2573          *
2574          * Otherwise, If ownership transfer is going from hardware to software,
2575          * then allocate an "Out" mailbox.  This will be filled in later as a
2576          * result of the Tavor command.
2577          *
2578          * And if the ownership transfer is going from software to hardware,
2579          * then we need an "In" mailbox, and we need to fill it in and sync it
2580          * (if necessary).  Then the mailbox can be passed to the Tavor
2581          * firmware.
2582          *
2583          * For the HW2SW (dereg) case, we only use an out mbox if hwrsrc is !=
2584          * NULL.  This implies a re-reg, and the out mbox must be used.  If
2585          * hwrsrc is == NULL, then we can save some time and resources by not
2586          * using an out mbox at all.  We must set opmod to TAVOR_CMD_DO_OUTMBOX
2587          * and TAVOR_CMD_NO_OUTMBOX appropriately in this case.
2588          *
2589          * For the SW2HW (reg) case, no out mbox is possible.  We set opmod to
2590          * 0 anyway, but this field is not used in this case.
2591          */
2592         if (direction == TAVOR_CMD_RSRC_HW2SW) {
2593                 if (hwrsrc != NULL) {
2594                         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
2595                         status = tavor_mbox_alloc(state, &mbox_info,
2596                             sleepflag);
2597                         if (status != TAVOR_CMD_SUCCESS) {
2598                                 TNF_PROBE_0(tavor_cmn_ownership_mbox_fail,
2599                                     TAVOR_TNF_ERROR, "");
2600                                 TAVOR_TNF_EXIT(tavor_cmn_ownership_cmd_post);
2601                                 return (status);
2602                         }
2603                         in_mapaddr  = 0;
2604                         out_mapaddr = mbox_info.mbi_out->mb_mapaddr;
2605                         opmod = TAVOR_CMD_DO_OUTMBOX;
2606                 } else {
2607                         in_mapaddr = 0;
2608                         out_mapaddr = 0;
2609                         opmod = TAVOR_CMD_NO_OUTMBOX;
2610                 }
2611         } else {  /* TAVOR_CMD_RSRC_SW2HW */
2612                 mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2613                 status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2614                 if (status != TAVOR_CMD_SUCCESS) {
2615                         TNF_PROBE_0(tavor_cmn_ownership_mbox_fail,
2616                             TAVOR_TNF_ERROR, "");
2617                         TAVOR_TNF_EXIT(tavor_sw2hw_mpt_cmd_post);
2618                         return (status);
2619                 }
2620 
2621                 /* Copy the SW2HW ownership command into mailbox */
2622                 for (i = 0; i < (size >> 3); i++) {
2623                         data = ((uint64_t *)hwrsrc)[i];
2624                         ddi_put64(mbox_info.mbi_in->mb_acchdl,
2625                             ((uint64_t *)mbox_info.mbi_in->mb_addr + i),
2626                             data);
2627                 }
2628 
2629                 /* Sync the mailbox for the device to read */
2630                 tavor_mbox_sync(mbox_info.mbi_in, 0, size,
2631                     DDI_DMA_SYNC_FORDEV);
2632 
2633                 in_mapaddr  = mbox_info.mbi_in->mb_mapaddr;
2634                 out_mapaddr = 0;
2635                 opmod = 0;
2636         }
2637 
2638 
2639         /* Setup and post the Tavor ownership command */
2640         cmd.cp_inparm   = in_mapaddr;
2641         cmd.cp_outparm  = out_mapaddr;
2642         cmd.cp_inmod    = hwrsrcindx;
2643         cmd.cp_opcode   = opcode;
2644         cmd.cp_opmod    = opmod;
2645         cmd.cp_flags    = sleepflag;
2646         status = tavor_cmd_post(state, &cmd);
2647         if (status != TAVOR_CMD_SUCCESS) {
2648                 TNF_PROBE_0(tavor_cmn_ownership_cmd_post_fail,
2649                     TAVOR_TNF_ERROR, "");
2650                 goto cmn_ownership_fail;
2651         }
2652 
2653         /*
2654          * As mentioned above, for HW2SW ownership transfers we need to
2655          * sync (if necessary) and copy out the resulting data from the
2656          * "Out" mailbox" (assuming the above command was successful).
2657          */
2658         if (direction == TAVOR_CMD_RSRC_HW2SW && hwrsrc != NULL) {
2659                 /* Sync the mailbox to read the results */
2660                 tavor_mbox_sync(mbox_info.mbi_out, 0, size,
2661                     DDI_DMA_SYNC_FORCPU);
2662 
2663                 /* Copy HW2SW ownership command results into "hwrsrc" */
2664                 for (i = 0; i < (size >> 3); i++) {
2665                         data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
2666                             ((uint64_t *)mbox_info.mbi_out->mb_addr + i));
2667                         ((uint64_t *)hwrsrc)[i] = data;
2668                 }
2669         }
2670 
2671 cmn_ownership_fail:
2672         if (hwrsrc != NULL) {
2673                 /* Free the mailbox */
2674                 tavor_mbox_free(state, &mbox_info);
2675         }
2676 
2677         TAVOR_TNF_EXIT(tavor_cmn_ownership_cmd_post);
2678         return (status);
2679 }
2680 
2681 
2682 /*
2683  * tavor_conf_special_qp_cmd_post()
2684  *    Context: Can be called from interrupt or base context.
2685  */
2686 int
2687 tavor_conf_special_qp_cmd_post(tavor_state_t *state, uint_t qpindx,
2688     uint_t qptype, uint_t sleepflag)
2689 {
2690         tavor_cmd_post_t        cmd;
2691         int                     status;
2692 
2693         TAVOR_TNF_ENTER(tavor_conf_special_qp_cmd_post);
2694 
2695         /* Setup and post Tavor "CONF_SPECIAL_QP" command */
2696         cmd.cp_inparm   = 0;
2697         cmd.cp_outparm  = 0;
2698         cmd.cp_inmod    = qpindx;
2699         cmd.cp_opcode   = CONF_SPECIAL_QP;
2700         cmd.cp_opmod    = qptype;
2701         cmd.cp_flags    = sleepflag;
2702         status = tavor_cmd_post(state, &cmd);
2703         if (status != TAVOR_CMD_SUCCESS) {
2704                 TNF_PROBE_0(tavor_conf_special_qp_cmd_post_fail,
2705                     TAVOR_TNF_ERROR, "");
2706         }
2707 
2708         TAVOR_TNF_EXIT(tavor_conf_special_qp_cmd_post);
2709         return (status);
2710 }
2711 
2712 
2713 /*
2714  * tavor_mgid_hash_cmd_post()
2715  *    Context: Can be called from interrupt or base context.
2716  */
2717 int
2718 tavor_mgid_hash_cmd_post(tavor_state_t *state, uint64_t mgid_h,
2719     uint64_t mgid_l, uint64_t *mgid_hash, uint_t sleepflag)
2720 {
2721         tavor_mbox_info_t       mbox_info;
2722         tavor_cmd_post_t        cmd;
2723         int                     status;
2724 
2725         TAVOR_TNF_ENTER(tavor_mgid_hash_cmd_post);
2726 
2727         /* Get an "In" mailbox for the command */
2728         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2729         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2730         if (status != TAVOR_CMD_SUCCESS) {
2731                 TNF_PROBE_0(tavor_mgid_hash_mbox_fail, TAVOR_TNF_ERROR, "");
2732                 TAVOR_TNF_EXIT(tavor_mgid_hash_cmd_post);
2733                 return (status);
2734         }
2735 
2736         /* Copy the Tavor "MGID_HASH" command into mailbox */
2737         ddi_put64(mbox_info.mbi_in->mb_acchdl,
2738             ((uint64_t *)mbox_info.mbi_in->mb_addr + 0), mgid_h);
2739         ddi_put64(mbox_info.mbi_in->mb_acchdl,
2740             ((uint64_t *)mbox_info.mbi_in->mb_addr + 1), mgid_l);
2741 
2742         /* Sync the mailbox for the device to read */
2743         tavor_mbox_sync(mbox_info.mbi_in, 0, TAVOR_CMD_MGIDHASH_SZ,
2744             DDI_DMA_SYNC_FORDEV);
2745 
2746         /* Setup and post the Tavor "MGID_HASH" command */
2747         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
2748         cmd.cp_outparm  = 0;
2749         cmd.cp_inmod    = 0;
2750         cmd.cp_opcode   = MGID_HASH;
2751         cmd.cp_opmod    = 0;
2752         cmd.cp_flags    = sleepflag;
2753         status = tavor_cmd_post(state, &cmd);
2754         if (status != TAVOR_CMD_SUCCESS) {
2755                 TNF_PROBE_0(tavor_mgid_hash_cmd_post_fail, TAVOR_TNF_ERROR, "");
2756         }
2757 
2758         /* MGID hash value is returned in command "outparam" */
2759         *mgid_hash = cmd.cp_outparm;
2760 
2761         /* Free the mailbox */
2762         tavor_mbox_free(state, &mbox_info);
2763 
2764         TAVOR_TNF_EXIT(tavor_mgid_hash_cmd_post);
2765         return (status);
2766 }
2767 
2768 
2769 /*
2770  * tavor_read_mgm_cmd_post()
2771  *    Context: Can be called from interrupt or base context.
2772  *
2773  * Note: It is assumed that the "mcg" parameter is actually a pointer to a
2774  *    "tavor_hw_mcg_t" struct and some number of "tavor_hw_mcg_qp_list_t"
2775  *    structs.  Combined size should be equal to result of TAVOR_MCGMEM_SZ()
2776  *    macro.
2777  */
2778 int
2779 tavor_read_mgm_cmd_post(tavor_state_t *state, tavor_hw_mcg_t *mcg,
2780     uint_t mcgindx, uint_t sleepflag)
2781 {
2782         tavor_mbox_info_t       mbox_info;
2783         tavor_cmd_post_t        cmd;
2784         uint64_t                data;
2785         uint_t                  size, hdrsz, qplistsz;
2786         int                     status, i;
2787 
2788         TAVOR_TNF_ENTER(tavor_read_mgm_cmd_post);
2789 
2790         /* Get an "Out" mailbox for the results */
2791         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
2792         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2793         if (status != TAVOR_CMD_SUCCESS) {
2794                 TNF_PROBE_0(tavor_read_mgm_mbox_fail, TAVOR_TNF_ERROR, "");
2795                 TAVOR_TNF_EXIT(tavor_read_mgm_cmd_post);
2796                 return (status);
2797         }
2798 
2799         /* Setup and post Tavor "READ_MGM" command */
2800         cmd.cp_inparm   = 0;
2801         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
2802         cmd.cp_inmod    = mcgindx;
2803         cmd.cp_opcode   = READ_MGM;
2804         cmd.cp_opmod    = 0;
2805         cmd.cp_flags    = sleepflag;
2806         status = tavor_cmd_post(state, &cmd);
2807         if (status != TAVOR_CMD_SUCCESS) {
2808                 TNF_PROBE_0(tavor_read_mgm_cmd_post_fail, TAVOR_TNF_ERROR, "");
2809                 goto read_mgm_fail;
2810         }
2811 
2812         /* Sync the mailbox to read the results */
2813         size = TAVOR_MCGMEM_SZ(state);
2814         tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
2815 
2816         /* Copy the READ_MGM command results into "mcg" */
2817         hdrsz = sizeof (tavor_hw_mcg_t);
2818         for (i = 0; i < (hdrsz >> 3); i++) {
2819                 data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
2820                     ((uint64_t *)mbox_info.mbi_out->mb_addr + i));
2821                 ((uint64_t *)mcg)[i] = data;
2822         }
2823         qplistsz = size - hdrsz;
2824         for (i = 0; i < (qplistsz >> 2); i++) {
2825                 data = ddi_get32(mbox_info.mbi_out->mb_acchdl,
2826                     ((uint32_t *)mbox_info.mbi_out->mb_addr + i + 8));
2827                 ((uint32_t *)mcg)[i + 8] = data;
2828         }
2829 
2830 read_mgm_fail:
2831         /* Free the mailbox */
2832         tavor_mbox_free(state, &mbox_info);
2833 
2834         TAVOR_TNF_EXIT(tavor_read_mgm_cmd_post);
2835         return (status);
2836 }
2837 
2838 
2839 /*
2840  * tavor_write_mgm_cmd_post()
2841  *    Context: Can be called from interrupt or base context.
2842  *
2843  * Note: It is assumed that the "mcg" parameter is actually a pointer to a
2844  *    "tavor_hw_mcg_t" struct and some number of "tavor_hw_mcg_qp_list_t"
2845  *    structs.  Combined size should be equal to result of TAVOR_MCGMEM_SZ()
2846  *    macro.
2847  */
2848 int
2849 tavor_write_mgm_cmd_post(tavor_state_t *state, tavor_hw_mcg_t *mcg,
2850     uint_t mcgindx, uint_t sleepflag)
2851 {
2852         tavor_mbox_info_t       mbox_info;
2853         tavor_cmd_post_t        cmd;
2854         uint64_t                data;
2855         uint_t                  size, hdrsz, qplistsz;
2856         int                     status, i;
2857 
2858         TAVOR_TNF_ENTER(tavor_write_mgm_cmd_post);
2859 
2860         /* Get an "In" mailbox for the command */
2861         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2862         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2863         if (status != TAVOR_CMD_SUCCESS) {
2864                 TNF_PROBE_0(tavor_write_mcg_mbox_fail, TAVOR_TNF_ERROR, "");
2865                 TAVOR_TNF_EXIT(tavor_write_mgm_cmd_post);
2866                 return (status);
2867         }
2868 
2869         /* Copy the Tavor "WRITE_MGM" command into mailbox */
2870         size  = TAVOR_MCGMEM_SZ(state);
2871         hdrsz = sizeof (tavor_hw_mcg_t);
2872         for (i = 0; i < (hdrsz >> 3); i++) {
2873                 data = ((uint64_t *)mcg)[i];
2874                 ddi_put64(mbox_info.mbi_in->mb_acchdl,
2875                     ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
2876         }
2877         qplistsz = size - hdrsz;
2878         for (i = 0; i < (qplistsz >> 2); i++) {
2879                 data = ((uint32_t *)mcg)[i + 8];
2880                 ddi_put32(mbox_info.mbi_in->mb_acchdl,
2881                     ((uint32_t *)mbox_info.mbi_in->mb_addr + i + 8), data);
2882         }
2883 
2884         /* Sync the mailbox for the device to read */
2885         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2886 
2887         /* Setup and post Tavor "WRITE_MGM" command */
2888         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
2889         cmd.cp_outparm  = 0;
2890         cmd.cp_inmod    = mcgindx;
2891         cmd.cp_opcode   = WRITE_MGM;
2892         cmd.cp_opmod    = 0;
2893         cmd.cp_flags    = sleepflag;
2894         status = tavor_cmd_post(state, &cmd);
2895         if (status != TAVOR_CMD_SUCCESS) {
2896                 TNF_PROBE_0(tavor_write_mgm_cmd_post_fail, TAVOR_TNF_ERROR, "");
2897         }
2898 
2899         /* Free the mailbox */
2900         tavor_mbox_free(state, &mbox_info);
2901 
2902         TAVOR_TNF_EXIT(tavor_write_mgm_cmd_post);
2903         return (status);
2904 
2905 }
2906 
2907 
2908 /*
2909  * tavor_modify_mpt_cmd_post()
2910  *    Context: Can be called from interrupt or base context.
2911  */
2912 int
2913 tavor_modify_mpt_cmd_post(tavor_state_t *state, tavor_hw_mpt_t *mpt,
2914     uint_t mptindx, uint_t flags, uint_t sleepflag)
2915 {
2916         tavor_mbox_info_t       mbox_info;
2917         tavor_cmd_post_t        cmd;
2918         uint64_t                data;
2919         uint_t                  size;
2920         int                     status, i;
2921 
2922         TAVOR_TNF_ENTER(tavor_modify_mpt_cmd_post);
2923 
2924         /* Get an "In" mailbox for the command */
2925         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2926         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2927         if (status != TAVOR_CMD_SUCCESS) {
2928                 TNF_PROBE_0(tavor_modify_mpt_mbox_fail, TAVOR_TNF_ERROR, "");
2929                 TAVOR_TNF_EXIT(tavor_modify_mpt_cmd_post);
2930                 return (status);
2931         }
2932 
2933         /* Copy the Tavor "MODIFY_MPT" command into mailbox */
2934         size = sizeof (tavor_hw_mpt_t);
2935         for (i = 0; i < (size >> 3); i++) {
2936                 data = ((uint64_t *)mpt)[i];
2937                 ddi_put64(mbox_info.mbi_in->mb_acchdl,
2938                     ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
2939         }
2940 
2941         /* Sync the mailbox for the device to read */
2942         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2943 
2944         /* Setup and post Tavor "MODIFY_MPT" command */
2945         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
2946         cmd.cp_outparm  = 0;
2947         cmd.cp_inmod    = mptindx;
2948         cmd.cp_opcode   = MODIFY_MPT;
2949         cmd.cp_opmod    = flags;
2950         cmd.cp_flags    = sleepflag;
2951         status = tavor_cmd_post(state, &cmd);
2952         if (status != TAVOR_CMD_SUCCESS) {
2953                 TNF_PROBE_0(tavor_modify_mpt_cmd_post_fail,
2954                     TAVOR_TNF_ERROR, "");
2955         }
2956 
2957         /* Free the mailbox */
2958         tavor_mbox_free(state, &mbox_info);
2959 
2960         TAVOR_TNF_EXIT(tavor_modify_mpt_cmd_post);
2961         return (status);
2962 }
2963 
2964 /*
2965  * tavor_getpefcntr_cmd_post()
2966  *    Context: Can be called from interrupt or base context.
2967  *
2968  * If reset is zero, read the performance counters of the specified port and
2969  * copy them into perfinfo.
2970  * If reset is non-zero reset the performance counters of the specified port.
2971  */
2972 int
2973 tavor_getperfcntr_cmd_post(tavor_state_t *state, uint_t port,
2974     uint_t sleepflag, tavor_hw_sm_perfcntr_t *perfinfo, int reset)
2975 {
2976         tavor_mbox_info_t       mbox_info;
2977         tavor_cmd_post_t        cmd;
2978         uint64_t                data;
2979         uint32_t                *mbox;
2980         uint_t                  size;
2981         int                     status, i;
2982 
2983         bzero((void *)&cmd, sizeof (tavor_cmd_post_t));
2984 
2985         /* Get "In" and "Out" mailboxes for the command */
2986         mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
2987         status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2988         if (status != TAVOR_CMD_SUCCESS) {
2989                 return (status);
2990         }
2991 
2992         /* Build request MAD in the "In" mailbox */
2993         size = TAVOR_CMD_MAD_IFC_SIZE;
2994         mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
2995 
2996         if (reset) {
2997                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0],
2998                     TAVOR_CMD_PERF_SET);
2999         } else {
3000                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0],
3001                     TAVOR_CMD_PERF_GET);
3002         }
3003         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
3004         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
3005         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
3006         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_PERFCNTRS);
3007         ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], TAVOR_CMD_PERFATTR);
3008 
3009         if (reset) {
3010                 /* reset counters for XmitData, RcvData, XmitPkts, RcvPkts */
3011                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[16],
3012                     ((port << 16) | 0xf000));
3013 
3014                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[22], 0);
3015                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[23], 0);
3016                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[24], 0);
3017                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[25], 0);
3018         } else
3019                 ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[16], (port << 16));
3020 
3021         /* Sync the mailbox for the device to read */
3022         tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
3023 
3024         /* Setup the Hermon "MAD_IFC" command */
3025         cmd.cp_inparm   = mbox_info.mbi_in->mb_mapaddr;
3026         cmd.cp_outparm  = mbox_info.mbi_out->mb_mapaddr;
3027         cmd.cp_inmod    = port;
3028         cmd.cp_opcode   = MAD_IFC;
3029         /* No MKey and BKey checking */
3030         cmd.cp_opmod    = TAVOR_CMD_MKEY_DONTCHECK | TAVOR_CMD_BKEY_DONTCHECK;
3031         cmd.cp_flags    = TAVOR_CMD_NOSLEEP_SPIN; /* NO SLEEP */
3032         status = tavor_cmd_post(state, &cmd);
3033         if (status != TAVOR_CMD_SUCCESS) {
3034                 goto getperfinfo_fail;
3035         }
3036 
3037         /* Sync the mailbox to read the results */
3038         size = TAVOR_CMD_MAD_IFC_SIZE;
3039         tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
3040 
3041         if (reset == 0) {
3042                 size = sizeof (tavor_hw_sm_perfcntr_t); /* for the copy */
3043                 /*
3044                  * Copy Perfcounters into "perfinfo".  We can discard the MAD
3045                  * header and the 8 Quadword reserved area of the PERM mgmt
3046                  * class MAD
3047                  */
3048 
3049                 for (i = 0; i < size >> 3; i++) {
3050                         data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
3051                             ((uint64_t *)mbox_info.mbi_out->mb_addr + i + 8));
3052                         ((uint64_t *)(void *)perfinfo)[i] = data;
3053                 }
3054         }
3055 
3056 getperfinfo_fail:
3057         /* Free the mailbox */
3058         tavor_mbox_free(state, &mbox_info);
3059         return (status);
3060 }