1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 #include <sys/ib/ibtl/impl/ibtl.h>
  26 #include <sys/ib/ibtl/impl/ibtl_cm.h>
  27 #include <sys/taskq.h>
  28 #include <sys/disp.h>
  29 #include <sys/callb.h>
  30 #include <sys/proc.h>
  31 
  32 /*
  33  * ibtl_handlers.c
  34  */
  35 
  36 /*
  37  * What's in this file?
  38  *
  39  *   This file started as an implementation of Asynchronous Event/Error
  40  *   handling and Completion Queue handling.  As the implementation
  41  *   evolved, code has been added for other ibc_* interfaces (resume,
  42  *   predetach, etc.) that use the same mechanisms as used for asyncs.
  43  *
  44  * Async and CQ handling at interrupt level.
  45  *
  46  *   CQ handling is normally done at interrupt level using the CQ callback
  47  *   handler to call the appropriate IBT Client (owner of the CQ).  For
  48  *   clients that would prefer a fully flexible non-interrupt context to
  49  *   do their CQ handling, a CQ can be created so that its handler is
  50  *   called from a non-interrupt thread.  CQ handling is done frequently
  51  *   whereas Async handling is expected to occur very infrequently.
  52  *
  53  *   Async handling is done by marking (or'ing in of an async_code of) the
  54  *   pertinent IBTL data structure, and then notifying the async_thread(s)
  55  *   that the data structure has async work to be done.  The notification
  56  *   occurs by linking the data structure through its async_link onto a
  57  *   list of like data structures and waking up an async_thread.  This
  58  *   list append is not done if there is already async work pending on
  59  *   this data structure (IBTL_ASYNC_PENDING).
  60  *
  61  * Async Mutex and CQ Mutex
  62  *
  63  *   The global ibtl_async_mutex is "the" mutex used to control access
  64  *   to all the data needed by ibc_async_handler.  All the threads that
  65  *   use this mutex are written so that the mutex is held for very short
  66  *   periods of time, and never held while making calls to functions
  67  *   that may block.
  68  *
  69  *   The global ibtl_cq_mutex is used similarly by ibc_cq_handler and
  70  *   the ibtl_cq_thread(s).
  71  *
  72  * Mutex hierarchy
  73  *
  74  *   The ibtl_clnt_list_mutex is above the ibtl_async_mutex.
  75  *   ibtl_clnt_list_mutex protects all of the various lists.
  76  *   The ibtl_async_mutex is below this in the hierarchy.
  77  *
  78  *   The ibtl_cq_mutex is independent of the above mutexes.
  79  *
  80  * Threads
  81  *
  82  *   There are "ibtl_cq_threads" number of threads created for handling
  83  *   Completion Queues in threads.  If this feature really gets used,
  84  *   then we will want to do some suitable tuning.  Similarly, we may
  85  *   want to tune the number of "ibtl_async_thread_init".
  86  *
  87  *   The function ibtl_cq_thread is the main loop for handling a CQ in a
  88  *   thread.  There can be multiple threads executing this same code.
  89  *   The code sleeps when there is no work to be done (list is empty),
  90  *   otherwise it pulls the first CQ structure off the list and performs
  91  *   the CQ handler callback to the client.  After that returns, a check
  92  *   is made, and if another ibc_cq_handler call was made for this CQ,
  93  *   the client is called again.
  94  *
  95  *   The function ibtl_async_thread is the main loop for handling async
  96  *   events/errors.  There can be multiple threads executing this same code.
  97  *   The code sleeps when there is no work to be done (lists are empty),
  98  *   otherwise it pulls the first structure off one of the lists and
  99  *   performs the async callback(s) to the client(s).  Note that HCA
 100  *   async handling is done by calling each of the clients using the HCA.
 101  *   When the async handling completes, the data structure having the async
 102  *   event/error is checked for more work before it's considered "done".
 103  *
 104  * Taskq
 105  *
 106  *   The async_taskq is used here for allowing async handler callbacks to
 107  *   occur simultaneously to multiple clients of an HCA.  This taskq could
 108  *   be used for other purposes, e.g., if all the async_threads are in
 109  *   use, but this is deemed as overkill since asyncs should occur rarely.
 110  */
 111 
 112 /* Globals */
 113 static char ibtf_handlers[] = "ibtl_handlers";
 114 
 115 /* priority for IBTL threads (async, cq, and taskq) */
 116 static pri_t ibtl_pri = MAXCLSYSPRI - 1; /* maybe override in /etc/system */
 117 
 118 /* taskq used for HCA asyncs */
 119 #define ibtl_async_taskq system_taskq
 120 
 121 /* data for async handling by threads */
 122 static kmutex_t ibtl_async_mutex;       /* protects most *_async_* data */
 123 static kcondvar_t ibtl_async_cv;        /* async_threads wait on this */
 124 static kcondvar_t ibtl_clnt_cv;         /* ibt_detach might wait on this */
 125 static void ibtl_dec_clnt_async_cnt(ibtl_clnt_t *clntp);
 126 static void ibtl_inc_clnt_async_cnt(ibtl_clnt_t *clntp);
 127 
 128 static kt_did_t *ibtl_async_did;        /* for thread_join() */
 129 int ibtl_async_thread_init = 4; /* total # of async_threads to create */
 130 static int ibtl_async_thread_exit = 0;  /* set if/when thread(s) should exit */
 131 
 132 /* async lists for various structures */
 133 static ibtl_hca_devinfo_t *ibtl_async_hca_list_start, *ibtl_async_hca_list_end;
 134 static ibtl_eec_t *ibtl_async_eec_list_start, *ibtl_async_eec_list_end;
 135 static ibtl_qp_t *ibtl_async_qp_list_start, *ibtl_async_qp_list_end;
 136 static ibtl_cq_t *ibtl_async_cq_list_start, *ibtl_async_cq_list_end;
 137 static ibtl_srq_t *ibtl_async_srq_list_start, *ibtl_async_srq_list_end;
 138 
 139 /* data for CQ completion handling by threads */
 140 static kmutex_t ibtl_cq_mutex;  /* protects the cv and the list below */
 141 static kcondvar_t ibtl_cq_cv;
 142 static ibtl_cq_t *ibtl_cq_list_start, *ibtl_cq_list_end;
 143 
 144 static int ibtl_cq_threads = 0;         /* total # of cq threads */
 145 static int ibtl_cqs_using_threads = 0;  /* total # of cqs using threads */
 146 static int ibtl_cq_thread_exit = 0;     /* set if/when thread(s) should exit */
 147 
 148 /* value used to tell IBTL threads to exit */
 149 #define IBTL_THREAD_EXIT 0x1b7fdead     /* IBTF DEAD */
 150 /* Cisco Topspin Vendor ID for Rereg hack */
 151 #define IBT_VENDOR_CISCO 0x05ad
 152 
 153 int ibtl_eec_not_supported = 1;
 154 
 155 char *ibtl_last_client_name;    /* may help debugging */
 156 typedef ibt_status_t (*ibtl_node_info_cb_t)(ib_guid_t, uint8_t, ib_lid_t,
 157     ibt_node_info_t *);
 158 
 159 ibtl_node_info_cb_t ibtl_node_info_cb;
 160 
 161 void
 162 ibtl_cm_set_node_info_cb(ibt_status_t (*node_info_cb)(ib_guid_t, uint8_t,
 163     ib_lid_t, ibt_node_info_t *))
 164 {
 165         mutex_enter(&ibtl_clnt_list_mutex);
 166         ibtl_node_info_cb = node_info_cb;
 167         mutex_exit(&ibtl_clnt_list_mutex);
 168 }
 169 
 170 /*
 171  * ibc_async_handler()
 172  *
 173  * Asynchronous Event/Error Handler.
 174  *
 175  *      This is the function called HCA drivers to post various async
 176  *      event and errors mention in the IB architecture spec.  See
 177  *      ibtl_types.h for additional details of this.
 178  *
 179  *      This function marks the pertinent IBTF object with the async_code,
 180  *      and queues the object for handling by an ibtl_async_thread.  If
 181  *      the object is NOT already marked for async processing, it is added
 182  *      to the associated list for that type of object, and an
 183  *      ibtl_async_thread is signaled to finish the async work.
 184  */
 185 void
 186 ibc_async_handler(ibc_clnt_hdl_t hca_devp, ibt_async_code_t code,
 187     ibc_async_event_t *event_p)
 188 {
 189         ibtl_qp_t       *ibtl_qp;
 190         ibtl_cq_t       *ibtl_cq;
 191         ibtl_srq_t      *ibtl_srq;
 192         ibtl_eec_t      *ibtl_eec;
 193         uint8_t         port_minus1;
 194 
 195         ibtl_async_port_event_t *portp;
 196 
 197         IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler(%p, 0x%x, %p)",
 198             hca_devp, code, event_p);
 199 
 200         mutex_enter(&ibtl_async_mutex);
 201 
 202         switch (code) {
 203         case IBT_EVENT_PATH_MIGRATED_QP:
 204         case IBT_EVENT_SQD:
 205         case IBT_ERROR_CATASTROPHIC_QP:
 206         case IBT_ERROR_PATH_MIGRATE_REQ_QP:
 207         case IBT_EVENT_COM_EST_QP:
 208         case IBT_ERROR_INVALID_REQUEST_QP:
 209         case IBT_ERROR_ACCESS_VIOLATION_QP:
 210         case IBT_EVENT_EMPTY_QP:
 211         case IBT_FEXCH_ERROR:
 212                 ibtl_qp = event_p->ev_qp_hdl;
 213                 if (ibtl_qp == NULL) {
 214                         IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
 215                             "bad qp handle");
 216                         break;
 217                 }
 218                 switch (code) {
 219                 case IBT_ERROR_CATASTROPHIC_QP:
 220                         ibtl_qp->qp_cat_fma_ena = event_p->ev_fma_ena; break;
 221                 case IBT_ERROR_PATH_MIGRATE_REQ_QP:
 222                         ibtl_qp->qp_pth_fma_ena = event_p->ev_fma_ena; break;
 223                 case IBT_ERROR_INVALID_REQUEST_QP:
 224                         ibtl_qp->qp_inv_fma_ena = event_p->ev_fma_ena; break;
 225                 case IBT_ERROR_ACCESS_VIOLATION_QP:
 226                         ibtl_qp->qp_acc_fma_ena = event_p->ev_fma_ena; break;
 227                 }
 228 
 229                 ibtl_qp->qp_async_codes |= code;
 230                 if ((ibtl_qp->qp_async_flags & IBTL_ASYNC_PENDING) == 0) {
 231                         ibtl_qp->qp_async_flags |= IBTL_ASYNC_PENDING;
 232                         ibtl_qp->qp_async_link = NULL;
 233                         if (ibtl_async_qp_list_end == NULL)
 234                                 ibtl_async_qp_list_start = ibtl_qp;
 235                         else
 236                                 ibtl_async_qp_list_end->qp_async_link = ibtl_qp;
 237                         ibtl_async_qp_list_end = ibtl_qp;
 238                         cv_signal(&ibtl_async_cv);
 239                 }
 240                 break;
 241 
 242         case IBT_ERROR_CQ:
 243                 ibtl_cq = event_p->ev_cq_hdl;
 244                 if (ibtl_cq == NULL) {
 245                         IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
 246                             "bad cq handle");
 247                         break;
 248                 }
 249                 ibtl_cq->cq_async_codes |= code;
 250                 ibtl_cq->cq_fma_ena = event_p->ev_fma_ena;
 251                 if ((ibtl_cq->cq_async_flags & IBTL_ASYNC_PENDING) == 0) {
 252                         ibtl_cq->cq_async_flags |= IBTL_ASYNC_PENDING;
 253                         ibtl_cq->cq_async_link = NULL;
 254                         if (ibtl_async_cq_list_end == NULL)
 255                                 ibtl_async_cq_list_start = ibtl_cq;
 256                         else
 257                                 ibtl_async_cq_list_end->cq_async_link = ibtl_cq;
 258                         ibtl_async_cq_list_end = ibtl_cq;
 259                         cv_signal(&ibtl_async_cv);
 260                 }
 261                 break;
 262 
 263         case IBT_ERROR_CATASTROPHIC_SRQ:
 264         case IBT_EVENT_LIMIT_REACHED_SRQ:
 265                 ibtl_srq = event_p->ev_srq_hdl;
 266                 if (ibtl_srq == NULL) {
 267                         IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
 268                             "bad srq handle");
 269                         break;
 270                 }
 271                 ibtl_srq->srq_async_codes |= code;
 272                 ibtl_srq->srq_fma_ena = event_p->ev_fma_ena;
 273                 if ((ibtl_srq->srq_async_flags & IBTL_ASYNC_PENDING) == 0) {
 274                         ibtl_srq->srq_async_flags |= IBTL_ASYNC_PENDING;
 275                         ibtl_srq->srq_async_link = NULL;
 276                         if (ibtl_async_srq_list_end == NULL)
 277                                 ibtl_async_srq_list_start = ibtl_srq;
 278                         else
 279                                 ibtl_async_srq_list_end->srq_async_link =
 280                                     ibtl_srq;
 281                         ibtl_async_srq_list_end = ibtl_srq;
 282                         cv_signal(&ibtl_async_cv);
 283                 }
 284                 break;
 285 
 286         case IBT_EVENT_PATH_MIGRATED_EEC:
 287         case IBT_ERROR_PATH_MIGRATE_REQ_EEC:
 288         case IBT_ERROR_CATASTROPHIC_EEC:
 289         case IBT_EVENT_COM_EST_EEC:
 290                 if (ibtl_eec_not_supported) {
 291                         IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
 292                             "EEC events are disabled.");
 293                         break;
 294                 }
 295                 ibtl_eec = event_p->ev_eec_hdl;
 296                 if (ibtl_eec == NULL) {
 297                         IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
 298                             "bad eec handle");
 299                         break;
 300                 }
 301                 switch (code) {
 302                 case IBT_ERROR_PATH_MIGRATE_REQ_EEC:
 303                         ibtl_eec->eec_pth_fma_ena = event_p->ev_fma_ena; break;
 304                 case IBT_ERROR_CATASTROPHIC_EEC:
 305                         ibtl_eec->eec_cat_fma_ena = event_p->ev_fma_ena; break;
 306                 }
 307                 ibtl_eec->eec_async_codes |= code;
 308                 if ((ibtl_eec->eec_async_flags & IBTL_ASYNC_PENDING) == 0) {
 309                         ibtl_eec->eec_async_flags |= IBTL_ASYNC_PENDING;
 310                         ibtl_eec->eec_async_link = NULL;
 311                         if (ibtl_async_eec_list_end == NULL)
 312                                 ibtl_async_eec_list_start = ibtl_eec;
 313                         else
 314                                 ibtl_async_eec_list_end->eec_async_link =
 315                                     ibtl_eec;
 316                         ibtl_async_eec_list_end = ibtl_eec;
 317                         cv_signal(&ibtl_async_cv);
 318                 }
 319                 break;
 320 
 321         case IBT_ERROR_LOCAL_CATASTROPHIC:
 322                 hca_devp->hd_async_codes |= code;
 323                 hca_devp->hd_fma_ena = event_p->ev_fma_ena;
 324                 /* FALLTHROUGH */
 325 
 326         case IBT_EVENT_PORT_UP:
 327         case IBT_PORT_CHANGE_EVENT:
 328         case IBT_CLNT_REREG_EVENT:
 329         case IBT_ERROR_PORT_DOWN:
 330                 if ((code & IBT_PORT_EVENTS) != 0) {
 331                         if ((port_minus1 = event_p->ev_port - 1) >=
 332                             hca_devp->hd_hca_attr->hca_nports) {
 333                                 IBTF_DPRINTF_L2(ibtf_handlers,
 334                                     "ibc_async_handler: bad port #: %d",
 335                                     event_p->ev_port);
 336                                 break;
 337                         }
 338                         portp = &hca_devp->hd_async_port[port_minus1];
 339                         if (code == IBT_EVENT_PORT_UP) {
 340                                 /*
 341                                  * The port is just coming UP we can't have any
 342                                  * valid older events.
 343                                  */
 344                                 portp->status = IBTL_HCA_PORT_UP;
 345                         } else if (code == IBT_ERROR_PORT_DOWN) {
 346                                 /*
 347                                  * The port is going DOWN older events don't
 348                                  * count.
 349                                  */
 350                                 portp->status = IBTL_HCA_PORT_DOWN;
 351                         } else if (code == IBT_PORT_CHANGE_EVENT) {
 352                                 /*
 353                                  * For port UP and DOWN events only the latest
 354                                  * event counts. If we get a UP after DOWN it
 355                                  * is sufficient to send just UP and vice versa.
 356                                  * In the case of port CHANGE event it is valid
 357                                  * only when the port is UP already but if we
 358                                  * receive it after UP but before UP is
 359                                  * delivered we still need to deliver CHANGE
 360                                  * after we deliver UP event.
 361                                  *
 362                                  * We will not get a CHANGE event when the port
 363                                  * is down or DOWN event is pending.
 364                                  */
 365                                 portp->flags |= event_p->ev_port_flags;
 366                                 portp->status |= IBTL_HCA_PORT_CHG;
 367                         } else if (code == IBT_CLNT_REREG_EVENT) {
 368                                 /*
 369                                  * SM has requested a re-register of
 370                                  * subscription to SM events notification.
 371                                  */
 372                                 portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
 373                         }
 374 
 375                         hca_devp->hd_async_codes |= code;
 376                 }
 377 
 378                 if ((hca_devp->hd_async_flags & IBTL_ASYNC_PENDING) == 0) {
 379                         hca_devp->hd_async_flags |= IBTL_ASYNC_PENDING;
 380                         hca_devp->hd_async_link = NULL;
 381                         if (ibtl_async_hca_list_end == NULL)
 382                                 ibtl_async_hca_list_start = hca_devp;
 383                         else
 384                                 ibtl_async_hca_list_end->hd_async_link =
 385                                     hca_devp;
 386                         ibtl_async_hca_list_end = hca_devp;
 387                         cv_signal(&ibtl_async_cv);
 388                 }
 389 
 390                 break;
 391 
 392         default:
 393                 IBTF_DPRINTF_L1(ibtf_handlers, "ibc_async_handler: "
 394                     "invalid code (0x%x)", code);
 395         }
 396 
 397         mutex_exit(&ibtl_async_mutex);
 398 }
 399 
 400 
 401 /* Finally, make the async call to the client. */
 402 
 403 static void
 404 ibtl_async_client_call(ibtl_hca_t *ibt_hca, ibt_async_code_t code,
 405     ibt_async_event_t *event_p)
 406 {
 407         ibtl_clnt_t             *clntp;
 408         void                    *client_private;
 409         ibt_async_handler_t     async_handler;
 410         char                    *client_name;
 411 
 412         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call(%p, 0x%x, %p)",
 413             ibt_hca, code, event_p);
 414 
 415         clntp = ibt_hca->ha_clnt_devp;
 416 
 417         /* Record who is being called (just a debugging aid) */
 418         ibtl_last_client_name = client_name = clntp->clnt_name;
 419 
 420         client_private = clntp->clnt_private;
 421         async_handler = clntp->clnt_modinfop->mi_async_handler;
 422 
 423         if (code & (IBT_EVENT_COM_EST_QP | IBT_EVENT_COM_EST_EEC)) {
 424                 mutex_enter(&ibtl_clnt_list_mutex);
 425                 async_handler = ibtl_cm_async_handler;
 426                 client_private = ibtl_cm_clnt_private;
 427                 mutex_exit(&ibtl_clnt_list_mutex);
 428                 ibt_hca = NULL;
 429                 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
 430                     "calling CM for COM_EST");
 431         } else {
 432                 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
 433                     "calling client '%s'", client_name);
 434         }
 435         if (async_handler != NULL)
 436                 async_handler(client_private, ibt_hca, code, event_p);
 437         else
 438                 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
 439                     "client '%s' has no async handler", client_name);
 440 }
 441 
 442 /*
 443  * Inform CM or DM about HCA events.
 444  *
 445  *      We use taskqs to allow simultaneous notification, with sleeping.
 446  *      Since taskqs only allow one argument, we define a structure
 447  *      because we need to pass in more than one argument.
 448  */
 449 
 450 struct ibtl_mgr_s {
 451         ibtl_hca_devinfo_t      *mgr_hca_devp;
 452         ibt_async_handler_t     mgr_async_handler;
 453         void                    *mgr_clnt_private;
 454 };
 455 
 456 /*
 457  * Asyncs of HCA level events for CM and DM.  Call CM or DM and tell them
 458  * about the HCA for the event recorded in the ibtl_hca_devinfo_t.
 459  */
 460 static void
 461 ibtl_do_mgr_async_task(void *arg)
 462 {
 463         struct ibtl_mgr_s       *mgrp = (struct ibtl_mgr_s *)arg;
 464         ibtl_hca_devinfo_t      *hca_devp = mgrp->mgr_hca_devp;
 465 
 466         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_mgr_async_task(0x%x)",
 467             hca_devp->hd_async_code);
 468 
 469         mgrp->mgr_async_handler(mgrp->mgr_clnt_private, NULL,
 470             hca_devp->hd_async_code, &hca_devp->hd_async_event);
 471         kmem_free(mgrp, sizeof (*mgrp));
 472 
 473         mutex_enter(&ibtl_clnt_list_mutex);
 474         if (--hca_devp->hd_async_task_cnt == 0)
 475                 cv_signal(&hca_devp->hd_async_task_cv);
 476         mutex_exit(&ibtl_clnt_list_mutex);
 477 }
 478 
 479 static void
 480 ibt_cisco_embedded_sm_rereg_fix(void *arg)
 481 {
 482         struct ibtl_mgr_s *mgrp = arg;
 483         ibtl_hca_devinfo_t *hca_devp;
 484         ibt_node_info_t node_info;
 485         ibt_status_t ibt_status;
 486         ibtl_async_port_event_t *portp;
 487         ib_lid_t sm_lid;
 488         ib_guid_t hca_guid;
 489         ibt_async_event_t *event_p;
 490         ibt_hca_portinfo_t *pinfop;
 491         uint8_t port;
 492 
 493         hca_devp = mgrp->mgr_hca_devp;
 494 
 495         mutex_enter(&ibtl_clnt_list_mutex);
 496         event_p = &hca_devp->hd_async_event;
 497         port = event_p->ev_port;
 498         portp = &hca_devp->hd_async_port[port - 1];
 499         pinfop = &hca_devp->hd_portinfop[port - 1];
 500         sm_lid = pinfop->p_sm_lid;
 501         hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
 502         mutex_exit(&ibtl_clnt_list_mutex);
 503 
 504         ibt_status = ((ibtl_node_info_cb_t)mgrp->mgr_async_handler)(hca_guid,
 505             port, sm_lid, &node_info);
 506         if (ibt_status == IBT_SUCCESS) {
 507                 if ((node_info.n_vendor_id == IBT_VENDOR_CISCO) &&
 508                     (node_info.n_node_type == IBT_NODE_TYPE_SWITCH)) {
 509                         mutex_enter(&ibtl_async_mutex);
 510                         portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
 511                         hca_devp->hd_async_codes |= IBT_CLNT_REREG_EVENT;
 512                         mutex_exit(&ibtl_async_mutex);
 513                 }
 514         }
 515         kmem_free(mgrp, sizeof (*mgrp));
 516 
 517         mutex_enter(&ibtl_clnt_list_mutex);
 518         if (--hca_devp->hd_async_task_cnt == 0)
 519                 cv_signal(&hca_devp->hd_async_task_cv);
 520         mutex_exit(&ibtl_clnt_list_mutex);
 521 }
 522 
 523 static void
 524 ibtl_cm_get_node_info(ibtl_hca_devinfo_t *hca_devp,
 525     ibt_async_handler_t async_handler)
 526 {
 527         struct ibtl_mgr_s *mgrp;
 528 
 529         if (async_handler == NULL)
 530                 return;
 531 
 532         mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
 533         mgrp->mgr_hca_devp = hca_devp;
 534         mgrp->mgr_async_handler = async_handler;
 535         mgrp->mgr_clnt_private = NULL;
 536         hca_devp->hd_async_task_cnt++;
 537 
 538         (void) taskq_dispatch(ibtl_async_taskq,
 539             ibt_cisco_embedded_sm_rereg_fix, mgrp, TQ_SLEEP);
 540 }
 541 
 542 static void
 543 ibtl_tell_mgr(ibtl_hca_devinfo_t *hca_devp, ibt_async_handler_t async_handler,
 544     void *clnt_private)
 545 {
 546         struct ibtl_mgr_s *mgrp;
 547 
 548         if (async_handler == NULL)
 549                 return;
 550 
 551         mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
 552         mgrp->mgr_hca_devp = hca_devp;
 553         mgrp->mgr_async_handler = async_handler;
 554         mgrp->mgr_clnt_private = clnt_private;
 555         hca_devp->hd_async_task_cnt++;
 556 
 557         (void) taskq_dispatch(ibtl_async_taskq, ibtl_do_mgr_async_task, mgrp,
 558             TQ_SLEEP);
 559 }
 560 
 561 /*
 562  * Per client-device asyncs for HCA level events.  Call each client that is
 563  * using the HCA for the event recorded in the ibtl_hca_devinfo_t.
 564  */
 565 static void
 566 ibtl_hca_client_async_task(void *arg)
 567 {
 568         ibtl_hca_t              *ibt_hca = (ibtl_hca_t *)arg;
 569         ibtl_hca_devinfo_t      *hca_devp = ibt_hca->ha_hca_devp;
 570         ibtl_clnt_t             *clntp = ibt_hca->ha_clnt_devp;
 571         ibt_async_event_t       async_event;
 572 
 573         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_hca_client_async_task(%p, 0x%x)",
 574             ibt_hca, hca_devp->hd_async_code);
 575 
 576         bcopy(&hca_devp->hd_async_event, &async_event, sizeof (async_event));
 577         ibtl_async_client_call(ibt_hca, hca_devp->hd_async_code, &async_event);
 578 
 579         mutex_enter(&ibtl_async_mutex);
 580         if (--ibt_hca->ha_async_cnt == 0 &&
 581             (ibt_hca->ha_async_flags & IBTL_ASYNC_FREE_OBJECT)) {
 582                 mutex_exit(&ibtl_async_mutex);
 583                 kmem_free(ibt_hca, sizeof (ibtl_hca_t));
 584         } else
 585                 mutex_exit(&ibtl_async_mutex);
 586 
 587         mutex_enter(&ibtl_clnt_list_mutex);
 588         if (--hca_devp->hd_async_task_cnt == 0)
 589                 cv_signal(&hca_devp->hd_async_task_cv);
 590         if (--clntp->clnt_async_cnt == 0)
 591                 cv_broadcast(&ibtl_clnt_cv);
 592 
 593         mutex_exit(&ibtl_clnt_list_mutex);
 594 }
 595 
 596 /*
 597  * Asyncs for HCA level events.
 598  *
 599  * The function continues to run until there are no more async
 600  * events/errors for this HCA.  An event is chosen for dispatch
 601  * to all clients of this HCA.  This thread dispatches them via
 602  * the ibtl_async_taskq, then sleeps until all tasks are done.
 603  *
 604  * This thread records the async_code and async_event in the
 605  * ibtl_hca_devinfo_t for all client taskq threads to reference.
 606  *
 607  * This is called from an async or taskq thread with ibtl_async_mutex held.
 608  */
 609 static void
 610 ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
 611 {
 612         ibtl_hca_t                      *ibt_hca;
 613         ibt_async_event_t               *eventp;
 614         ibt_async_code_t                code;
 615         ibtl_async_port_status_t        temp;
 616         uint8_t                         nports;
 617         uint8_t                         port_minus1;
 618         ibtl_async_port_event_t         *portp;
 619 
 620         mutex_exit(&ibtl_async_mutex);
 621 
 622         mutex_enter(&ibtl_clnt_list_mutex);
 623         while (hca_devp->hd_async_busy)
 624                 cv_wait(&hca_devp->hd_async_busy_cv, &ibtl_clnt_list_mutex);
 625         hca_devp->hd_async_busy = 1;
 626         mutex_enter(&ibtl_async_mutex);
 627 
 628         bzero(&hca_devp->hd_async_event, sizeof (hca_devp->hd_async_event));
 629         for (;;) {
 630 
 631                 hca_devp->hd_async_event.ev_fma_ena = 0;
 632 
 633                 code = hca_devp->hd_async_codes;
 634                 if (code & IBT_ERROR_LOCAL_CATASTROPHIC) {
 635                         code = IBT_ERROR_LOCAL_CATASTROPHIC;
 636                         hca_devp->hd_async_event.ev_fma_ena =
 637                             hca_devp->hd_fma_ena;
 638                 } else if (code & IBT_ERROR_PORT_DOWN) {
 639                         code = IBT_ERROR_PORT_DOWN;
 640                         temp = IBTL_HCA_PORT_DOWN;
 641                 } else if (code & IBT_EVENT_PORT_UP) {
 642                         code = IBT_EVENT_PORT_UP;
 643                         temp = IBTL_HCA_PORT_UP;
 644                 } else if (code & IBT_PORT_CHANGE_EVENT) {
 645                         code = IBT_PORT_CHANGE_EVENT;
 646                         temp = IBTL_HCA_PORT_CHG;
 647                 } else if (code & IBT_CLNT_REREG_EVENT) {
 648                         code = IBT_CLNT_REREG_EVENT;
 649                         temp = IBTL_HCA_PORT_ASYNC_CLNT_REREG;
 650                 } else {
 651                         hca_devp->hd_async_codes = 0;
 652                         code = 0;
 653                 }
 654 
 655                 if (code == 0) {
 656                         hca_devp->hd_async_flags &= ~IBTL_ASYNC_PENDING;
 657                         break;
 658                 }
 659                 hca_devp->hd_async_codes &= ~code;
 660 
 661                 /* PORT_UP, PORT_CHANGE, PORT_DOWN or ASYNC_REREG */
 662                 if ((code & IBT_PORT_EVENTS) != 0) {
 663                         portp = hca_devp->hd_async_port;
 664                         nports = hca_devp->hd_hca_attr->hca_nports;
 665                         for (port_minus1 = 0; port_minus1 < nports;
 666                             port_minus1++) {
 667                                 /*
 668                                  * Matching event in this port, let's go handle
 669                                  * it.
 670                                  */
 671                                 if ((portp[port_minus1].status & temp) != 0)
 672                                         break;
 673                         }
 674                         if (port_minus1 >= nports) {
 675                                 /* we checked again, but found nothing */
 676                                 continue;
 677                         }
 678                         IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_do_hca_asyncs: "
 679                             "async: port# %x code %x", port_minus1 + 1, code);
 680                         /* mark it to check for other ports after we're done */
 681                         hca_devp->hd_async_codes |= code;
 682 
 683                         /*
 684                          * Copy the event information into hca_devp and clear
 685                          * event information from the per port data.
 686                          */
 687                         hca_devp->hd_async_event.ev_port = port_minus1 + 1;
 688                         if (temp == IBTL_HCA_PORT_CHG) {
 689                                 hca_devp->hd_async_event.ev_port_flags =
 690                                     hca_devp->hd_async_port[port_minus1].flags;
 691                                 hca_devp->hd_async_port[port_minus1].flags = 0;
 692                         }
 693                         hca_devp->hd_async_port[port_minus1].status &= ~temp;
 694 
 695                         mutex_exit(&ibtl_async_mutex);
 696                         ibtl_reinit_hca_portinfo(hca_devp, port_minus1 + 1);
 697                         mutex_enter(&ibtl_async_mutex);
 698                         eventp = &hca_devp->hd_async_event;
 699                         eventp->ev_hca_guid =
 700                             hca_devp->hd_hca_attr->hca_node_guid;
 701                 }
 702 
 703                 hca_devp->hd_async_code = code;
 704                 hca_devp->hd_async_event.ev_hca_guid =
 705                     hca_devp->hd_hca_attr->hca_node_guid;
 706                 mutex_exit(&ibtl_async_mutex);
 707 
 708                 /*
 709                  * Make sure to inform CM, DM, and IBMA if we know of them.
 710                  * Also, make sure not to inform them a second time, which
 711                  * would occur if they have the HCA open.
 712                  */
 713 
 714                 if (ibtl_ibma_async_handler)
 715                         ibtl_tell_mgr(hca_devp, ibtl_ibma_async_handler,
 716                             ibtl_ibma_clnt_private);
 717                 /* wait for all tasks to complete */
 718                 while (hca_devp->hd_async_task_cnt != 0)
 719                         cv_wait(&hca_devp->hd_async_task_cv,
 720                             &ibtl_clnt_list_mutex);
 721 
 722                 /*
 723                  * Hack Alert:
 724                  * The ibmf handler would have updated the Master SM LID if it
 725                  * was SM LID change event. Now lets check if the new Master SM
 726                  * is a Embedded Cisco Topspin SM.
 727                  */
 728                 if ((code == IBT_PORT_CHANGE_EVENT) &&
 729                     eventp->ev_port_flags & IBT_PORT_CHANGE_SM_LID)
 730                         ibtl_cm_get_node_info(hca_devp,
 731                             (ibt_async_handler_t)ibtl_node_info_cb);
 732                 /* wait for node info task to complete */
 733                 while (hca_devp->hd_async_task_cnt != 0)
 734                         cv_wait(&hca_devp->hd_async_task_cv,
 735                             &ibtl_clnt_list_mutex);
 736 
 737                 if (ibtl_dm_async_handler)
 738                         ibtl_tell_mgr(hca_devp, ibtl_dm_async_handler,
 739                             ibtl_dm_clnt_private);
 740                 if (ibtl_cm_async_handler)
 741                         ibtl_tell_mgr(hca_devp, ibtl_cm_async_handler,
 742                             ibtl_cm_clnt_private);
 743                 /* wait for all tasks to complete */
 744                 while (hca_devp->hd_async_task_cnt != 0)
 745                         cv_wait(&hca_devp->hd_async_task_cv,
 746                             &ibtl_clnt_list_mutex);
 747 
 748                 for (ibt_hca = hca_devp->hd_clnt_list;
 749                     ibt_hca != NULL;
 750                     ibt_hca = ibt_hca->ha_clnt_link) {
 751 
 752                         /* Managers are handled above */
 753                         if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
 754                             ibtl_cm_async_handler)
 755                                 continue;
 756                         if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
 757                             ibtl_dm_async_handler)
 758                                 continue;
 759                         if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
 760                             ibtl_ibma_async_handler)
 761                                 continue;
 762                         ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
 763 
 764                         mutex_enter(&ibtl_async_mutex);
 765                         ibt_hca->ha_async_cnt++;
 766                         mutex_exit(&ibtl_async_mutex);
 767                         hca_devp->hd_async_task_cnt++;
 768                         (void) taskq_dispatch(ibtl_async_taskq,
 769                             ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
 770                 }
 771 
 772                 /* wait for all tasks to complete */
 773                 while (hca_devp->hd_async_task_cnt != 0)
 774                         cv_wait(&hca_devp->hd_async_task_cv,
 775                             &ibtl_clnt_list_mutex);
 776 
 777                 mutex_enter(&ibtl_async_mutex);
 778         }
 779         hca_devp->hd_async_code = 0;
 780         hca_devp->hd_async_busy = 0;
 781         cv_broadcast(&hca_devp->hd_async_busy_cv);
 782         mutex_exit(&ibtl_clnt_list_mutex);
 783 }
 784 
 785 /*
 786  * Asyncs for QP objects.
 787  *
 788  * The function continues to run until there are no more async
 789  * events/errors for this object.
 790  */
 791 static void
 792 ibtl_do_qp_asyncs(ibtl_qp_t *ibtl_qp)
 793 {
 794         ibt_async_code_t        code;
 795         ibt_async_event_t       async_event;
 796 
 797         ASSERT(MUTEX_HELD(&ibtl_async_mutex));
 798         bzero(&async_event, sizeof (async_event));
 799         async_event.ev_chan_hdl = IBTL_QP2CHAN(ibtl_qp);
 800 
 801         while ((code = ibtl_qp->qp_async_codes) != 0) {
 802                 async_event.ev_fma_ena = 0;
 803                 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_FREE_OBJECT)
 804                         code = 0;       /* fallthrough to "kmem_free" */
 805                 else if (code & IBT_ERROR_CATASTROPHIC_QP) {
 806                         code = IBT_ERROR_CATASTROPHIC_QP;
 807                         async_event.ev_fma_ena = ibtl_qp->qp_cat_fma_ena;
 808                 } else if (code & IBT_ERROR_INVALID_REQUEST_QP) {
 809                         code = IBT_ERROR_INVALID_REQUEST_QP;
 810                         async_event.ev_fma_ena = ibtl_qp->qp_inv_fma_ena;
 811                 } else if (code & IBT_ERROR_ACCESS_VIOLATION_QP) {
 812                         code = IBT_ERROR_ACCESS_VIOLATION_QP;
 813                         async_event.ev_fma_ena = ibtl_qp->qp_acc_fma_ena;
 814                 } else if (code & IBT_ERROR_PATH_MIGRATE_REQ_QP) {
 815                         code = IBT_ERROR_PATH_MIGRATE_REQ_QP;
 816                         async_event.ev_fma_ena = ibtl_qp->qp_pth_fma_ena;
 817                 } else if (code & IBT_EVENT_PATH_MIGRATED_QP)
 818                         code = IBT_EVENT_PATH_MIGRATED_QP;
 819                 else if (code & IBT_EVENT_SQD)
 820                         code = IBT_EVENT_SQD;
 821                 else if (code & IBT_EVENT_COM_EST_QP)
 822                         code = IBT_EVENT_COM_EST_QP;
 823                 else if (code & IBT_EVENT_EMPTY_QP)
 824                         code = IBT_EVENT_EMPTY_QP;
 825                 else {
 826                         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_qp_asyncs: "
 827                             "async: unexpected QP async code 0x%x", code);
 828                         ibtl_qp->qp_async_codes = 0;
 829                         code = 0;
 830                 }
 831                 ibtl_qp->qp_async_codes &= ~code;
 832 
 833                 if (code) {
 834                         mutex_exit(&ibtl_async_mutex);
 835                         ibtl_async_client_call(ibtl_qp->qp_hca,
 836                             code, &async_event);
 837                         mutex_enter(&ibtl_async_mutex);
 838                 }
 839 
 840                 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_FREE_OBJECT) {
 841                         mutex_exit(&ibtl_async_mutex);
 842                         cv_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_cv);
 843                         mutex_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_mutex);
 844                         kmem_free(IBTL_QP2CHAN(ibtl_qp),
 845                             sizeof (ibtl_channel_t));
 846                         mutex_enter(&ibtl_async_mutex);
 847                         return;
 848                 }
 849         }
 850         ibtl_qp->qp_async_flags &= ~IBTL_ASYNC_PENDING;
 851 }
 852 
 853 /*
 854  * Asyncs for SRQ objects.
 855  *
 856  * The function continues to run until there are no more async
 857  * events/errors for this object.
 858  */
 859 static void
 860 ibtl_do_srq_asyncs(ibtl_srq_t *ibtl_srq)
 861 {
 862         ibt_async_code_t        code;
 863         ibt_async_event_t       async_event;
 864 
 865         ASSERT(MUTEX_HELD(&ibtl_async_mutex));
 866         bzero(&async_event, sizeof (async_event));
 867         async_event.ev_srq_hdl = ibtl_srq;
 868         async_event.ev_fma_ena = ibtl_srq->srq_fma_ena;
 869 
 870         while ((code = ibtl_srq->srq_async_codes) != 0) {
 871                 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_FREE_OBJECT)
 872                         code = 0;       /* fallthrough to "kmem_free" */
 873                 else if (code & IBT_ERROR_CATASTROPHIC_SRQ)
 874                         code = IBT_ERROR_CATASTROPHIC_SRQ;
 875                 else if (code & IBT_EVENT_LIMIT_REACHED_SRQ)
 876                         code = IBT_EVENT_LIMIT_REACHED_SRQ;
 877                 else {
 878                         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_srq_asyncs: "
 879                             "async: unexpected SRQ async code 0x%x", code);
 880                         ibtl_srq->srq_async_codes = 0;
 881                         code = 0;
 882                 }
 883                 ibtl_srq->srq_async_codes &= ~code;
 884 
 885                 if (code) {
 886                         mutex_exit(&ibtl_async_mutex);
 887                         ibtl_async_client_call(ibtl_srq->srq_hca,
 888                             code, &async_event);
 889                         mutex_enter(&ibtl_async_mutex);
 890                 }
 891 
 892                 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_FREE_OBJECT) {
 893                         mutex_exit(&ibtl_async_mutex);
 894                         kmem_free(ibtl_srq, sizeof (struct ibtl_srq_s));
 895                         mutex_enter(&ibtl_async_mutex);
 896                         return;
 897                 }
 898         }
 899         ibtl_srq->srq_async_flags &= ~IBTL_ASYNC_PENDING;
 900 }
 901 
 902 /*
 903  * Asyncs for CQ objects.
 904  *
 905  * The function continues to run until there are no more async
 906  * events/errors for this object.
 907  */
 908 static void
 909 ibtl_do_cq_asyncs(ibtl_cq_t *ibtl_cq)
 910 {
 911         ibt_async_code_t        code;
 912         ibt_async_event_t       async_event;
 913 
 914         ASSERT(MUTEX_HELD(&ibtl_async_mutex));
 915         bzero(&async_event, sizeof (async_event));
 916         async_event.ev_cq_hdl = ibtl_cq;
 917         async_event.ev_fma_ena = ibtl_cq->cq_fma_ena;
 918 
 919         while ((code = ibtl_cq->cq_async_codes) != 0) {
 920                 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_FREE_OBJECT)
 921                         code = 0;       /* fallthrough to "kmem_free" */
 922                 else if (code & IBT_ERROR_CQ)
 923                         code = IBT_ERROR_CQ;
 924                 else {
 925                         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_cq_asyncs: "
 926                             "async: unexpected CQ async code 0x%x", code);
 927                         ibtl_cq->cq_async_codes = 0;
 928                         code = 0;
 929                 }
 930                 ibtl_cq->cq_async_codes &= ~code;
 931 
 932                 if (code) {
 933                         mutex_exit(&ibtl_async_mutex);
 934                         ibtl_async_client_call(ibtl_cq->cq_hca,
 935                             code, &async_event);
 936                         mutex_enter(&ibtl_async_mutex);
 937                 }
 938 
 939                 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_FREE_OBJECT) {
 940                         mutex_exit(&ibtl_async_mutex);
 941                         mutex_destroy(&ibtl_cq->cq_mutex);
 942                         kmem_free(ibtl_cq, sizeof (struct ibtl_cq_s));
 943                         mutex_enter(&ibtl_async_mutex);
 944                         return;
 945                 }
 946         }
 947         ibtl_cq->cq_async_flags &= ~IBTL_ASYNC_PENDING;
 948 }
 949 
 950 /*
 951  * Asyncs for EEC objects.
 952  *
 953  * The function continues to run until there are no more async
 954  * events/errors for this object.
 955  */
 956 static void
 957 ibtl_do_eec_asyncs(ibtl_eec_t *ibtl_eec)
 958 {
 959         ibt_async_code_t        code;
 960         ibt_async_event_t       async_event;
 961 
 962         ASSERT(MUTEX_HELD(&ibtl_async_mutex));
 963         bzero(&async_event, sizeof (async_event));
 964         async_event.ev_chan_hdl = ibtl_eec->eec_channel;
 965 
 966         while ((code = ibtl_eec->eec_async_codes) != 0) {
 967                 async_event.ev_fma_ena = 0;
 968                 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_FREE_OBJECT)
 969                         code = 0;       /* fallthrough to "kmem_free" */
 970                 else if (code & IBT_ERROR_CATASTROPHIC_EEC) {
 971                         code = IBT_ERROR_CATASTROPHIC_CHAN;
 972                         async_event.ev_fma_ena = ibtl_eec->eec_cat_fma_ena;
 973                 } else if (code & IBT_ERROR_PATH_MIGRATE_REQ_EEC) {
 974                         code = IBT_ERROR_PATH_MIGRATE_REQ;
 975                         async_event.ev_fma_ena = ibtl_eec->eec_pth_fma_ena;
 976                 } else if (code & IBT_EVENT_PATH_MIGRATED_EEC)
 977                         code = IBT_EVENT_PATH_MIGRATED;
 978                 else if (code & IBT_EVENT_COM_EST_EEC)
 979                         code = IBT_EVENT_COM_EST;
 980                 else {
 981                         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_eec_asyncs: "
 982                             "async: unexpected code 0x%x", code);
 983                         ibtl_eec->eec_async_codes = 0;
 984                         code = 0;
 985                 }
 986                 ibtl_eec->eec_async_codes &= ~code;
 987 
 988                 if (code) {
 989                         mutex_exit(&ibtl_async_mutex);
 990                         ibtl_async_client_call(ibtl_eec->eec_hca,
 991                             code, &async_event);
 992                         mutex_enter(&ibtl_async_mutex);
 993                 }
 994 
 995                 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_FREE_OBJECT) {
 996                         mutex_exit(&ibtl_async_mutex);
 997                         kmem_free(ibtl_eec, sizeof (struct ibtl_eec_s));
 998                         mutex_enter(&ibtl_async_mutex);
 999                         return;
1000                 }
1001         }
1002         ibtl_eec->eec_async_flags &= ~IBTL_ASYNC_PENDING;
1003 }
1004 
1005 /*
1006  * Loop forever, calling async_handlers until all of the async lists
1007  * are empty.
1008  */
1009 
1010 static void
1011 ibtl_async_thread(void)
1012 {
1013         kmutex_t cpr_mutex;
1014         callb_cpr_t     cprinfo;
1015 
1016         mutex_init(&cpr_mutex, NULL, MUTEX_DRIVER, NULL);
1017         CALLB_CPR_INIT(&cprinfo, &cpr_mutex, callb_generic_cpr,
1018             "ibtl_async_thread");
1019 
1020         mutex_enter(&ibtl_async_mutex);
1021 
1022         for (;;) {
1023                 if (ibtl_async_hca_list_start) {
1024                         ibtl_hca_devinfo_t *hca_devp;
1025 
1026                         /* remove first entry from list */
1027                         hca_devp = ibtl_async_hca_list_start;
1028                         ibtl_async_hca_list_start = hca_devp->hd_async_link;
1029                         hca_devp->hd_async_link = NULL;
1030                         if (ibtl_async_hca_list_start == NULL)
1031                                 ibtl_async_hca_list_end = NULL;
1032 
1033                         ibtl_do_hca_asyncs(hca_devp);
1034 
1035                 } else if (ibtl_async_qp_list_start) {
1036                         ibtl_qp_t *ibtl_qp;
1037 
1038                         /* remove from list */
1039                         ibtl_qp = ibtl_async_qp_list_start;
1040                         ibtl_async_qp_list_start = ibtl_qp->qp_async_link;
1041                         ibtl_qp->qp_async_link = NULL;
1042                         if (ibtl_async_qp_list_start == NULL)
1043                                 ibtl_async_qp_list_end = NULL;
1044 
1045                         ibtl_do_qp_asyncs(ibtl_qp);
1046 
1047                 } else if (ibtl_async_srq_list_start) {
1048                         ibtl_srq_t *ibtl_srq;
1049 
1050                         /* remove from list */
1051                         ibtl_srq = ibtl_async_srq_list_start;
1052                         ibtl_async_srq_list_start = ibtl_srq->srq_async_link;
1053                         ibtl_srq->srq_async_link = NULL;
1054                         if (ibtl_async_srq_list_start == NULL)
1055                                 ibtl_async_srq_list_end = NULL;
1056 
1057                         ibtl_do_srq_asyncs(ibtl_srq);
1058 
1059                 } else if (ibtl_async_eec_list_start) {
1060                         ibtl_eec_t *ibtl_eec;
1061 
1062                         /* remove from list */
1063                         ibtl_eec = ibtl_async_eec_list_start;
1064                         ibtl_async_eec_list_start = ibtl_eec->eec_async_link;
1065                         ibtl_eec->eec_async_link = NULL;
1066                         if (ibtl_async_eec_list_start == NULL)
1067                                 ibtl_async_eec_list_end = NULL;
1068 
1069                         ibtl_do_eec_asyncs(ibtl_eec);
1070 
1071                 } else if (ibtl_async_cq_list_start) {
1072                         ibtl_cq_t *ibtl_cq;
1073 
1074                         /* remove from list */
1075                         ibtl_cq = ibtl_async_cq_list_start;
1076                         ibtl_async_cq_list_start = ibtl_cq->cq_async_link;
1077                         ibtl_cq->cq_async_link = NULL;
1078                         if (ibtl_async_cq_list_start == NULL)
1079                                 ibtl_async_cq_list_end = NULL;
1080 
1081                         ibtl_do_cq_asyncs(ibtl_cq);
1082 
1083                 } else {
1084                         if (ibtl_async_thread_exit == IBTL_THREAD_EXIT)
1085                                 break;
1086                         mutex_enter(&cpr_mutex);
1087                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
1088                         mutex_exit(&cpr_mutex);
1089 
1090                         cv_wait(&ibtl_async_cv, &ibtl_async_mutex);
1091 
1092                         mutex_exit(&ibtl_async_mutex);
1093                         mutex_enter(&cpr_mutex);
1094                         CALLB_CPR_SAFE_END(&cprinfo, &cpr_mutex);
1095                         mutex_exit(&cpr_mutex);
1096                         mutex_enter(&ibtl_async_mutex);
1097                 }
1098         }
1099 
1100         mutex_exit(&ibtl_async_mutex);
1101 
1102         mutex_enter(&cpr_mutex);
1103         CALLB_CPR_EXIT(&cprinfo);
1104         mutex_destroy(&cpr_mutex);
1105 }
1106 
1107 
1108 void
1109 ibtl_free_qp_async_check(ibtl_qp_t *ibtl_qp)
1110 {
1111         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_qp_async_check(%p)", ibtl_qp);
1112 
1113         mutex_enter(&ibtl_async_mutex);
1114 
1115         /*
1116          * If there is an active async, mark this object to be freed
1117          * by the async_thread when it's done.
1118          */
1119         if (ibtl_qp->qp_async_flags & IBTL_ASYNC_PENDING) {
1120                 ibtl_qp->qp_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1121                 mutex_exit(&ibtl_async_mutex);
1122         } else {        /* free the object now */
1123                 mutex_exit(&ibtl_async_mutex);
1124                 cv_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_cv);
1125                 mutex_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_mutex);
1126                 kmem_free(IBTL_QP2CHAN(ibtl_qp), sizeof (ibtl_channel_t));
1127         }
1128 }
1129 
1130 void
1131 ibtl_free_cq_async_check(ibtl_cq_t *ibtl_cq)
1132 {
1133         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_cq_async_check(%p)", ibtl_cq);
1134 
1135         mutex_enter(&ibtl_async_mutex);
1136 
1137         /* if there is an active async, mark this object to be freed */
1138         if (ibtl_cq->cq_async_flags & IBTL_ASYNC_PENDING) {
1139                 ibtl_cq->cq_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1140                 mutex_exit(&ibtl_async_mutex);
1141         } else {        /* free the object now */
1142                 mutex_exit(&ibtl_async_mutex);
1143                 mutex_destroy(&ibtl_cq->cq_mutex);
1144                 kmem_free(ibtl_cq, sizeof (struct ibtl_cq_s));
1145         }
1146 }
1147 
1148 void
1149 ibtl_free_srq_async_check(ibtl_srq_t *ibtl_srq)
1150 {
1151         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_srq_async_check(%p)",
1152             ibtl_srq);
1153 
1154         mutex_enter(&ibtl_async_mutex);
1155 
1156         /* if there is an active async, mark this object to be freed */
1157         if (ibtl_srq->srq_async_flags & IBTL_ASYNC_PENDING) {
1158                 ibtl_srq->srq_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1159                 mutex_exit(&ibtl_async_mutex);
1160         } else {        /* free the object now */
1161                 mutex_exit(&ibtl_async_mutex);
1162                 kmem_free(ibtl_srq, sizeof (struct ibtl_srq_s));
1163         }
1164 }
1165 
1166 void
1167 ibtl_free_eec_async_check(ibtl_eec_t *ibtl_eec)
1168 {
1169         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_eec_async_check(%p)",
1170             ibtl_eec);
1171 
1172         mutex_enter(&ibtl_async_mutex);
1173 
1174         /* if there is an active async, mark this object to be freed */
1175         if (ibtl_eec->eec_async_flags & IBTL_ASYNC_PENDING) {
1176                 ibtl_eec->eec_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1177                 mutex_exit(&ibtl_async_mutex);
1178         } else {        /* free the object now */
1179                 mutex_exit(&ibtl_async_mutex);
1180                 kmem_free(ibtl_eec, sizeof (struct ibtl_eec_s));
1181         }
1182 }
1183 
1184 /*
1185  * This function differs from above in that we assume this is called
1186  * from non-interrupt context, and never called from the async_thread.
1187  */
1188 
1189 void
1190 ibtl_free_hca_async_check(ibtl_hca_t *ibt_hca)
1191 {
1192         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_hca_async_check(%p)",
1193             ibt_hca);
1194 
1195         mutex_enter(&ibtl_async_mutex);
1196 
1197         /* if there is an active async, mark this object to be freed */
1198         if (ibt_hca->ha_async_cnt > 0) {
1199                 ibt_hca->ha_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1200                 mutex_exit(&ibtl_async_mutex);
1201         } else {        /* free the object now */
1202                 mutex_exit(&ibtl_async_mutex);
1203                 kmem_free(ibt_hca, sizeof (ibtl_hca_t));
1204         }
1205 }
1206 
1207 /*
1208  * Completion Queue Handling.
1209  *
1210  *      A completion queue can be handled through a simple callback
1211  *      at interrupt level, or it may be queued for an ibtl_cq_thread
1212  *      to handle.  The latter is chosen during ibt_alloc_cq when the
1213  *      IBTF_CQ_HANDLER_IN_THREAD is specified.
1214  */
1215 
1216 static void
1217 ibtl_cq_handler_call(ibtl_cq_t *ibtl_cq)
1218 {
1219         ibt_cq_handler_t        cq_handler;
1220         void                    *arg;
1221 
1222         IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_cq_handler_call(%p)", ibtl_cq);
1223 
1224         cq_handler = ibtl_cq->cq_comp_handler;
1225         arg = ibtl_cq->cq_arg;
1226         if (cq_handler != NULL)
1227                 cq_handler(ibtl_cq, arg);
1228         else
1229                 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_cq_handler_call: "
1230                     "no cq_handler for cq %p", ibtl_cq);
1231 }
1232 
1233 /*
1234  * Before ibt_free_cq can continue, we need to ensure no more cq_handler
1235  * callbacks can occur.  When we get the mutex, we know there are no
1236  * outstanding cq_handler callbacks.  We set the cq_handler to NULL to
1237  * prohibit future callbacks.
1238  */
1239 void
1240 ibtl_free_cq_check(ibtl_cq_t *ibtl_cq)
1241 {
1242         mutex_enter(&ibtl_cq->cq_mutex);
1243         ibtl_cq->cq_comp_handler = NULL;
1244         mutex_exit(&ibtl_cq->cq_mutex);
1245         if (ibtl_cq->cq_in_thread) {
1246                 mutex_enter(&ibtl_cq_mutex);
1247                 --ibtl_cqs_using_threads;
1248                 while (ibtl_cq->cq_impl_flags & IBTL_CQ_PENDING) {
1249                         ibtl_cq->cq_impl_flags &= ~IBTL_CQ_CALL_CLIENT;
1250                         ibtl_cq->cq_impl_flags |= IBTL_CQ_FREE;
1251                         cv_wait(&ibtl_cq_cv, &ibtl_cq_mutex);
1252                 }
1253                 mutex_exit(&ibtl_cq_mutex);
1254         }
1255 }
1256 
1257 /*
1258  * Loop forever, calling cq_handlers until the cq list
1259  * is empty.
1260  */
1261 
1262 static void
1263 ibtl_cq_thread(void)
1264 {
1265         kmutex_t cpr_mutex;
1266         callb_cpr_t     cprinfo;
1267 
1268         mutex_init(&cpr_mutex, NULL, MUTEX_DRIVER, NULL);
1269         CALLB_CPR_INIT(&cprinfo, &cpr_mutex, callb_generic_cpr,
1270             "ibtl_cq_thread");
1271 
1272         mutex_enter(&ibtl_cq_mutex);
1273 
1274         for (;;) {
1275                 if (ibtl_cq_list_start) {
1276                         ibtl_cq_t *ibtl_cq;
1277 
1278                         ibtl_cq = ibtl_cq_list_start;
1279                         ibtl_cq_list_start = ibtl_cq->cq_link;
1280                         ibtl_cq->cq_link = NULL;
1281                         if (ibtl_cq == ibtl_cq_list_end)
1282                                 ibtl_cq_list_end = NULL;
1283 
1284                         while (ibtl_cq->cq_impl_flags & IBTL_CQ_CALL_CLIENT) {
1285                                 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_CALL_CLIENT;
1286                                 mutex_exit(&ibtl_cq_mutex);
1287                                 ibtl_cq_handler_call(ibtl_cq);
1288                                 mutex_enter(&ibtl_cq_mutex);
1289                         }
1290                         ibtl_cq->cq_impl_flags &= ~IBTL_CQ_PENDING;
1291                         if (ibtl_cq->cq_impl_flags & IBTL_CQ_FREE)
1292                                 cv_broadcast(&ibtl_cq_cv);
1293                 } else {
1294                         if (ibtl_cq_thread_exit == IBTL_THREAD_EXIT)
1295                                 break;
1296                         mutex_enter(&cpr_mutex);
1297                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
1298                         mutex_exit(&cpr_mutex);
1299 
1300                         cv_wait(&ibtl_cq_cv, &ibtl_cq_mutex);
1301 
1302                         mutex_exit(&ibtl_cq_mutex);
1303                         mutex_enter(&cpr_mutex);
1304                         CALLB_CPR_SAFE_END(&cprinfo, &cpr_mutex);
1305                         mutex_exit(&cpr_mutex);
1306                         mutex_enter(&ibtl_cq_mutex);
1307                 }
1308         }
1309 
1310         mutex_exit(&ibtl_cq_mutex);
1311         mutex_enter(&cpr_mutex);
1312         CALLB_CPR_EXIT(&cprinfo);
1313         mutex_destroy(&cpr_mutex);
1314 }
1315 
1316 
1317 /*
1318  * ibc_cq_handler()
1319  *
1320  *    Completion Queue Notification Handler.
1321  *
1322  */
1323 /*ARGSUSED*/
1324 void
1325 ibc_cq_handler(ibc_clnt_hdl_t ibc_hdl, ibt_cq_hdl_t ibtl_cq)
1326 {
1327         IBTF_DPRINTF_L4(ibtf_handlers, "ibc_cq_handler(%p, %p)",
1328             ibc_hdl, ibtl_cq);
1329 
1330         if (ibtl_cq->cq_in_thread) {
1331                 mutex_enter(&ibtl_cq_mutex);
1332                 ibtl_cq->cq_impl_flags |= IBTL_CQ_CALL_CLIENT;
1333                 if ((ibtl_cq->cq_impl_flags & IBTL_CQ_PENDING) == 0) {
1334                         ibtl_cq->cq_impl_flags |= IBTL_CQ_PENDING;
1335                         ibtl_cq->cq_link = NULL;
1336                         if (ibtl_cq_list_end == NULL)
1337                                 ibtl_cq_list_start = ibtl_cq;
1338                         else
1339                                 ibtl_cq_list_end->cq_link = ibtl_cq;
1340                         ibtl_cq_list_end = ibtl_cq;
1341                         cv_signal(&ibtl_cq_cv);
1342                 }
1343                 mutex_exit(&ibtl_cq_mutex);
1344                 return;
1345         } else
1346                 ibtl_cq_handler_call(ibtl_cq);
1347 }
1348 
1349 
1350 /*
1351  * ibt_enable_cq_notify()
1352  *      Enable Notification requests on the specified CQ.
1353  *
1354  *      ibt_cq          The CQ handle.
1355  *
1356  *      notify_type     Enable notifications for all (IBT_NEXT_COMPLETION)
1357  *                      completions, or the next Solicited completion
1358  *                      (IBT_NEXT_SOLICITED) only.
1359  *
1360  *      Completion notifications are disabled by setting the completion
1361  *      handler to NULL by calling ibt_set_cq_handler().
1362  */
1363 ibt_status_t
1364 ibt_enable_cq_notify(ibt_cq_hdl_t ibtl_cq, ibt_cq_notify_flags_t notify_type)
1365 {
1366         IBTF_DPRINTF_L3(ibtf_handlers, "ibt_enable_cq_notify(%p, %d)",
1367             ibtl_cq, notify_type);
1368 
1369         return (IBTL_CQ2CIHCAOPS_P(ibtl_cq)->ibc_notify_cq(
1370             IBTL_CQ2CIHCA(ibtl_cq), ibtl_cq->cq_ibc_cq_hdl, notify_type));
1371 }
1372 
1373 
1374 /*
1375  * ibt_set_cq_handler()
1376  *      Register a work request completion handler with the IBTF.
1377  *
1378  *      ibt_cq                  The CQ handle.
1379  *
1380  *      completion_handler      The completion handler.
1381  *
1382  *      arg                     The IBTF client private argument to be passed
1383  *                              back to the client when calling the CQ
1384  *                              completion handler.
1385  *
1386  *      Completion notifications are disabled by setting the completion
1387  *      handler to NULL.  When setting the handler to NULL, no additional
1388  *      calls to the previous CQ handler will be initiated, but there may
1389  *      be one in progress.
1390  *
1391  *      This function does not otherwise change the state of previous
1392  *      calls to ibt_enable_cq_notify().
1393  */
1394 void
1395 ibt_set_cq_handler(ibt_cq_hdl_t ibtl_cq, ibt_cq_handler_t completion_handler,
1396     void *arg)
1397 {
1398         IBTF_DPRINTF_L3(ibtf_handlers, "ibt_set_cq_handler(%p, %p, %p)",
1399             ibtl_cq, completion_handler, arg);
1400 
1401         ibtl_cq->cq_comp_handler = completion_handler;
1402         ibtl_cq->cq_arg = arg;
1403 }
1404 
1405 
1406 /*
1407  * Inform IBT clients about New HCAs.
1408  *
1409  *      We use taskqs to allow simultaneous notification, with sleeping.
1410  *      Since taskqs only allow one argument, we define a structure
1411  *      because we need to pass in two arguments.
1412  */
1413 
1414 struct ibtl_new_hca_s {
1415         ibtl_clnt_t             *nh_clntp;
1416         ibtl_hca_devinfo_t      *nh_hca_devp;
1417         ibt_async_code_t        nh_code;
1418 };
1419 
1420 static void
1421 ibtl_tell_client_about_new_hca(void *arg)
1422 {
1423         struct ibtl_new_hca_s   *new_hcap = (struct ibtl_new_hca_s *)arg;
1424         ibtl_clnt_t             *clntp = new_hcap->nh_clntp;
1425         ibt_async_event_t       async_event;
1426         ibtl_hca_devinfo_t      *hca_devp = new_hcap->nh_hca_devp;
1427 
1428         bzero(&async_event, sizeof (async_event));
1429         async_event.ev_hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
1430         clntp->clnt_modinfop->mi_async_handler(
1431             clntp->clnt_private, NULL, new_hcap->nh_code, &async_event);
1432         kmem_free(new_hcap, sizeof (*new_hcap));
1433         mutex_enter(&ibtl_clnt_list_mutex);
1434         if (--hca_devp->hd_async_task_cnt == 0)
1435                 cv_signal(&hca_devp->hd_async_task_cv);
1436         if (--clntp->clnt_async_cnt == 0)
1437                 cv_broadcast(&ibtl_clnt_cv);
1438         mutex_exit(&ibtl_clnt_list_mutex);
1439 }
1440 
1441 /*
1442  * ibtl_announce_new_hca:
1443  *
1444  *      o First attach these clients in the given order
1445  *              IBMA
1446  *              IBCM
1447  *
1448  *      o Next attach all other clients in parallel.
1449  *
1450  * NOTE: Use the taskq to simultaneously notify all clients of the new HCA.
1451  * Retval from clients is ignored.
1452  */
1453 void
1454 ibtl_announce_new_hca(ibtl_hca_devinfo_t *hca_devp)
1455 {
1456         ibtl_clnt_t             *clntp;
1457         struct ibtl_new_hca_s   *new_hcap;
1458 
1459         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_announce_new_hca(%p, %llX)",
1460             hca_devp, hca_devp->hd_hca_attr->hca_node_guid);
1461 
1462         mutex_enter(&ibtl_clnt_list_mutex);
1463 
1464         clntp = ibtl_clnt_list;
1465         while (clntp != NULL) {
1466                 if (clntp->clnt_modinfop->mi_clnt_class == IBT_IBMA) {
1467                         IBTF_DPRINTF_L4(ibtf_handlers,
1468                             "ibtl_announce_new_hca: calling IBMF");
1469                         if (clntp->clnt_modinfop->mi_async_handler) {
1470                                 new_hcap = kmem_alloc(sizeof (*new_hcap),
1471                                     KM_SLEEP);
1472                                 new_hcap->nh_clntp = clntp;
1473                                 new_hcap->nh_hca_devp = hca_devp;
1474                                 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1475                                 clntp->clnt_async_cnt++;
1476                                 hca_devp->hd_async_task_cnt++;
1477 
1478                                 (void) taskq_dispatch(ibtl_async_taskq,
1479                                     ibtl_tell_client_about_new_hca, new_hcap,
1480                                     TQ_SLEEP);
1481                         }
1482                         break;
1483                 }
1484                 clntp = clntp->clnt_list_link;
1485         }
1486         if (clntp != NULL)
1487                 while (clntp->clnt_async_cnt > 0)
1488                         cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1489         clntp = ibtl_clnt_list;
1490         while (clntp != NULL) {
1491                 if (clntp->clnt_modinfop->mi_clnt_class == IBT_DM) {
1492                         IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_announce_new_hca: "
1493                             "calling  %s", clntp->clnt_modinfop->mi_clnt_name);
1494                         if (clntp->clnt_modinfop->mi_async_handler) {
1495                                 new_hcap = kmem_alloc(sizeof (*new_hcap),
1496                                     KM_SLEEP);
1497                                 new_hcap->nh_clntp = clntp;
1498                                 new_hcap->nh_hca_devp = hca_devp;
1499                                 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1500                                 clntp->clnt_async_cnt++;
1501                                 hca_devp->hd_async_task_cnt++;
1502 
1503                                 mutex_exit(&ibtl_clnt_list_mutex);
1504                                 (void) ibtl_tell_client_about_new_hca(
1505                                     new_hcap);
1506                                 mutex_enter(&ibtl_clnt_list_mutex);
1507                         }
1508                         break;
1509                 }
1510                 clntp = clntp->clnt_list_link;
1511         }
1512 
1513         clntp = ibtl_clnt_list;
1514         while (clntp != NULL) {
1515                 if (clntp->clnt_modinfop->mi_clnt_class == IBT_CM) {
1516                         IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_announce_new_hca: "
1517                             "calling  %s", clntp->clnt_modinfop->mi_clnt_name);
1518                         if (clntp->clnt_modinfop->mi_async_handler) {
1519                                 new_hcap = kmem_alloc(sizeof (*new_hcap),
1520                                     KM_SLEEP);
1521                                 new_hcap->nh_clntp = clntp;
1522                                 new_hcap->nh_hca_devp = hca_devp;
1523                                 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1524                                 clntp->clnt_async_cnt++;
1525                                 hca_devp->hd_async_task_cnt++;
1526 
1527                                 (void) taskq_dispatch(ibtl_async_taskq,
1528                                     ibtl_tell_client_about_new_hca, new_hcap,
1529                                     TQ_SLEEP);
1530                         }
1531                         break;
1532                 }
1533                 clntp = clntp->clnt_list_link;
1534         }
1535         if (clntp != NULL)
1536                 while (clntp->clnt_async_cnt > 0)
1537                         cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1538         clntp = ibtl_clnt_list;
1539         while (clntp != NULL) {
1540                 if ((clntp->clnt_modinfop->mi_clnt_class != IBT_DM) &&
1541                     (clntp->clnt_modinfop->mi_clnt_class != IBT_CM) &&
1542                     (clntp->clnt_modinfop->mi_clnt_class != IBT_IBMA)) {
1543                         IBTF_DPRINTF_L4(ibtf_handlers,
1544                             "ibtl_announce_new_hca: Calling %s ",
1545                             clntp->clnt_modinfop->mi_clnt_name);
1546                         if (clntp->clnt_modinfop->mi_async_handler) {
1547                                 new_hcap = kmem_alloc(sizeof (*new_hcap),
1548                                     KM_SLEEP);
1549                                 new_hcap->nh_clntp = clntp;
1550                                 new_hcap->nh_hca_devp = hca_devp;
1551                                 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1552                                 clntp->clnt_async_cnt++;
1553                                 hca_devp->hd_async_task_cnt++;
1554 
1555                                 (void) taskq_dispatch(ibtl_async_taskq,
1556                                     ibtl_tell_client_about_new_hca, new_hcap,
1557                                     TQ_SLEEP);
1558                         }
1559                 }
1560                 clntp = clntp->clnt_list_link;
1561         }
1562 
1563         /* wait for all tasks to complete */
1564         while (hca_devp->hd_async_task_cnt != 0)
1565                 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1566 
1567         /* wakeup thread that may be waiting to send an HCA async */
1568         ASSERT(hca_devp->hd_async_busy == 1);
1569         hca_devp->hd_async_busy = 0;
1570         cv_broadcast(&hca_devp->hd_async_busy_cv);
1571         mutex_exit(&ibtl_clnt_list_mutex);
1572 }
1573 
1574 /*
1575  * ibtl_detach_all_clients:
1576  *
1577  *      Return value - 0 for Success, 1 for Failure
1578  *
1579  *      o First detach general clients.
1580  *
1581  *      o Next detach these clients
1582  *              IBCM
1583  *              IBDM
1584  *
1585  *      o Finally, detach this client
1586  *              IBMA
1587  */
1588 int
1589 ibtl_detach_all_clients(ibtl_hca_devinfo_t *hca_devp)
1590 {
1591         ib_guid_t               hcaguid = hca_devp->hd_hca_attr->hca_node_guid;
1592         ibtl_hca_t              *ibt_hca;
1593         ibtl_clnt_t             *clntp;
1594         int                     retval;
1595 
1596         IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_detach_all_clients(%llX)",
1597             hcaguid);
1598 
1599         ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex));
1600 
1601         while (hca_devp->hd_async_busy)
1602                 cv_wait(&hca_devp->hd_async_busy_cv, &ibtl_clnt_list_mutex);
1603         hca_devp->hd_async_busy = 1;
1604 
1605         /* First inform general clients asynchronously */
1606         hca_devp->hd_async_event.ev_hca_guid = hcaguid;
1607         hca_devp->hd_async_event.ev_fma_ena = 0;
1608         hca_devp->hd_async_event.ev_chan_hdl = NULL;
1609         hca_devp->hd_async_event.ev_cq_hdl = NULL;
1610         hca_devp->hd_async_code = IBT_HCA_DETACH_EVENT;
1611 
1612         ibt_hca = hca_devp->hd_clnt_list;
1613         while (ibt_hca != NULL) {
1614                 clntp = ibt_hca->ha_clnt_devp;
1615                 if (IBTL_GENERIC_CLIENT(clntp)) {
1616                         ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1617                         mutex_enter(&ibtl_async_mutex);
1618                         ibt_hca->ha_async_cnt++;
1619                         mutex_exit(&ibtl_async_mutex);
1620                         hca_devp->hd_async_task_cnt++;
1621 
1622                         (void) taskq_dispatch(ibtl_async_taskq,
1623                             ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
1624                 }
1625                 ibt_hca = ibt_hca->ha_clnt_link;
1626         }
1627 
1628         /* wait for all clients to complete */
1629         while (hca_devp->hd_async_task_cnt != 0) {
1630                 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1631         }
1632         /* Go thru the clients and check if any have not closed this HCA. */
1633         retval = 0;
1634         ibt_hca = hca_devp->hd_clnt_list;
1635         while (ibt_hca != NULL) {
1636                 clntp = ibt_hca->ha_clnt_devp;
1637                 if (IBTL_GENERIC_CLIENT(clntp)) {
1638                         IBTF_DPRINTF_L2(ibtf_handlers,
1639                             "ibtl_detach_all_clients: "
1640                             "client '%s' failed to close the HCA.",
1641                             ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1642                         retval = 1;
1643                 }
1644                 ibt_hca = ibt_hca->ha_clnt_link;
1645         }
1646         if (retval == 1)
1647                 goto bailout;
1648 
1649         /* Next inform IBDM asynchronously */
1650         ibt_hca = hca_devp->hd_clnt_list;
1651         while (ibt_hca != NULL) {
1652                 clntp = ibt_hca->ha_clnt_devp;
1653                 if (clntp->clnt_modinfop->mi_clnt_class == IBT_DM) {
1654                         ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1655                         mutex_enter(&ibtl_async_mutex);
1656                         ibt_hca->ha_async_cnt++;
1657                         mutex_exit(&ibtl_async_mutex);
1658                         hca_devp->hd_async_task_cnt++;
1659 
1660                         mutex_exit(&ibtl_clnt_list_mutex);
1661                         ibtl_hca_client_async_task(ibt_hca);
1662                         mutex_enter(&ibtl_clnt_list_mutex);
1663                         break;
1664                 }
1665                 ibt_hca = ibt_hca->ha_clnt_link;
1666         }
1667 
1668         /*
1669          * Next inform IBCM.
1670          * As IBCM doesn't perform ibt_open_hca(), IBCM will not be
1671          * accessible via hca_devp->hd_clnt_list.
1672          * ibtl_cm_async_handler will NOT be NULL, if IBCM is registered.
1673          */
1674         if (ibtl_cm_async_handler) {
1675                 ibtl_tell_mgr(hca_devp, ibtl_cm_async_handler,
1676                     ibtl_cm_clnt_private);
1677 
1678                 /* wait for all tasks to complete */
1679                 while (hca_devp->hd_async_task_cnt != 0)
1680                         cv_wait(&hca_devp->hd_async_task_cv,
1681                             &ibtl_clnt_list_mutex);
1682         }
1683 
1684         /* Go thru the clients and check if any have not closed this HCA. */
1685         retval = 0;
1686         ibt_hca = hca_devp->hd_clnt_list;
1687         while (ibt_hca != NULL) {
1688                 clntp = ibt_hca->ha_clnt_devp;
1689                 if (clntp->clnt_modinfop->mi_clnt_class != IBT_IBMA) {
1690                         IBTF_DPRINTF_L2(ibtf_handlers,
1691                             "ibtl_detach_all_clients: "
1692                             "client '%s' failed to close the HCA.",
1693                             ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1694                         retval = 1;
1695                 }
1696                 ibt_hca = ibt_hca->ha_clnt_link;
1697         }
1698         if (retval == 1)
1699                 goto bailout;
1700 
1701         /* Finally, inform IBMA */
1702         ibt_hca = hca_devp->hd_clnt_list;
1703         while (ibt_hca != NULL) {
1704                 clntp = ibt_hca->ha_clnt_devp;
1705                 if (clntp->clnt_modinfop->mi_clnt_class == IBT_IBMA) {
1706                         ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1707                         mutex_enter(&ibtl_async_mutex);
1708                         ibt_hca->ha_async_cnt++;
1709                         mutex_exit(&ibtl_async_mutex);
1710                         hca_devp->hd_async_task_cnt++;
1711 
1712                         (void) taskq_dispatch(ibtl_async_taskq,
1713                             ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
1714                 } else
1715                         IBTF_DPRINTF_L2(ibtf_handlers,
1716                             "ibtl_detach_all_clients: "
1717                             "client '%s' is unexpectedly on the client list",
1718                             ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1719                 ibt_hca = ibt_hca->ha_clnt_link;
1720         }
1721 
1722         /* wait for IBMA to complete */
1723         while (hca_devp->hd_async_task_cnt != 0) {
1724                 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1725         }
1726 
1727         /* Check if this HCA's client list is empty. */
1728         ibt_hca = hca_devp->hd_clnt_list;
1729         if (ibt_hca != NULL) {
1730                 IBTF_DPRINTF_L2(ibtf_handlers,
1731                     "ibtl_detach_all_clients: "
1732                     "client '%s' failed to close the HCA.",
1733                     ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1734                 retval = 1;
1735         } else
1736                 retval = 0;
1737 
1738 bailout:
1739         if (retval) {
1740                 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */
1741                 mutex_exit(&ibtl_clnt_list_mutex);
1742                 ibtl_announce_new_hca(hca_devp);
1743                 mutex_enter(&ibtl_clnt_list_mutex);
1744         } else {
1745                 hca_devp->hd_async_busy = 0;
1746                 cv_broadcast(&hca_devp->hd_async_busy_cv);
1747         }
1748 
1749         return (retval);
1750 }
1751 
1752 void
1753 ibtl_free_clnt_async_check(ibtl_clnt_t *clntp)
1754 {
1755         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_clnt_async_check(%p)", clntp);
1756 
1757         ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex));
1758 
1759         /* wait for all asyncs based on "ibtl_clnt_list" to complete */
1760         while (clntp->clnt_async_cnt != 0) {
1761                 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1762         }
1763 }
1764 
1765 static void
1766 ibtl_dec_clnt_async_cnt(ibtl_clnt_t *clntp)
1767 {
1768         mutex_enter(&ibtl_clnt_list_mutex);
1769         if (--clntp->clnt_async_cnt == 0) {
1770                 cv_broadcast(&ibtl_clnt_cv);
1771         }
1772         mutex_exit(&ibtl_clnt_list_mutex);
1773 }
1774 
1775 static void
1776 ibtl_inc_clnt_async_cnt(ibtl_clnt_t *clntp)
1777 {
1778         mutex_enter(&ibtl_clnt_list_mutex);
1779         ++clntp->clnt_async_cnt;
1780         mutex_exit(&ibtl_clnt_list_mutex);
1781 }
1782 
1783 
1784 /*
1785  * Functions and data structures to inform clients that a notification
1786  * has occurred about Multicast Groups that might interest them.
1787  */
1788 struct ibtl_sm_notice {
1789         ibt_clnt_hdl_t          np_ibt_hdl;
1790         ib_gid_t                np_sgid;
1791         ibt_subnet_event_code_t np_code;
1792         ibt_subnet_event_t      np_event;
1793 };
1794 
1795 static void
1796 ibtl_sm_notice_task(void *arg)
1797 {
1798         struct ibtl_sm_notice *noticep = (struct ibtl_sm_notice *)arg;
1799         ibt_clnt_hdl_t ibt_hdl = noticep->np_ibt_hdl;
1800         ibt_sm_notice_handler_t sm_notice_handler;
1801 
1802         sm_notice_handler = ibt_hdl->clnt_sm_trap_handler;
1803         if (sm_notice_handler != NULL)
1804                 sm_notice_handler(ibt_hdl->clnt_sm_trap_handler_arg,
1805                     noticep->np_sgid, noticep->np_code, &noticep->np_event);
1806         kmem_free(noticep, sizeof (*noticep));
1807         ibtl_dec_clnt_async_cnt(ibt_hdl);
1808 }
1809 
1810 /*
1811  * Inform the client that MCG notices are not working at this time.
1812  */
1813 void
1814 ibtl_cm_sm_notice_init_failure(ibtl_cm_sm_init_fail_t *ifail)
1815 {
1816         ibt_clnt_hdl_t ibt_hdl = ifail->smf_ibt_hdl;
1817         struct ibtl_sm_notice *noticep;
1818         ib_gid_t *sgidp = &ifail->smf_sgid[0];
1819         int i;
1820 
1821         for (i = 0; i < ifail->smf_num_sgids; i++) {
1822                 noticep = kmem_zalloc(sizeof (*noticep), KM_SLEEP);
1823                 noticep->np_ibt_hdl = ibt_hdl;
1824                 noticep->np_sgid = *sgidp++;
1825                 noticep->np_code = IBT_SM_EVENT_UNAVAILABLE;
1826                 ibtl_inc_clnt_async_cnt(ibt_hdl);
1827                 (void) taskq_dispatch(ibtl_async_taskq,
1828                     ibtl_sm_notice_task, noticep, TQ_SLEEP);
1829         }
1830 }
1831 
1832 /*
1833  * Inform all clients of the event.
1834  */
1835 void
1836 ibtl_cm_sm_notice_handler(ib_gid_t sgid, ibt_subnet_event_code_t code,
1837     ibt_subnet_event_t *event)
1838 {
1839         struct ibtl_sm_notice   *noticep;
1840         ibtl_clnt_t             *clntp;
1841 
1842         mutex_enter(&ibtl_clnt_list_mutex);
1843         clntp = ibtl_clnt_list;
1844         while (clntp != NULL) {
1845                 if (clntp->clnt_sm_trap_handler) {
1846                         noticep = kmem_zalloc(sizeof (*noticep), KM_SLEEP);
1847                         noticep->np_ibt_hdl = clntp;
1848                         noticep->np_sgid = sgid;
1849                         noticep->np_code = code;
1850                         noticep->np_event = *event;
1851                         ++clntp->clnt_async_cnt;
1852                         (void) taskq_dispatch(ibtl_async_taskq,
1853                             ibtl_sm_notice_task, noticep, TQ_SLEEP);
1854                 }
1855                 clntp = clntp->clnt_list_link;
1856         }
1857         mutex_exit(&ibtl_clnt_list_mutex);
1858 }
1859 
1860 /*
1861  * Record the handler for this client.
1862  */
1863 void
1864 ibtl_cm_set_sm_notice_handler(ibt_clnt_hdl_t ibt_hdl,
1865     ibt_sm_notice_handler_t sm_notice_handler, void *private)
1866 {
1867         ibt_hdl->clnt_sm_trap_handler = sm_notice_handler;
1868         ibt_hdl->clnt_sm_trap_handler_arg = private;
1869 }
1870 
1871 
1872 /*
1873  * ibtl_another_cq_handler_in_thread()
1874  *
1875  * Conditionally increase the number of cq_threads.
1876  * The number of threads grows, based on the number of cqs using threads.
1877  *
1878  * The table below controls the number of threads as follows:
1879  *
1880  *      Number of CQs   Number of cq_threads
1881  *              0               0
1882  *              1               1
1883  *              2-3             2
1884  *              4-5             3
1885  *              6-9             4
1886  *              10-15           5
1887  *              16-23           6
1888  *              24-31           7
1889  *              32+             8
1890  */
1891 
1892 #define IBTL_CQ_MAXTHREADS 8
1893 static uint8_t ibtl_cq_scaling[IBTL_CQ_MAXTHREADS] = {
1894         1, 2, 4, 6, 10, 16, 24, 32
1895 };
1896 
1897 static kt_did_t ibtl_cq_did[IBTL_CQ_MAXTHREADS];
1898 
1899 void
1900 ibtl_another_cq_handler_in_thread(void)
1901 {
1902         kthread_t *t;
1903         int my_idx;
1904 
1905         mutex_enter(&ibtl_cq_mutex);
1906         if ((ibtl_cq_threads == IBTL_CQ_MAXTHREADS) ||
1907             (++ibtl_cqs_using_threads < ibtl_cq_scaling[ibtl_cq_threads])) {
1908                 mutex_exit(&ibtl_cq_mutex);
1909                 return;
1910         }
1911         my_idx = ibtl_cq_threads++;
1912         mutex_exit(&ibtl_cq_mutex);
1913         t = thread_create(NULL, 0, ibtl_cq_thread, NULL, 0, &p0, TS_RUN,
1914             ibtl_pri - 1);
1915         ibtl_cq_did[my_idx] = t->t_did;      /* save for thread_join() */
1916 }
1917 
1918 void
1919 ibtl_thread_init(void)
1920 {
1921         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_init()");
1922 
1923         mutex_init(&ibtl_async_mutex, NULL, MUTEX_DEFAULT, NULL);
1924         cv_init(&ibtl_async_cv, NULL, CV_DEFAULT, NULL);
1925         cv_init(&ibtl_clnt_cv, NULL, CV_DEFAULT, NULL);
1926 
1927         mutex_init(&ibtl_cq_mutex, NULL, MUTEX_DEFAULT, NULL);
1928         cv_init(&ibtl_cq_cv, NULL, CV_DEFAULT, NULL);
1929 }
1930 
1931 void
1932 ibtl_thread_init2(void)
1933 {
1934         int i;
1935         static int initted = 0;
1936         kthread_t *t;
1937 
1938         mutex_enter(&ibtl_async_mutex);
1939         if (initted == 1) {
1940                 mutex_exit(&ibtl_async_mutex);
1941                 return;
1942         }
1943         initted = 1;
1944         mutex_exit(&ibtl_async_mutex);
1945         ibtl_async_did = kmem_zalloc(ibtl_async_thread_init * sizeof (kt_did_t),
1946             KM_SLEEP);
1947 
1948         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_init2()");
1949 
1950         for (i = 0; i < ibtl_async_thread_init; i++) {
1951                 t = thread_create(NULL, 0, ibtl_async_thread, NULL, 0, &p0,
1952                     TS_RUN, ibtl_pri - 1);
1953                 ibtl_async_did[i] = t->t_did; /* thread_join() */
1954         }
1955         for (i = 0; i < ibtl_cq_threads; i++) {
1956                 t = thread_create(NULL, 0, ibtl_cq_thread, NULL, 0, &p0,
1957                     TS_RUN, ibtl_pri - 1);
1958                 ibtl_cq_did[i] = t->t_did; /* save for thread_join() */
1959         }
1960 }
1961 
1962 void
1963 ibtl_thread_fini(void)
1964 {
1965         int i;
1966 
1967         IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_fini()");
1968 
1969         /* undo the work done by ibtl_thread_init() */
1970 
1971         mutex_enter(&ibtl_cq_mutex);
1972         ibtl_cq_thread_exit = IBTL_THREAD_EXIT;
1973         cv_broadcast(&ibtl_cq_cv);
1974         mutex_exit(&ibtl_cq_mutex);
1975 
1976         mutex_enter(&ibtl_async_mutex);
1977         ibtl_async_thread_exit = IBTL_THREAD_EXIT;
1978         cv_broadcast(&ibtl_async_cv);
1979         mutex_exit(&ibtl_async_mutex);
1980 
1981         for (i = 0; i < ibtl_cq_threads; i++)
1982                 thread_join(ibtl_cq_did[i]);
1983 
1984         if (ibtl_async_did) {
1985                 for (i = 0; i < ibtl_async_thread_init; i++)
1986                         thread_join(ibtl_async_did[i]);
1987 
1988                 kmem_free(ibtl_async_did,
1989                     ibtl_async_thread_init * sizeof (kt_did_t));
1990         }
1991         mutex_destroy(&ibtl_cq_mutex);
1992         cv_destroy(&ibtl_cq_cv);
1993 
1994         mutex_destroy(&ibtl_async_mutex);
1995         cv_destroy(&ibtl_async_cv);
1996         cv_destroy(&ibtl_clnt_cv);
1997 }
1998 
1999 /* ARGSUSED */
2000 ibt_status_t ibtl_dummy_node_info_cb(ib_guid_t hca_guid, uint8_t port,
2001     ib_lid_t lid, ibt_node_info_t *node_info)
2002 {
2003         return (IBT_SUCCESS);
2004 }