1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/ib/ibtl/impl/ibtl.h>
26 #include <sys/ib/ibtl/impl/ibtl_cm.h>
27 #include <sys/taskq.h>
28 #include <sys/disp.h>
29 #include <sys/callb.h>
30 #include <sys/proc.h>
31
32 /*
33 * ibtl_handlers.c
34 */
35
36 /*
37 * What's in this file?
38 *
39 * This file started as an implementation of Asynchronous Event/Error
40 * handling and Completion Queue handling. As the implementation
41 * evolved, code has been added for other ibc_* interfaces (resume,
42 * predetach, etc.) that use the same mechanisms as used for asyncs.
43 *
44 * Async and CQ handling at interrupt level.
45 *
46 * CQ handling is normally done at interrupt level using the CQ callback
47 * handler to call the appropriate IBT Client (owner of the CQ). For
48 * clients that would prefer a fully flexible non-interrupt context to
49 * do their CQ handling, a CQ can be created so that its handler is
50 * called from a non-interrupt thread. CQ handling is done frequently
51 * whereas Async handling is expected to occur very infrequently.
52 *
53 * Async handling is done by marking (or'ing in of an async_code of) the
54 * pertinent IBTL data structure, and then notifying the async_thread(s)
55 * that the data structure has async work to be done. The notification
56 * occurs by linking the data structure through its async_link onto a
57 * list of like data structures and waking up an async_thread. This
58 * list append is not done if there is already async work pending on
59 * this data structure (IBTL_ASYNC_PENDING).
60 *
61 * Async Mutex and CQ Mutex
62 *
63 * The global ibtl_async_mutex is "the" mutex used to control access
64 * to all the data needed by ibc_async_handler. All the threads that
65 * use this mutex are written so that the mutex is held for very short
66 * periods of time, and never held while making calls to functions
67 * that may block.
68 *
69 * The global ibtl_cq_mutex is used similarly by ibc_cq_handler and
70 * the ibtl_cq_thread(s).
71 *
72 * Mutex hierarchy
73 *
74 * The ibtl_clnt_list_mutex is above the ibtl_async_mutex.
75 * ibtl_clnt_list_mutex protects all of the various lists.
76 * The ibtl_async_mutex is below this in the hierarchy.
77 *
78 * The ibtl_cq_mutex is independent of the above mutexes.
79 *
80 * Threads
81 *
82 * There are "ibtl_cq_threads" number of threads created for handling
83 * Completion Queues in threads. If this feature really gets used,
84 * then we will want to do some suitable tuning. Similarly, we may
85 * want to tune the number of "ibtl_async_thread_init".
86 *
87 * The function ibtl_cq_thread is the main loop for handling a CQ in a
88 * thread. There can be multiple threads executing this same code.
89 * The code sleeps when there is no work to be done (list is empty),
90 * otherwise it pulls the first CQ structure off the list and performs
91 * the CQ handler callback to the client. After that returns, a check
92 * is made, and if another ibc_cq_handler call was made for this CQ,
93 * the client is called again.
94 *
95 * The function ibtl_async_thread is the main loop for handling async
96 * events/errors. There can be multiple threads executing this same code.
97 * The code sleeps when there is no work to be done (lists are empty),
98 * otherwise it pulls the first structure off one of the lists and
99 * performs the async callback(s) to the client(s). Note that HCA
100 * async handling is done by calling each of the clients using the HCA.
101 * When the async handling completes, the data structure having the async
102 * event/error is checked for more work before it's considered "done".
103 *
104 * Taskq
105 *
106 * The async_taskq is used here for allowing async handler callbacks to
107 * occur simultaneously to multiple clients of an HCA. This taskq could
108 * be used for other purposes, e.g., if all the async_threads are in
109 * use, but this is deemed as overkill since asyncs should occur rarely.
110 */
111
112 /* Globals */
113 static char ibtf_handlers[] = "ibtl_handlers";
114
115 /* priority for IBTL threads (async, cq, and taskq) */
116 static pri_t ibtl_pri = MAXCLSYSPRI - 1; /* maybe override in /etc/system */
117
118 /* taskq used for HCA asyncs */
119 #define ibtl_async_taskq system_taskq
120
121 /* data for async handling by threads */
122 static kmutex_t ibtl_async_mutex; /* protects most *_async_* data */
123 static kcondvar_t ibtl_async_cv; /* async_threads wait on this */
124 static kcondvar_t ibtl_clnt_cv; /* ibt_detach might wait on this */
125 static void ibtl_dec_clnt_async_cnt(ibtl_clnt_t *clntp);
126 static void ibtl_inc_clnt_async_cnt(ibtl_clnt_t *clntp);
127
128 static kt_did_t *ibtl_async_did; /* for thread_join() */
129 int ibtl_async_thread_init = 4; /* total # of async_threads to create */
130 static int ibtl_async_thread_exit = 0; /* set if/when thread(s) should exit */
131
132 /* async lists for various structures */
133 static ibtl_hca_devinfo_t *ibtl_async_hca_list_start, *ibtl_async_hca_list_end;
134 static ibtl_eec_t *ibtl_async_eec_list_start, *ibtl_async_eec_list_end;
135 static ibtl_qp_t *ibtl_async_qp_list_start, *ibtl_async_qp_list_end;
136 static ibtl_cq_t *ibtl_async_cq_list_start, *ibtl_async_cq_list_end;
137 static ibtl_srq_t *ibtl_async_srq_list_start, *ibtl_async_srq_list_end;
138
139 /* data for CQ completion handling by threads */
140 static kmutex_t ibtl_cq_mutex; /* protects the cv and the list below */
141 static kcondvar_t ibtl_cq_cv;
142 static ibtl_cq_t *ibtl_cq_list_start, *ibtl_cq_list_end;
143
144 static int ibtl_cq_threads = 0; /* total # of cq threads */
145 static int ibtl_cqs_using_threads = 0; /* total # of cqs using threads */
146 static int ibtl_cq_thread_exit = 0; /* set if/when thread(s) should exit */
147
148 /* value used to tell IBTL threads to exit */
149 #define IBTL_THREAD_EXIT 0x1b7fdead /* IBTF DEAD */
150 /* Cisco Topspin Vendor ID for Rereg hack */
151 #define IBT_VENDOR_CISCO 0x05ad
152
153 int ibtl_eec_not_supported = 1;
154
155 char *ibtl_last_client_name; /* may help debugging */
156 typedef ibt_status_t (*ibtl_node_info_cb_t)(ib_guid_t, uint8_t, ib_lid_t,
157 ibt_node_info_t *);
158
159 ibtl_node_info_cb_t ibtl_node_info_cb;
160
161 _NOTE(LOCK_ORDER(ibtl_clnt_list_mutex ibtl_async_mutex))
162
163 void
164 ibtl_cm_set_node_info_cb(ibt_status_t (*node_info_cb)(ib_guid_t, uint8_t,
165 ib_lid_t, ibt_node_info_t *))
166 {
167 mutex_enter(&ibtl_clnt_list_mutex);
168 ibtl_node_info_cb = node_info_cb;
169 mutex_exit(&ibtl_clnt_list_mutex);
170 }
171
172 /*
173 * ibc_async_handler()
174 *
175 * Asynchronous Event/Error Handler.
176 *
177 * This is the function called HCA drivers to post various async
178 * event and errors mention in the IB architecture spec. See
179 * ibtl_types.h for additional details of this.
180 *
181 * This function marks the pertinent IBTF object with the async_code,
182 * and queues the object for handling by an ibtl_async_thread. If
183 * the object is NOT already marked for async processing, it is added
184 * to the associated list for that type of object, and an
185 * ibtl_async_thread is signaled to finish the async work.
186 */
187 void
188 ibc_async_handler(ibc_clnt_hdl_t hca_devp, ibt_async_code_t code,
189 ibc_async_event_t *event_p)
190 {
191 ibtl_qp_t *ibtl_qp;
192 ibtl_cq_t *ibtl_cq;
193 ibtl_srq_t *ibtl_srq;
194 ibtl_eec_t *ibtl_eec;
195 uint8_t port_minus1;
196
197 ibtl_async_port_event_t *portp;
198
199 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler(%p, 0x%x, %p)",
200 hca_devp, code, event_p);
201
202 mutex_enter(&ibtl_async_mutex);
203
204 switch (code) {
205 case IBT_EVENT_PATH_MIGRATED_QP:
206 case IBT_EVENT_SQD:
207 case IBT_ERROR_CATASTROPHIC_QP:
208 case IBT_ERROR_PATH_MIGRATE_REQ_QP:
209 case IBT_EVENT_COM_EST_QP:
210 case IBT_ERROR_INVALID_REQUEST_QP:
211 case IBT_ERROR_ACCESS_VIOLATION_QP:
212 case IBT_EVENT_EMPTY_QP:
213 case IBT_FEXCH_ERROR:
214 ibtl_qp = event_p->ev_qp_hdl;
215 if (ibtl_qp == NULL) {
216 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
217 "bad qp handle");
218 break;
219 }
220 switch (code) {
221 case IBT_ERROR_CATASTROPHIC_QP:
222 ibtl_qp->qp_cat_fma_ena = event_p->ev_fma_ena; break;
223 case IBT_ERROR_PATH_MIGRATE_REQ_QP:
224 ibtl_qp->qp_pth_fma_ena = event_p->ev_fma_ena; break;
225 case IBT_ERROR_INVALID_REQUEST_QP:
226 ibtl_qp->qp_inv_fma_ena = event_p->ev_fma_ena; break;
227 case IBT_ERROR_ACCESS_VIOLATION_QP:
228 ibtl_qp->qp_acc_fma_ena = event_p->ev_fma_ena; break;
229 }
230
231 ibtl_qp->qp_async_codes |= code;
232 if ((ibtl_qp->qp_async_flags & IBTL_ASYNC_PENDING) == 0) {
233 ibtl_qp->qp_async_flags |= IBTL_ASYNC_PENDING;
234 ibtl_qp->qp_async_link = NULL;
235 if (ibtl_async_qp_list_end == NULL)
236 ibtl_async_qp_list_start = ibtl_qp;
237 else
238 ibtl_async_qp_list_end->qp_async_link = ibtl_qp;
239 ibtl_async_qp_list_end = ibtl_qp;
240 cv_signal(&ibtl_async_cv);
241 }
242 break;
243
244 case IBT_ERROR_CQ:
245 ibtl_cq = event_p->ev_cq_hdl;
246 if (ibtl_cq == NULL) {
247 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
248 "bad cq handle");
249 break;
250 }
251 ibtl_cq->cq_async_codes |= code;
252 ibtl_cq->cq_fma_ena = event_p->ev_fma_ena;
253 if ((ibtl_cq->cq_async_flags & IBTL_ASYNC_PENDING) == 0) {
254 ibtl_cq->cq_async_flags |= IBTL_ASYNC_PENDING;
255 ibtl_cq->cq_async_link = NULL;
256 if (ibtl_async_cq_list_end == NULL)
257 ibtl_async_cq_list_start = ibtl_cq;
258 else
259 ibtl_async_cq_list_end->cq_async_link = ibtl_cq;
260 ibtl_async_cq_list_end = ibtl_cq;
261 cv_signal(&ibtl_async_cv);
262 }
263 break;
264
265 case IBT_ERROR_CATASTROPHIC_SRQ:
266 case IBT_EVENT_LIMIT_REACHED_SRQ:
267 ibtl_srq = event_p->ev_srq_hdl;
268 if (ibtl_srq == NULL) {
269 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
270 "bad srq handle");
271 break;
272 }
273 ibtl_srq->srq_async_codes |= code;
274 ibtl_srq->srq_fma_ena = event_p->ev_fma_ena;
275 if ((ibtl_srq->srq_async_flags & IBTL_ASYNC_PENDING) == 0) {
276 ibtl_srq->srq_async_flags |= IBTL_ASYNC_PENDING;
277 ibtl_srq->srq_async_link = NULL;
278 if (ibtl_async_srq_list_end == NULL)
279 ibtl_async_srq_list_start = ibtl_srq;
280 else
281 ibtl_async_srq_list_end->srq_async_link =
282 ibtl_srq;
283 ibtl_async_srq_list_end = ibtl_srq;
284 cv_signal(&ibtl_async_cv);
285 }
286 break;
287
288 case IBT_EVENT_PATH_MIGRATED_EEC:
289 case IBT_ERROR_PATH_MIGRATE_REQ_EEC:
290 case IBT_ERROR_CATASTROPHIC_EEC:
291 case IBT_EVENT_COM_EST_EEC:
292 if (ibtl_eec_not_supported) {
293 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
294 "EEC events are disabled.");
295 break;
296 }
297 ibtl_eec = event_p->ev_eec_hdl;
298 if (ibtl_eec == NULL) {
299 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
300 "bad eec handle");
301 break;
302 }
303 switch (code) {
304 case IBT_ERROR_PATH_MIGRATE_REQ_EEC:
305 ibtl_eec->eec_pth_fma_ena = event_p->ev_fma_ena; break;
306 case IBT_ERROR_CATASTROPHIC_EEC:
307 ibtl_eec->eec_cat_fma_ena = event_p->ev_fma_ena; break;
308 }
309 ibtl_eec->eec_async_codes |= code;
310 if ((ibtl_eec->eec_async_flags & IBTL_ASYNC_PENDING) == 0) {
311 ibtl_eec->eec_async_flags |= IBTL_ASYNC_PENDING;
312 ibtl_eec->eec_async_link = NULL;
313 if (ibtl_async_eec_list_end == NULL)
314 ibtl_async_eec_list_start = ibtl_eec;
315 else
316 ibtl_async_eec_list_end->eec_async_link =
317 ibtl_eec;
318 ibtl_async_eec_list_end = ibtl_eec;
319 cv_signal(&ibtl_async_cv);
320 }
321 break;
322
323 case IBT_ERROR_LOCAL_CATASTROPHIC:
324 hca_devp->hd_async_codes |= code;
325 hca_devp->hd_fma_ena = event_p->ev_fma_ena;
326 /* FALLTHROUGH */
327
328 case IBT_EVENT_PORT_UP:
329 case IBT_PORT_CHANGE_EVENT:
330 case IBT_CLNT_REREG_EVENT:
331 case IBT_ERROR_PORT_DOWN:
332 if ((code & IBT_PORT_EVENTS) != 0) {
333 if ((port_minus1 = event_p->ev_port - 1) >=
334 hca_devp->hd_hca_attr->hca_nports) {
335 IBTF_DPRINTF_L2(ibtf_handlers,
336 "ibc_async_handler: bad port #: %d",
337 event_p->ev_port);
338 break;
339 }
340 portp = &hca_devp->hd_async_port[port_minus1];
341 if (code == IBT_EVENT_PORT_UP) {
342 /*
343 * The port is just coming UP we can't have any
344 * valid older events.
345 */
346 portp->status = IBTL_HCA_PORT_UP;
347 } else if (code == IBT_ERROR_PORT_DOWN) {
348 /*
349 * The port is going DOWN older events don't
350 * count.
351 */
352 portp->status = IBTL_HCA_PORT_DOWN;
353 } else if (code == IBT_PORT_CHANGE_EVENT) {
354 /*
355 * For port UP and DOWN events only the latest
356 * event counts. If we get a UP after DOWN it
357 * is sufficient to send just UP and vice versa.
358 * In the case of port CHANGE event it is valid
359 * only when the port is UP already but if we
360 * receive it after UP but before UP is
361 * delivered we still need to deliver CHANGE
362 * after we deliver UP event.
363 *
364 * We will not get a CHANGE event when the port
365 * is down or DOWN event is pending.
366 */
367 portp->flags |= event_p->ev_port_flags;
368 portp->status |= IBTL_HCA_PORT_CHG;
369 } else if (code == IBT_CLNT_REREG_EVENT) {
370 /*
371 * SM has requested a re-register of
372 * subscription to SM events notification.
373 */
374 portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
375 }
376
377 hca_devp->hd_async_codes |= code;
378 }
379
380 if ((hca_devp->hd_async_flags & IBTL_ASYNC_PENDING) == 0) {
381 hca_devp->hd_async_flags |= IBTL_ASYNC_PENDING;
382 hca_devp->hd_async_link = NULL;
383 if (ibtl_async_hca_list_end == NULL)
384 ibtl_async_hca_list_start = hca_devp;
385 else
386 ibtl_async_hca_list_end->hd_async_link =
387 hca_devp;
388 ibtl_async_hca_list_end = hca_devp;
389 cv_signal(&ibtl_async_cv);
390 }
391
392 break;
393
394 default:
395 IBTF_DPRINTF_L1(ibtf_handlers, "ibc_async_handler: "
396 "invalid code (0x%x)", code);
397 }
398
399 mutex_exit(&ibtl_async_mutex);
400 }
401
402
403 /* Finally, make the async call to the client. */
404
405 static void
406 ibtl_async_client_call(ibtl_hca_t *ibt_hca, ibt_async_code_t code,
407 ibt_async_event_t *event_p)
408 {
409 ibtl_clnt_t *clntp;
410 void *client_private;
411 ibt_async_handler_t async_handler;
412 char *client_name;
413
414 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call(%p, 0x%x, %p)",
415 ibt_hca, code, event_p);
416
417 clntp = ibt_hca->ha_clnt_devp;
418
419 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibtl_last_client_name))
420 /* Record who is being called (just a debugging aid) */
421 ibtl_last_client_name = client_name = clntp->clnt_name;
422 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibtl_last_client_name))
423
424 client_private = clntp->clnt_private;
425 async_handler = clntp->clnt_modinfop->mi_async_handler;
426
427 if (code & (IBT_EVENT_COM_EST_QP | IBT_EVENT_COM_EST_EEC)) {
428 mutex_enter(&ibtl_clnt_list_mutex);
429 async_handler = ibtl_cm_async_handler;
430 client_private = ibtl_cm_clnt_private;
431 mutex_exit(&ibtl_clnt_list_mutex);
432 ibt_hca = NULL;
433 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
434 "calling CM for COM_EST");
435 } else {
436 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
437 "calling client '%s'", client_name);
438 }
439 if (async_handler != NULL)
440 async_handler(client_private, ibt_hca, code, event_p);
441 else
442 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
443 "client '%s' has no async handler", client_name);
444 }
445
446 /*
447 * Inform CM or DM about HCA events.
448 *
449 * We use taskqs to allow simultaneous notification, with sleeping.
450 * Since taskqs only allow one argument, we define a structure
451 * because we need to pass in more than one argument.
452 */
453
454 struct ibtl_mgr_s {
455 ibtl_hca_devinfo_t *mgr_hca_devp;
456 ibt_async_handler_t mgr_async_handler;
457 void *mgr_clnt_private;
458 };
459
460 /*
461 * Asyncs of HCA level events for CM and DM. Call CM or DM and tell them
462 * about the HCA for the event recorded in the ibtl_hca_devinfo_t.
463 */
464 static void
465 ibtl_do_mgr_async_task(void *arg)
466 {
467 struct ibtl_mgr_s *mgrp = (struct ibtl_mgr_s *)arg;
468 ibtl_hca_devinfo_t *hca_devp = mgrp->mgr_hca_devp;
469
470 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_mgr_async_task(0x%x)",
471 hca_devp->hd_async_code);
472
473 mgrp->mgr_async_handler(mgrp->mgr_clnt_private, NULL,
474 hca_devp->hd_async_code, &hca_devp->hd_async_event);
475 kmem_free(mgrp, sizeof (*mgrp));
476
477 mutex_enter(&ibtl_clnt_list_mutex);
478 if (--hca_devp->hd_async_task_cnt == 0)
479 cv_signal(&hca_devp->hd_async_task_cv);
480 mutex_exit(&ibtl_clnt_list_mutex);
481 }
482
483 static void
484 ibt_cisco_embedded_sm_rereg_fix(void *arg)
485 {
486 struct ibtl_mgr_s *mgrp = arg;
487 ibtl_hca_devinfo_t *hca_devp;
488 ibt_node_info_t node_info;
489 ibt_status_t ibt_status;
490 ibtl_async_port_event_t *portp;
491 ib_lid_t sm_lid;
492 ib_guid_t hca_guid;
493 ibt_async_event_t *event_p;
494 ibt_hca_portinfo_t *pinfop;
495 uint8_t port;
496
497 hca_devp = mgrp->mgr_hca_devp;
498
499 mutex_enter(&ibtl_clnt_list_mutex);
500 event_p = &hca_devp->hd_async_event;
501 port = event_p->ev_port;
502 portp = &hca_devp->hd_async_port[port - 1];
503 pinfop = &hca_devp->hd_portinfop[port - 1];
504 sm_lid = pinfop->p_sm_lid;
505 hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
506 mutex_exit(&ibtl_clnt_list_mutex);
507
508 ibt_status = ((ibtl_node_info_cb_t)mgrp->mgr_async_handler)(hca_guid,
509 port, sm_lid, &node_info);
510 if (ibt_status == IBT_SUCCESS) {
511 if ((node_info.n_vendor_id == IBT_VENDOR_CISCO) &&
512 (node_info.n_node_type == IBT_NODE_TYPE_SWITCH)) {
513 mutex_enter(&ibtl_async_mutex);
514 portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
515 hca_devp->hd_async_codes |= IBT_CLNT_REREG_EVENT;
516 mutex_exit(&ibtl_async_mutex);
517 }
518 }
519 kmem_free(mgrp, sizeof (*mgrp));
520
521 mutex_enter(&ibtl_clnt_list_mutex);
522 if (--hca_devp->hd_async_task_cnt == 0)
523 cv_signal(&hca_devp->hd_async_task_cv);
524 mutex_exit(&ibtl_clnt_list_mutex);
525 }
526
527 static void
528 ibtl_cm_get_node_info(ibtl_hca_devinfo_t *hca_devp,
529 ibt_async_handler_t async_handler)
530 {
531 struct ibtl_mgr_s *mgrp;
532
533 if (async_handler == NULL)
534 return;
535
536 _NOTE(NO_COMPETING_THREADS_NOW)
537 mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
538 mgrp->mgr_hca_devp = hca_devp;
539 mgrp->mgr_async_handler = async_handler;
540 mgrp->mgr_clnt_private = NULL;
541 hca_devp->hd_async_task_cnt++;
542
543 (void) taskq_dispatch(ibtl_async_taskq,
544 ibt_cisco_embedded_sm_rereg_fix, mgrp, TQ_SLEEP);
545 #ifndef lint
546 _NOTE(COMPETING_THREADS_NOW)
547 #endif
548 }
549
550 static void
551 ibtl_tell_mgr(ibtl_hca_devinfo_t *hca_devp, ibt_async_handler_t async_handler,
552 void *clnt_private)
553 {
554 struct ibtl_mgr_s *mgrp;
555
556 if (async_handler == NULL)
557 return;
558
559 _NOTE(NO_COMPETING_THREADS_NOW)
560 mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
561 mgrp->mgr_hca_devp = hca_devp;
562 mgrp->mgr_async_handler = async_handler;
563 mgrp->mgr_clnt_private = clnt_private;
564 hca_devp->hd_async_task_cnt++;
565
566 (void) taskq_dispatch(ibtl_async_taskq, ibtl_do_mgr_async_task, mgrp,
567 TQ_SLEEP);
568 #ifndef lint
569 _NOTE(COMPETING_THREADS_NOW)
570 #endif
571 }
572
573 /*
574 * Per client-device asyncs for HCA level events. Call each client that is
575 * using the HCA for the event recorded in the ibtl_hca_devinfo_t.
576 */
577 static void
578 ibtl_hca_client_async_task(void *arg)
579 {
580 ibtl_hca_t *ibt_hca = (ibtl_hca_t *)arg;
581 ibtl_hca_devinfo_t *hca_devp = ibt_hca->ha_hca_devp;
582 ibtl_clnt_t *clntp = ibt_hca->ha_clnt_devp;
583 ibt_async_event_t async_event;
584
585 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_hca_client_async_task(%p, 0x%x)",
586 ibt_hca, hca_devp->hd_async_code);
587
588 bcopy(&hca_devp->hd_async_event, &async_event, sizeof (async_event));
589 ibtl_async_client_call(ibt_hca, hca_devp->hd_async_code, &async_event);
590
591 mutex_enter(&ibtl_async_mutex);
592 if (--ibt_hca->ha_async_cnt == 0 &&
593 (ibt_hca->ha_async_flags & IBTL_ASYNC_FREE_OBJECT)) {
594 mutex_exit(&ibtl_async_mutex);
595 kmem_free(ibt_hca, sizeof (ibtl_hca_t));
596 } else
597 mutex_exit(&ibtl_async_mutex);
598
599 mutex_enter(&ibtl_clnt_list_mutex);
600 if (--hca_devp->hd_async_task_cnt == 0)
601 cv_signal(&hca_devp->hd_async_task_cv);
602 if (--clntp->clnt_async_cnt == 0)
603 cv_broadcast(&ibtl_clnt_cv);
604
605 mutex_exit(&ibtl_clnt_list_mutex);
606 }
607
608 /*
609 * Asyncs for HCA level events.
610 *
611 * The function continues to run until there are no more async
612 * events/errors for this HCA. An event is chosen for dispatch
613 * to all clients of this HCA. This thread dispatches them via
614 * the ibtl_async_taskq, then sleeps until all tasks are done.
615 *
616 * This thread records the async_code and async_event in the
617 * ibtl_hca_devinfo_t for all client taskq threads to reference.
618 *
619 * This is called from an async or taskq thread with ibtl_async_mutex held.
620 */
621 static void
622 ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
623 {
624 ibtl_hca_t *ibt_hca;
625 ibt_async_event_t *eventp;
626 ibt_async_code_t code;
627 ibtl_async_port_status_t temp;
628 uint8_t nports;
629 uint8_t port_minus1;
630 ibtl_async_port_event_t *portp;
631
632 mutex_exit(&ibtl_async_mutex);
633
634 mutex_enter(&ibtl_clnt_list_mutex);
635 while (hca_devp->hd_async_busy)
636 cv_wait(&hca_devp->hd_async_busy_cv, &ibtl_clnt_list_mutex);
637 hca_devp->hd_async_busy = 1;
638 mutex_enter(&ibtl_async_mutex);
639
640 bzero(&hca_devp->hd_async_event, sizeof (hca_devp->hd_async_event));
641 for (;;) {
642
643 hca_devp->hd_async_event.ev_fma_ena = 0;
644
645 code = hca_devp->hd_async_codes;
646 if (code & IBT_ERROR_LOCAL_CATASTROPHIC) {
647 code = IBT_ERROR_LOCAL_CATASTROPHIC;
648 hca_devp->hd_async_event.ev_fma_ena =
649 hca_devp->hd_fma_ena;
650 } else if (code & IBT_ERROR_PORT_DOWN) {
651 code = IBT_ERROR_PORT_DOWN;
652 temp = IBTL_HCA_PORT_DOWN;
653 } else if (code & IBT_EVENT_PORT_UP) {
654 code = IBT_EVENT_PORT_UP;
655 temp = IBTL_HCA_PORT_UP;
656 } else if (code & IBT_PORT_CHANGE_EVENT) {
657 code = IBT_PORT_CHANGE_EVENT;
658 temp = IBTL_HCA_PORT_CHG;
659 } else if (code & IBT_CLNT_REREG_EVENT) {
660 code = IBT_CLNT_REREG_EVENT;
661 temp = IBTL_HCA_PORT_ASYNC_CLNT_REREG;
662 } else {
663 hca_devp->hd_async_codes = 0;
664 code = 0;
665 }
666
667 if (code == 0) {
668 hca_devp->hd_async_flags &= ~IBTL_ASYNC_PENDING;
669 break;
670 }
671 hca_devp->hd_async_codes &= ~code;
672
673 /* PORT_UP, PORT_CHANGE, PORT_DOWN or ASYNC_REREG */
674 if ((code & IBT_PORT_EVENTS) != 0) {
675 portp = hca_devp->hd_async_port;
676 nports = hca_devp->hd_hca_attr->hca_nports;
677 for (port_minus1 = 0; port_minus1 < nports;
678 port_minus1++) {
679 /*
680 * Matching event in this port, let's go handle
681 * it.
682 */
683 if ((portp[port_minus1].status & temp) != 0)
684 break;
685 }
686 if (port_minus1 >= nports) {
687 /* we checked again, but found nothing */
688 continue;
689 }
690 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_do_hca_asyncs: "
691 "async: port# %x code %x", port_minus1 + 1, code);
692 /* mark it to check for other ports after we're done */
693 hca_devp->hd_async_codes |= code;
694
695 /*
696 * Copy the event information into hca_devp and clear
697 * event information from the per port data.
698 */
699 hca_devp->hd_async_event.ev_port = port_minus1 + 1;
700 if (temp == IBTL_HCA_PORT_CHG) {
701 hca_devp->hd_async_event.ev_port_flags =
702 hca_devp->hd_async_port[port_minus1].flags;
703 hca_devp->hd_async_port[port_minus1].flags = 0;
704 }
705 hca_devp->hd_async_port[port_minus1].status &= ~temp;
706
707 mutex_exit(&ibtl_async_mutex);
708 ibtl_reinit_hca_portinfo(hca_devp, port_minus1 + 1);
709 mutex_enter(&ibtl_async_mutex);
710 eventp = &hca_devp->hd_async_event;
711 eventp->ev_hca_guid =
712 hca_devp->hd_hca_attr->hca_node_guid;
713 }
714
715 hca_devp->hd_async_code = code;
716 hca_devp->hd_async_event.ev_hca_guid =
717 hca_devp->hd_hca_attr->hca_node_guid;
718 mutex_exit(&ibtl_async_mutex);
719
720 /*
721 * Make sure to inform CM, DM, and IBMA if we know of them.
722 * Also, make sure not to inform them a second time, which
723 * would occur if they have the HCA open.
724 */
725
726 if (ibtl_ibma_async_handler)
727 ibtl_tell_mgr(hca_devp, ibtl_ibma_async_handler,
728 ibtl_ibma_clnt_private);
729 /* wait for all tasks to complete */
730 while (hca_devp->hd_async_task_cnt != 0)
731 cv_wait(&hca_devp->hd_async_task_cv,
732 &ibtl_clnt_list_mutex);
733
734 /*
735 * Hack Alert:
736 * The ibmf handler would have updated the Master SM LID if it
737 * was SM LID change event. Now lets check if the new Master SM
738 * is a Embedded Cisco Topspin SM.
739 */
740 if ((code == IBT_PORT_CHANGE_EVENT) &&
741 eventp->ev_port_flags & IBT_PORT_CHANGE_SM_LID)
742 ibtl_cm_get_node_info(hca_devp,
743 (ibt_async_handler_t)ibtl_node_info_cb);
744 /* wait for node info task to complete */
745 while (hca_devp->hd_async_task_cnt != 0)
746 cv_wait(&hca_devp->hd_async_task_cv,
747 &ibtl_clnt_list_mutex);
748
749 if (ibtl_dm_async_handler)
750 ibtl_tell_mgr(hca_devp, ibtl_dm_async_handler,
751 ibtl_dm_clnt_private);
752 if (ibtl_cm_async_handler)
753 ibtl_tell_mgr(hca_devp, ibtl_cm_async_handler,
754 ibtl_cm_clnt_private);
755 /* wait for all tasks to complete */
756 while (hca_devp->hd_async_task_cnt != 0)
757 cv_wait(&hca_devp->hd_async_task_cv,
758 &ibtl_clnt_list_mutex);
759
760 for (ibt_hca = hca_devp->hd_clnt_list;
761 ibt_hca != NULL;
762 ibt_hca = ibt_hca->ha_clnt_link) {
763
764 /* Managers are handled above */
765 if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
766 ibtl_cm_async_handler)
767 continue;
768 if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
769 ibtl_dm_async_handler)
770 continue;
771 if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
772 ibtl_ibma_async_handler)
773 continue;
774 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
775
776 mutex_enter(&ibtl_async_mutex);
777 ibt_hca->ha_async_cnt++;
778 mutex_exit(&ibtl_async_mutex);
779 hca_devp->hd_async_task_cnt++;
780 (void) taskq_dispatch(ibtl_async_taskq,
781 ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
782 }
783
784 /* wait for all tasks to complete */
785 while (hca_devp->hd_async_task_cnt != 0)
786 cv_wait(&hca_devp->hd_async_task_cv,
787 &ibtl_clnt_list_mutex);
788
789 mutex_enter(&ibtl_async_mutex);
790 }
791 hca_devp->hd_async_code = 0;
792 hca_devp->hd_async_busy = 0;
793 cv_broadcast(&hca_devp->hd_async_busy_cv);
794 mutex_exit(&ibtl_clnt_list_mutex);
795 }
796
797 /*
798 * Asyncs for QP objects.
799 *
800 * The function continues to run until there are no more async
801 * events/errors for this object.
802 */
803 static void
804 ibtl_do_qp_asyncs(ibtl_qp_t *ibtl_qp)
805 {
806 ibt_async_code_t code;
807 ibt_async_event_t async_event;
808
809 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
810 bzero(&async_event, sizeof (async_event));
811 async_event.ev_chan_hdl = IBTL_QP2CHAN(ibtl_qp);
812
813 while ((code = ibtl_qp->qp_async_codes) != 0) {
814 async_event.ev_fma_ena = 0;
815 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_FREE_OBJECT)
816 code = 0; /* fallthrough to "kmem_free" */
817 else if (code & IBT_ERROR_CATASTROPHIC_QP) {
818 code = IBT_ERROR_CATASTROPHIC_QP;
819 async_event.ev_fma_ena = ibtl_qp->qp_cat_fma_ena;
820 } else if (code & IBT_ERROR_INVALID_REQUEST_QP) {
821 code = IBT_ERROR_INVALID_REQUEST_QP;
822 async_event.ev_fma_ena = ibtl_qp->qp_inv_fma_ena;
823 } else if (code & IBT_ERROR_ACCESS_VIOLATION_QP) {
824 code = IBT_ERROR_ACCESS_VIOLATION_QP;
825 async_event.ev_fma_ena = ibtl_qp->qp_acc_fma_ena;
826 } else if (code & IBT_ERROR_PATH_MIGRATE_REQ_QP) {
827 code = IBT_ERROR_PATH_MIGRATE_REQ_QP;
828 async_event.ev_fma_ena = ibtl_qp->qp_pth_fma_ena;
829 } else if (code & IBT_EVENT_PATH_MIGRATED_QP)
830 code = IBT_EVENT_PATH_MIGRATED_QP;
831 else if (code & IBT_EVENT_SQD)
832 code = IBT_EVENT_SQD;
833 else if (code & IBT_EVENT_COM_EST_QP)
834 code = IBT_EVENT_COM_EST_QP;
835 else if (code & IBT_EVENT_EMPTY_QP)
836 code = IBT_EVENT_EMPTY_QP;
837 else {
838 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_qp_asyncs: "
839 "async: unexpected QP async code 0x%x", code);
840 ibtl_qp->qp_async_codes = 0;
841 code = 0;
842 }
843 ibtl_qp->qp_async_codes &= ~code;
844
845 if (code) {
846 mutex_exit(&ibtl_async_mutex);
847 ibtl_async_client_call(ibtl_qp->qp_hca,
848 code, &async_event);
849 mutex_enter(&ibtl_async_mutex);
850 }
851
852 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_FREE_OBJECT) {
853 mutex_exit(&ibtl_async_mutex);
854 cv_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_cv);
855 mutex_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_mutex);
856 kmem_free(IBTL_QP2CHAN(ibtl_qp),
857 sizeof (ibtl_channel_t));
858 mutex_enter(&ibtl_async_mutex);
859 return;
860 }
861 }
862 ibtl_qp->qp_async_flags &= ~IBTL_ASYNC_PENDING;
863 }
864
865 /*
866 * Asyncs for SRQ objects.
867 *
868 * The function continues to run until there are no more async
869 * events/errors for this object.
870 */
871 static void
872 ibtl_do_srq_asyncs(ibtl_srq_t *ibtl_srq)
873 {
874 ibt_async_code_t code;
875 ibt_async_event_t async_event;
876
877 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
878 bzero(&async_event, sizeof (async_event));
879 async_event.ev_srq_hdl = ibtl_srq;
880 async_event.ev_fma_ena = ibtl_srq->srq_fma_ena;
881
882 while ((code = ibtl_srq->srq_async_codes) != 0) {
883 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_FREE_OBJECT)
884 code = 0; /* fallthrough to "kmem_free" */
885 else if (code & IBT_ERROR_CATASTROPHIC_SRQ)
886 code = IBT_ERROR_CATASTROPHIC_SRQ;
887 else if (code & IBT_EVENT_LIMIT_REACHED_SRQ)
888 code = IBT_EVENT_LIMIT_REACHED_SRQ;
889 else {
890 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_srq_asyncs: "
891 "async: unexpected SRQ async code 0x%x", code);
892 ibtl_srq->srq_async_codes = 0;
893 code = 0;
894 }
895 ibtl_srq->srq_async_codes &= ~code;
896
897 if (code) {
898 mutex_exit(&ibtl_async_mutex);
899 ibtl_async_client_call(ibtl_srq->srq_hca,
900 code, &async_event);
901 mutex_enter(&ibtl_async_mutex);
902 }
903
904 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_FREE_OBJECT) {
905 mutex_exit(&ibtl_async_mutex);
906 kmem_free(ibtl_srq, sizeof (struct ibtl_srq_s));
907 mutex_enter(&ibtl_async_mutex);
908 return;
909 }
910 }
911 ibtl_srq->srq_async_flags &= ~IBTL_ASYNC_PENDING;
912 }
913
914 /*
915 * Asyncs for CQ objects.
916 *
917 * The function continues to run until there are no more async
918 * events/errors for this object.
919 */
920 static void
921 ibtl_do_cq_asyncs(ibtl_cq_t *ibtl_cq)
922 {
923 ibt_async_code_t code;
924 ibt_async_event_t async_event;
925
926 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
927 bzero(&async_event, sizeof (async_event));
928 async_event.ev_cq_hdl = ibtl_cq;
929 async_event.ev_fma_ena = ibtl_cq->cq_fma_ena;
930
931 while ((code = ibtl_cq->cq_async_codes) != 0) {
932 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_FREE_OBJECT)
933 code = 0; /* fallthrough to "kmem_free" */
934 else if (code & IBT_ERROR_CQ)
935 code = IBT_ERROR_CQ;
936 else {
937 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_cq_asyncs: "
938 "async: unexpected CQ async code 0x%x", code);
939 ibtl_cq->cq_async_codes = 0;
940 code = 0;
941 }
942 ibtl_cq->cq_async_codes &= ~code;
943
944 if (code) {
945 mutex_exit(&ibtl_async_mutex);
946 ibtl_async_client_call(ibtl_cq->cq_hca,
947 code, &async_event);
948 mutex_enter(&ibtl_async_mutex);
949 }
950
951 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_FREE_OBJECT) {
952 mutex_exit(&ibtl_async_mutex);
953 mutex_destroy(&ibtl_cq->cq_mutex);
954 kmem_free(ibtl_cq, sizeof (struct ibtl_cq_s));
955 mutex_enter(&ibtl_async_mutex);
956 return;
957 }
958 }
959 ibtl_cq->cq_async_flags &= ~IBTL_ASYNC_PENDING;
960 }
961
962 /*
963 * Asyncs for EEC objects.
964 *
965 * The function continues to run until there are no more async
966 * events/errors for this object.
967 */
968 static void
969 ibtl_do_eec_asyncs(ibtl_eec_t *ibtl_eec)
970 {
971 ibt_async_code_t code;
972 ibt_async_event_t async_event;
973
974 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
975 bzero(&async_event, sizeof (async_event));
976 async_event.ev_chan_hdl = ibtl_eec->eec_channel;
977
978 while ((code = ibtl_eec->eec_async_codes) != 0) {
979 async_event.ev_fma_ena = 0;
980 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_FREE_OBJECT)
981 code = 0; /* fallthrough to "kmem_free" */
982 else if (code & IBT_ERROR_CATASTROPHIC_EEC) {
983 code = IBT_ERROR_CATASTROPHIC_CHAN;
984 async_event.ev_fma_ena = ibtl_eec->eec_cat_fma_ena;
985 } else if (code & IBT_ERROR_PATH_MIGRATE_REQ_EEC) {
986 code = IBT_ERROR_PATH_MIGRATE_REQ;
987 async_event.ev_fma_ena = ibtl_eec->eec_pth_fma_ena;
988 } else if (code & IBT_EVENT_PATH_MIGRATED_EEC)
989 code = IBT_EVENT_PATH_MIGRATED;
990 else if (code & IBT_EVENT_COM_EST_EEC)
991 code = IBT_EVENT_COM_EST;
992 else {
993 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_eec_asyncs: "
994 "async: unexpected code 0x%x", code);
995 ibtl_eec->eec_async_codes = 0;
996 code = 0;
997 }
998 ibtl_eec->eec_async_codes &= ~code;
999
1000 if (code) {
1001 mutex_exit(&ibtl_async_mutex);
1002 ibtl_async_client_call(ibtl_eec->eec_hca,
1003 code, &async_event);
1004 mutex_enter(&ibtl_async_mutex);
1005 }
1006
1007 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_FREE_OBJECT) {
1008 mutex_exit(&ibtl_async_mutex);
1009 kmem_free(ibtl_eec, sizeof (struct ibtl_eec_s));
1010 mutex_enter(&ibtl_async_mutex);
1011 return;
1012 }
1013 }
1014 ibtl_eec->eec_async_flags &= ~IBTL_ASYNC_PENDING;
1015 }
1016
1017 #ifdef __lock_lint
1018 kmutex_t cpr_mutex;
1019 #endif
1020
1021 /*
1022 * Loop forever, calling async_handlers until all of the async lists
1023 * are empty.
1024 */
1025
1026 static void
1027 ibtl_async_thread(void)
1028 {
1029 #ifndef __lock_lint
1030 kmutex_t cpr_mutex;
1031 #endif
1032 callb_cpr_t cprinfo;
1033
1034 _NOTE(MUTEX_PROTECTS_DATA(cpr_mutex, cprinfo))
1035 _NOTE(NO_COMPETING_THREADS_NOW)
1036 mutex_init(&cpr_mutex, NULL, MUTEX_DRIVER, NULL);
1037 CALLB_CPR_INIT(&cprinfo, &cpr_mutex, callb_generic_cpr,
1038 "ibtl_async_thread");
1039 #ifndef lint
1040 _NOTE(COMPETING_THREADS_NOW)
1041 #endif
1042
1043 mutex_enter(&ibtl_async_mutex);
1044
1045 for (;;) {
1046 if (ibtl_async_hca_list_start) {
1047 ibtl_hca_devinfo_t *hca_devp;
1048
1049 /* remove first entry from list */
1050 hca_devp = ibtl_async_hca_list_start;
1051 ibtl_async_hca_list_start = hca_devp->hd_async_link;
1052 hca_devp->hd_async_link = NULL;
1053 if (ibtl_async_hca_list_start == NULL)
1054 ibtl_async_hca_list_end = NULL;
1055
1056 ibtl_do_hca_asyncs(hca_devp);
1057
1058 } else if (ibtl_async_qp_list_start) {
1059 ibtl_qp_t *ibtl_qp;
1060
1061 /* remove from list */
1062 ibtl_qp = ibtl_async_qp_list_start;
1063 ibtl_async_qp_list_start = ibtl_qp->qp_async_link;
1064 ibtl_qp->qp_async_link = NULL;
1065 if (ibtl_async_qp_list_start == NULL)
1066 ibtl_async_qp_list_end = NULL;
1067
1068 ibtl_do_qp_asyncs(ibtl_qp);
1069
1070 } else if (ibtl_async_srq_list_start) {
1071 ibtl_srq_t *ibtl_srq;
1072
1073 /* remove from list */
1074 ibtl_srq = ibtl_async_srq_list_start;
1075 ibtl_async_srq_list_start = ibtl_srq->srq_async_link;
1076 ibtl_srq->srq_async_link = NULL;
1077 if (ibtl_async_srq_list_start == NULL)
1078 ibtl_async_srq_list_end = NULL;
1079
1080 ibtl_do_srq_asyncs(ibtl_srq);
1081
1082 } else if (ibtl_async_eec_list_start) {
1083 ibtl_eec_t *ibtl_eec;
1084
1085 /* remove from list */
1086 ibtl_eec = ibtl_async_eec_list_start;
1087 ibtl_async_eec_list_start = ibtl_eec->eec_async_link;
1088 ibtl_eec->eec_async_link = NULL;
1089 if (ibtl_async_eec_list_start == NULL)
1090 ibtl_async_eec_list_end = NULL;
1091
1092 ibtl_do_eec_asyncs(ibtl_eec);
1093
1094 } else if (ibtl_async_cq_list_start) {
1095 ibtl_cq_t *ibtl_cq;
1096
1097 /* remove from list */
1098 ibtl_cq = ibtl_async_cq_list_start;
1099 ibtl_async_cq_list_start = ibtl_cq->cq_async_link;
1100 ibtl_cq->cq_async_link = NULL;
1101 if (ibtl_async_cq_list_start == NULL)
1102 ibtl_async_cq_list_end = NULL;
1103
1104 ibtl_do_cq_asyncs(ibtl_cq);
1105
1106 } else {
1107 if (ibtl_async_thread_exit == IBTL_THREAD_EXIT)
1108 break;
1109 mutex_enter(&cpr_mutex);
1110 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1111 mutex_exit(&cpr_mutex);
1112
1113 cv_wait(&ibtl_async_cv, &ibtl_async_mutex);
1114
1115 mutex_exit(&ibtl_async_mutex);
1116 mutex_enter(&cpr_mutex);
1117 CALLB_CPR_SAFE_END(&cprinfo, &cpr_mutex);
1118 mutex_exit(&cpr_mutex);
1119 mutex_enter(&ibtl_async_mutex);
1120 }
1121 }
1122
1123 mutex_exit(&ibtl_async_mutex);
1124
1125 #ifndef __lock_lint
1126 mutex_enter(&cpr_mutex);
1127 CALLB_CPR_EXIT(&cprinfo);
1128 #endif
1129 mutex_destroy(&cpr_mutex);
1130 }
1131
1132
1133 void
1134 ibtl_free_qp_async_check(ibtl_qp_t *ibtl_qp)
1135 {
1136 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_qp_async_check(%p)", ibtl_qp);
1137
1138 mutex_enter(&ibtl_async_mutex);
1139
1140 /*
1141 * If there is an active async, mark this object to be freed
1142 * by the async_thread when it's done.
1143 */
1144 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_PENDING) {
1145 ibtl_qp->qp_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1146 mutex_exit(&ibtl_async_mutex);
1147 } else { /* free the object now */
1148 mutex_exit(&ibtl_async_mutex);
1149 cv_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_cv);
1150 mutex_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_mutex);
1151 kmem_free(IBTL_QP2CHAN(ibtl_qp), sizeof (ibtl_channel_t));
1152 }
1153 }
1154
1155 void
1156 ibtl_free_cq_async_check(ibtl_cq_t *ibtl_cq)
1157 {
1158 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_cq_async_check(%p)", ibtl_cq);
1159
1160 mutex_enter(&ibtl_async_mutex);
1161
1162 /* if there is an active async, mark this object to be freed */
1163 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_PENDING) {
1164 ibtl_cq->cq_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1165 mutex_exit(&ibtl_async_mutex);
1166 } else { /* free the object now */
1167 mutex_exit(&ibtl_async_mutex);
1168 mutex_destroy(&ibtl_cq->cq_mutex);
1169 kmem_free(ibtl_cq, sizeof (struct ibtl_cq_s));
1170 }
1171 }
1172
1173 void
1174 ibtl_free_srq_async_check(ibtl_srq_t *ibtl_srq)
1175 {
1176 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_srq_async_check(%p)",
1177 ibtl_srq);
1178
1179 mutex_enter(&ibtl_async_mutex);
1180
1181 /* if there is an active async, mark this object to be freed */
1182 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_PENDING) {
1183 ibtl_srq->srq_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1184 mutex_exit(&ibtl_async_mutex);
1185 } else { /* free the object now */
1186 mutex_exit(&ibtl_async_mutex);
1187 kmem_free(ibtl_srq, sizeof (struct ibtl_srq_s));
1188 }
1189 }
1190
1191 void
1192 ibtl_free_eec_async_check(ibtl_eec_t *ibtl_eec)
1193 {
1194 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_eec_async_check(%p)",
1195 ibtl_eec);
1196
1197 mutex_enter(&ibtl_async_mutex);
1198
1199 /* if there is an active async, mark this object to be freed */
1200 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_PENDING) {
1201 ibtl_eec->eec_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1202 mutex_exit(&ibtl_async_mutex);
1203 } else { /* free the object now */
1204 mutex_exit(&ibtl_async_mutex);
1205 kmem_free(ibtl_eec, sizeof (struct ibtl_eec_s));
1206 }
1207 }
1208
1209 /*
1210 * This function differs from above in that we assume this is called
1211 * from non-interrupt context, and never called from the async_thread.
1212 */
1213
1214 void
1215 ibtl_free_hca_async_check(ibtl_hca_t *ibt_hca)
1216 {
1217 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_hca_async_check(%p)",
1218 ibt_hca);
1219
1220 mutex_enter(&ibtl_async_mutex);
1221
1222 /* if there is an active async, mark this object to be freed */
1223 if (ibt_hca->ha_async_cnt > 0) {
1224 ibt_hca->ha_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1225 mutex_exit(&ibtl_async_mutex);
1226 } else { /* free the object now */
1227 mutex_exit(&ibtl_async_mutex);
1228 kmem_free(ibt_hca, sizeof (ibtl_hca_t));
1229 }
1230 }
1231
1232 /*
1233 * Completion Queue Handling.
1234 *
1235 * A completion queue can be handled through a simple callback
1236 * at interrupt level, or it may be queued for an ibtl_cq_thread
1237 * to handle. The latter is chosen during ibt_alloc_cq when the
1238 * IBTF_CQ_HANDLER_IN_THREAD is specified.
1239 */
1240
1241 static void
1242 ibtl_cq_handler_call(ibtl_cq_t *ibtl_cq)
1243 {
1244 ibt_cq_handler_t cq_handler;
1245 void *arg;
1246
1247 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_cq_handler_call(%p)", ibtl_cq);
1248
1249 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibtl_cq))
1250 cq_handler = ibtl_cq->cq_comp_handler;
1251 arg = ibtl_cq->cq_arg;
1252 if (cq_handler != NULL)
1253 cq_handler(ibtl_cq, arg);
1254 else
1255 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_cq_handler_call: "
1256 "no cq_handler for cq %p", ibtl_cq);
1257 }
1258
1259 /*
1260 * Before ibt_free_cq can continue, we need to ensure no more cq_handler
1261 * callbacks can occur. When we get the mutex, we know there are no
1262 * outstanding cq_handler callbacks. We set the cq_handler to NULL to
1263 * prohibit future callbacks.
1264 */
1265 void
1266 ibtl_free_cq_check(ibtl_cq_t *ibtl_cq)
1267 {
1268 mutex_enter(&ibtl_cq->cq_mutex);
1269 ibtl_cq->cq_comp_handler = NULL;
1270 mutex_exit(&ibtl_cq->cq_mutex);
1271 if (ibtl_cq->cq_in_thread) {
1272 mutex_enter(&ibtl_cq_mutex);
1273 --ibtl_cqs_using_threads;
1274 while (ibtl_cq->cq_impl_flags & IBTL_CQ_PENDING) {
1275 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_CALL_CLIENT;
1276 ibtl_cq->cq_impl_flags |= IBTL_CQ_FREE;
1277 cv_wait(&ibtl_cq_cv, &ibtl_cq_mutex);
1278 }
1279 mutex_exit(&ibtl_cq_mutex);
1280 }
1281 }
1282
1283 /*
1284 * Loop forever, calling cq_handlers until the cq list
1285 * is empty.
1286 */
1287
1288 static void
1289 ibtl_cq_thread(void)
1290 {
1291 #ifndef __lock_lint
1292 kmutex_t cpr_mutex;
1293 #endif
1294 callb_cpr_t cprinfo;
1295
1296 _NOTE(MUTEX_PROTECTS_DATA(cpr_mutex, cprinfo))
1297 _NOTE(NO_COMPETING_THREADS_NOW)
1298 mutex_init(&cpr_mutex, NULL, MUTEX_DRIVER, NULL);
1299 CALLB_CPR_INIT(&cprinfo, &cpr_mutex, callb_generic_cpr,
1300 "ibtl_cq_thread");
1301 #ifndef lint
1302 _NOTE(COMPETING_THREADS_NOW)
1303 #endif
1304
1305 mutex_enter(&ibtl_cq_mutex);
1306
1307 for (;;) {
1308 if (ibtl_cq_list_start) {
1309 ibtl_cq_t *ibtl_cq;
1310
1311 ibtl_cq = ibtl_cq_list_start;
1312 ibtl_cq_list_start = ibtl_cq->cq_link;
1313 ibtl_cq->cq_link = NULL;
1314 if (ibtl_cq == ibtl_cq_list_end)
1315 ibtl_cq_list_end = NULL;
1316
1317 while (ibtl_cq->cq_impl_flags & IBTL_CQ_CALL_CLIENT) {
1318 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_CALL_CLIENT;
1319 mutex_exit(&ibtl_cq_mutex);
1320 ibtl_cq_handler_call(ibtl_cq);
1321 mutex_enter(&ibtl_cq_mutex);
1322 }
1323 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_PENDING;
1324 if (ibtl_cq->cq_impl_flags & IBTL_CQ_FREE)
1325 cv_broadcast(&ibtl_cq_cv);
1326 } else {
1327 if (ibtl_cq_thread_exit == IBTL_THREAD_EXIT)
1328 break;
1329 mutex_enter(&cpr_mutex);
1330 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1331 mutex_exit(&cpr_mutex);
1332
1333 cv_wait(&ibtl_cq_cv, &ibtl_cq_mutex);
1334
1335 mutex_exit(&ibtl_cq_mutex);
1336 mutex_enter(&cpr_mutex);
1337 CALLB_CPR_SAFE_END(&cprinfo, &cpr_mutex);
1338 mutex_exit(&cpr_mutex);
1339 mutex_enter(&ibtl_cq_mutex);
1340 }
1341 }
1342
1343 mutex_exit(&ibtl_cq_mutex);
1344 #ifndef __lock_lint
1345 mutex_enter(&cpr_mutex);
1346 CALLB_CPR_EXIT(&cprinfo);
1347 #endif
1348 mutex_destroy(&cpr_mutex);
1349 }
1350
1351
1352 /*
1353 * ibc_cq_handler()
1354 *
1355 * Completion Queue Notification Handler.
1356 *
1357 */
1358 /*ARGSUSED*/
1359 void
1360 ibc_cq_handler(ibc_clnt_hdl_t ibc_hdl, ibt_cq_hdl_t ibtl_cq)
1361 {
1362 IBTF_DPRINTF_L4(ibtf_handlers, "ibc_cq_handler(%p, %p)",
1363 ibc_hdl, ibtl_cq);
1364
1365 if (ibtl_cq->cq_in_thread) {
1366 mutex_enter(&ibtl_cq_mutex);
1367 ibtl_cq->cq_impl_flags |= IBTL_CQ_CALL_CLIENT;
1368 if ((ibtl_cq->cq_impl_flags & IBTL_CQ_PENDING) == 0) {
1369 ibtl_cq->cq_impl_flags |= IBTL_CQ_PENDING;
1370 ibtl_cq->cq_link = NULL;
1371 if (ibtl_cq_list_end == NULL)
1372 ibtl_cq_list_start = ibtl_cq;
1373 else
1374 ibtl_cq_list_end->cq_link = ibtl_cq;
1375 ibtl_cq_list_end = ibtl_cq;
1376 cv_signal(&ibtl_cq_cv);
1377 }
1378 mutex_exit(&ibtl_cq_mutex);
1379 return;
1380 } else
1381 ibtl_cq_handler_call(ibtl_cq);
1382 }
1383
1384
1385 /*
1386 * ibt_enable_cq_notify()
1387 * Enable Notification requests on the specified CQ.
1388 *
1389 * ibt_cq The CQ handle.
1390 *
1391 * notify_type Enable notifications for all (IBT_NEXT_COMPLETION)
1392 * completions, or the next Solicited completion
1393 * (IBT_NEXT_SOLICITED) only.
1394 *
1395 * Completion notifications are disabled by setting the completion
1396 * handler to NULL by calling ibt_set_cq_handler().
1397 */
1398 ibt_status_t
1399 ibt_enable_cq_notify(ibt_cq_hdl_t ibtl_cq, ibt_cq_notify_flags_t notify_type)
1400 {
1401 IBTF_DPRINTF_L3(ibtf_handlers, "ibt_enable_cq_notify(%p, %d)",
1402 ibtl_cq, notify_type);
1403
1404 return (IBTL_CQ2CIHCAOPS_P(ibtl_cq)->ibc_notify_cq(
1405 IBTL_CQ2CIHCA(ibtl_cq), ibtl_cq->cq_ibc_cq_hdl, notify_type));
1406 }
1407
1408
1409 /*
1410 * ibt_set_cq_handler()
1411 * Register a work request completion handler with the IBTF.
1412 *
1413 * ibt_cq The CQ handle.
1414 *
1415 * completion_handler The completion handler.
1416 *
1417 * arg The IBTF client private argument to be passed
1418 * back to the client when calling the CQ
1419 * completion handler.
1420 *
1421 * Completion notifications are disabled by setting the completion
1422 * handler to NULL. When setting the handler to NULL, no additional
1423 * calls to the previous CQ handler will be initiated, but there may
1424 * be one in progress.
1425 *
1426 * This function does not otherwise change the state of previous
1427 * calls to ibt_enable_cq_notify().
1428 */
1429 void
1430 ibt_set_cq_handler(ibt_cq_hdl_t ibtl_cq, ibt_cq_handler_t completion_handler,
1431 void *arg)
1432 {
1433 IBTF_DPRINTF_L3(ibtf_handlers, "ibt_set_cq_handler(%p, %p, %p)",
1434 ibtl_cq, completion_handler, arg);
1435
1436 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibtl_cq))
1437 ibtl_cq->cq_comp_handler = completion_handler;
1438 ibtl_cq->cq_arg = arg;
1439 }
1440
1441
1442 /*
1443 * Inform IBT clients about New HCAs.
1444 *
1445 * We use taskqs to allow simultaneous notification, with sleeping.
1446 * Since taskqs only allow one argument, we define a structure
1447 * because we need to pass in two arguments.
1448 */
1449
1450 struct ibtl_new_hca_s {
1451 ibtl_clnt_t *nh_clntp;
1452 ibtl_hca_devinfo_t *nh_hca_devp;
1453 ibt_async_code_t nh_code;
1454 };
1455
1456 static void
1457 ibtl_tell_client_about_new_hca(void *arg)
1458 {
1459 struct ibtl_new_hca_s *new_hcap = (struct ibtl_new_hca_s *)arg;
1460 ibtl_clnt_t *clntp = new_hcap->nh_clntp;
1461 ibt_async_event_t async_event;
1462 ibtl_hca_devinfo_t *hca_devp = new_hcap->nh_hca_devp;
1463
1464 bzero(&async_event, sizeof (async_event));
1465 async_event.ev_hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
1466 clntp->clnt_modinfop->mi_async_handler(
1467 clntp->clnt_private, NULL, new_hcap->nh_code, &async_event);
1468 kmem_free(new_hcap, sizeof (*new_hcap));
1469 #ifdef __lock_lint
1470 {
1471 ibt_hca_hdl_t hca_hdl;
1472 (void) ibt_open_hca(clntp, 0ULL, &hca_hdl);
1473 }
1474 #endif
1475 mutex_enter(&ibtl_clnt_list_mutex);
1476 if (--hca_devp->hd_async_task_cnt == 0)
1477 cv_signal(&hca_devp->hd_async_task_cv);
1478 if (--clntp->clnt_async_cnt == 0)
1479 cv_broadcast(&ibtl_clnt_cv);
1480 mutex_exit(&ibtl_clnt_list_mutex);
1481 }
1482
1483 /*
1484 * ibtl_announce_new_hca:
1485 *
1486 * o First attach these clients in the given order
1487 * IBMA
1488 * IBCM
1489 *
1490 * o Next attach all other clients in parallel.
1491 *
1492 * NOTE: Use the taskq to simultaneously notify all clients of the new HCA.
1493 * Retval from clients is ignored.
1494 */
1495 void
1496 ibtl_announce_new_hca(ibtl_hca_devinfo_t *hca_devp)
1497 {
1498 ibtl_clnt_t *clntp;
1499 struct ibtl_new_hca_s *new_hcap;
1500
1501 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_announce_new_hca(%p, %llX)",
1502 hca_devp, hca_devp->hd_hca_attr->hca_node_guid);
1503
1504 mutex_enter(&ibtl_clnt_list_mutex);
1505
1506 clntp = ibtl_clnt_list;
1507 while (clntp != NULL) {
1508 if (clntp->clnt_modinfop->mi_clnt_class == IBT_IBMA) {
1509 IBTF_DPRINTF_L4(ibtf_handlers,
1510 "ibtl_announce_new_hca: calling IBMF");
1511 if (clntp->clnt_modinfop->mi_async_handler) {
1512 _NOTE(NO_COMPETING_THREADS_NOW)
1513 new_hcap = kmem_alloc(sizeof (*new_hcap),
1514 KM_SLEEP);
1515 new_hcap->nh_clntp = clntp;
1516 new_hcap->nh_hca_devp = hca_devp;
1517 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1518 #ifndef lint
1519 _NOTE(COMPETING_THREADS_NOW)
1520 #endif
1521 clntp->clnt_async_cnt++;
1522 hca_devp->hd_async_task_cnt++;
1523
1524 (void) taskq_dispatch(ibtl_async_taskq,
1525 ibtl_tell_client_about_new_hca, new_hcap,
1526 TQ_SLEEP);
1527 }
1528 break;
1529 }
1530 clntp = clntp->clnt_list_link;
1531 }
1532 if (clntp != NULL)
1533 while (clntp->clnt_async_cnt > 0)
1534 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1535 clntp = ibtl_clnt_list;
1536 while (clntp != NULL) {
1537 if (clntp->clnt_modinfop->mi_clnt_class == IBT_DM) {
1538 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_announce_new_hca: "
1539 "calling %s", clntp->clnt_modinfop->mi_clnt_name);
1540 if (clntp->clnt_modinfop->mi_async_handler) {
1541 _NOTE(NO_COMPETING_THREADS_NOW)
1542 new_hcap = kmem_alloc(sizeof (*new_hcap),
1543 KM_SLEEP);
1544 new_hcap->nh_clntp = clntp;
1545 new_hcap->nh_hca_devp = hca_devp;
1546 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1547 #ifndef lint
1548 _NOTE(COMPETING_THREADS_NOW)
1549 #endif
1550 clntp->clnt_async_cnt++;
1551 hca_devp->hd_async_task_cnt++;
1552
1553 mutex_exit(&ibtl_clnt_list_mutex);
1554 (void) ibtl_tell_client_about_new_hca(
1555 new_hcap);
1556 mutex_enter(&ibtl_clnt_list_mutex);
1557 }
1558 break;
1559 }
1560 clntp = clntp->clnt_list_link;
1561 }
1562
1563 clntp = ibtl_clnt_list;
1564 while (clntp != NULL) {
1565 if (clntp->clnt_modinfop->mi_clnt_class == IBT_CM) {
1566 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_announce_new_hca: "
1567 "calling %s", clntp->clnt_modinfop->mi_clnt_name);
1568 if (clntp->clnt_modinfop->mi_async_handler) {
1569 _NOTE(NO_COMPETING_THREADS_NOW)
1570 new_hcap = kmem_alloc(sizeof (*new_hcap),
1571 KM_SLEEP);
1572 new_hcap->nh_clntp = clntp;
1573 new_hcap->nh_hca_devp = hca_devp;
1574 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1575 #ifndef lint
1576 _NOTE(COMPETING_THREADS_NOW)
1577 #endif
1578 clntp->clnt_async_cnt++;
1579 hca_devp->hd_async_task_cnt++;
1580
1581 (void) taskq_dispatch(ibtl_async_taskq,
1582 ibtl_tell_client_about_new_hca, new_hcap,
1583 TQ_SLEEP);
1584 }
1585 break;
1586 }
1587 clntp = clntp->clnt_list_link;
1588 }
1589 if (clntp != NULL)
1590 while (clntp->clnt_async_cnt > 0)
1591 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1592 clntp = ibtl_clnt_list;
1593 while (clntp != NULL) {
1594 if ((clntp->clnt_modinfop->mi_clnt_class != IBT_DM) &&
1595 (clntp->clnt_modinfop->mi_clnt_class != IBT_CM) &&
1596 (clntp->clnt_modinfop->mi_clnt_class != IBT_IBMA)) {
1597 IBTF_DPRINTF_L4(ibtf_handlers,
1598 "ibtl_announce_new_hca: Calling %s ",
1599 clntp->clnt_modinfop->mi_clnt_name);
1600 if (clntp->clnt_modinfop->mi_async_handler) {
1601 _NOTE(NO_COMPETING_THREADS_NOW)
1602 new_hcap = kmem_alloc(sizeof (*new_hcap),
1603 KM_SLEEP);
1604 new_hcap->nh_clntp = clntp;
1605 new_hcap->nh_hca_devp = hca_devp;
1606 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1607 #ifndef lint
1608 _NOTE(COMPETING_THREADS_NOW)
1609 #endif
1610 clntp->clnt_async_cnt++;
1611 hca_devp->hd_async_task_cnt++;
1612
1613 (void) taskq_dispatch(ibtl_async_taskq,
1614 ibtl_tell_client_about_new_hca, new_hcap,
1615 TQ_SLEEP);
1616 }
1617 }
1618 clntp = clntp->clnt_list_link;
1619 }
1620
1621 /* wait for all tasks to complete */
1622 while (hca_devp->hd_async_task_cnt != 0)
1623 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1624
1625 /* wakeup thread that may be waiting to send an HCA async */
1626 ASSERT(hca_devp->hd_async_busy == 1);
1627 hca_devp->hd_async_busy = 0;
1628 cv_broadcast(&hca_devp->hd_async_busy_cv);
1629 mutex_exit(&ibtl_clnt_list_mutex);
1630 }
1631
1632 /*
1633 * ibtl_detach_all_clients:
1634 *
1635 * Return value - 0 for Success, 1 for Failure
1636 *
1637 * o First detach general clients.
1638 *
1639 * o Next detach these clients
1640 * IBCM
1641 * IBDM
1642 *
1643 * o Finally, detach this client
1644 * IBMA
1645 */
1646 int
1647 ibtl_detach_all_clients(ibtl_hca_devinfo_t *hca_devp)
1648 {
1649 ib_guid_t hcaguid = hca_devp->hd_hca_attr->hca_node_guid;
1650 ibtl_hca_t *ibt_hca;
1651 ibtl_clnt_t *clntp;
1652 int retval;
1653
1654 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_detach_all_clients(%llX)",
1655 hcaguid);
1656
1657 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex));
1658
1659 while (hca_devp->hd_async_busy)
1660 cv_wait(&hca_devp->hd_async_busy_cv, &ibtl_clnt_list_mutex);
1661 hca_devp->hd_async_busy = 1;
1662
1663 /* First inform general clients asynchronously */
1664 hca_devp->hd_async_event.ev_hca_guid = hcaguid;
1665 hca_devp->hd_async_event.ev_fma_ena = 0;
1666 hca_devp->hd_async_event.ev_chan_hdl = NULL;
1667 hca_devp->hd_async_event.ev_cq_hdl = NULL;
1668 hca_devp->hd_async_code = IBT_HCA_DETACH_EVENT;
1669
1670 ibt_hca = hca_devp->hd_clnt_list;
1671 while (ibt_hca != NULL) {
1672 clntp = ibt_hca->ha_clnt_devp;
1673 if (IBTL_GENERIC_CLIENT(clntp)) {
1674 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1675 mutex_enter(&ibtl_async_mutex);
1676 ibt_hca->ha_async_cnt++;
1677 mutex_exit(&ibtl_async_mutex);
1678 hca_devp->hd_async_task_cnt++;
1679
1680 (void) taskq_dispatch(ibtl_async_taskq,
1681 ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
1682 }
1683 ibt_hca = ibt_hca->ha_clnt_link;
1684 }
1685
1686 /* wait for all clients to complete */
1687 while (hca_devp->hd_async_task_cnt != 0) {
1688 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1689 }
1690 /* Go thru the clients and check if any have not closed this HCA. */
1691 retval = 0;
1692 ibt_hca = hca_devp->hd_clnt_list;
1693 while (ibt_hca != NULL) {
1694 clntp = ibt_hca->ha_clnt_devp;
1695 if (IBTL_GENERIC_CLIENT(clntp)) {
1696 IBTF_DPRINTF_L2(ibtf_handlers,
1697 "ibtl_detach_all_clients: "
1698 "client '%s' failed to close the HCA.",
1699 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1700 retval = 1;
1701 }
1702 ibt_hca = ibt_hca->ha_clnt_link;
1703 }
1704 if (retval == 1)
1705 goto bailout;
1706
1707 /* Next inform IBDM asynchronously */
1708 ibt_hca = hca_devp->hd_clnt_list;
1709 while (ibt_hca != NULL) {
1710 clntp = ibt_hca->ha_clnt_devp;
1711 if (clntp->clnt_modinfop->mi_clnt_class == IBT_DM) {
1712 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1713 mutex_enter(&ibtl_async_mutex);
1714 ibt_hca->ha_async_cnt++;
1715 mutex_exit(&ibtl_async_mutex);
1716 hca_devp->hd_async_task_cnt++;
1717
1718 mutex_exit(&ibtl_clnt_list_mutex);
1719 ibtl_hca_client_async_task(ibt_hca);
1720 mutex_enter(&ibtl_clnt_list_mutex);
1721 break;
1722 }
1723 ibt_hca = ibt_hca->ha_clnt_link;
1724 }
1725
1726 /*
1727 * Next inform IBCM.
1728 * As IBCM doesn't perform ibt_open_hca(), IBCM will not be
1729 * accessible via hca_devp->hd_clnt_list.
1730 * ibtl_cm_async_handler will NOT be NULL, if IBCM is registered.
1731 */
1732 if (ibtl_cm_async_handler) {
1733 ibtl_tell_mgr(hca_devp, ibtl_cm_async_handler,
1734 ibtl_cm_clnt_private);
1735
1736 /* wait for all tasks to complete */
1737 while (hca_devp->hd_async_task_cnt != 0)
1738 cv_wait(&hca_devp->hd_async_task_cv,
1739 &ibtl_clnt_list_mutex);
1740 }
1741
1742 /* Go thru the clients and check if any have not closed this HCA. */
1743 retval = 0;
1744 ibt_hca = hca_devp->hd_clnt_list;
1745 while (ibt_hca != NULL) {
1746 clntp = ibt_hca->ha_clnt_devp;
1747 if (clntp->clnt_modinfop->mi_clnt_class != IBT_IBMA) {
1748 IBTF_DPRINTF_L2(ibtf_handlers,
1749 "ibtl_detach_all_clients: "
1750 "client '%s' failed to close the HCA.",
1751 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1752 retval = 1;
1753 }
1754 ibt_hca = ibt_hca->ha_clnt_link;
1755 }
1756 if (retval == 1)
1757 goto bailout;
1758
1759 /* Finally, inform IBMA */
1760 ibt_hca = hca_devp->hd_clnt_list;
1761 while (ibt_hca != NULL) {
1762 clntp = ibt_hca->ha_clnt_devp;
1763 if (clntp->clnt_modinfop->mi_clnt_class == IBT_IBMA) {
1764 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1765 mutex_enter(&ibtl_async_mutex);
1766 ibt_hca->ha_async_cnt++;
1767 mutex_exit(&ibtl_async_mutex);
1768 hca_devp->hd_async_task_cnt++;
1769
1770 (void) taskq_dispatch(ibtl_async_taskq,
1771 ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
1772 } else
1773 IBTF_DPRINTF_L2(ibtf_handlers,
1774 "ibtl_detach_all_clients: "
1775 "client '%s' is unexpectedly on the client list",
1776 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1777 ibt_hca = ibt_hca->ha_clnt_link;
1778 }
1779
1780 /* wait for IBMA to complete */
1781 while (hca_devp->hd_async_task_cnt != 0) {
1782 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1783 }
1784
1785 /* Check if this HCA's client list is empty. */
1786 ibt_hca = hca_devp->hd_clnt_list;
1787 if (ibt_hca != NULL) {
1788 IBTF_DPRINTF_L2(ibtf_handlers,
1789 "ibtl_detach_all_clients: "
1790 "client '%s' failed to close the HCA.",
1791 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1792 retval = 1;
1793 } else
1794 retval = 0;
1795
1796 bailout:
1797 if (retval) {
1798 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */
1799 mutex_exit(&ibtl_clnt_list_mutex);
1800 ibtl_announce_new_hca(hca_devp);
1801 mutex_enter(&ibtl_clnt_list_mutex);
1802 } else {
1803 hca_devp->hd_async_busy = 0;
1804 cv_broadcast(&hca_devp->hd_async_busy_cv);
1805 }
1806
1807 return (retval);
1808 }
1809
1810 void
1811 ibtl_free_clnt_async_check(ibtl_clnt_t *clntp)
1812 {
1813 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_clnt_async_check(%p)", clntp);
1814
1815 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex));
1816
1817 /* wait for all asyncs based on "ibtl_clnt_list" to complete */
1818 while (clntp->clnt_async_cnt != 0) {
1819 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1820 }
1821 }
1822
1823 static void
1824 ibtl_dec_clnt_async_cnt(ibtl_clnt_t *clntp)
1825 {
1826 mutex_enter(&ibtl_clnt_list_mutex);
1827 if (--clntp->clnt_async_cnt == 0) {
1828 cv_broadcast(&ibtl_clnt_cv);
1829 }
1830 mutex_exit(&ibtl_clnt_list_mutex);
1831 }
1832
1833 static void
1834 ibtl_inc_clnt_async_cnt(ibtl_clnt_t *clntp)
1835 {
1836 mutex_enter(&ibtl_clnt_list_mutex);
1837 ++clntp->clnt_async_cnt;
1838 mutex_exit(&ibtl_clnt_list_mutex);
1839 }
1840
1841
1842 /*
1843 * Functions and data structures to inform clients that a notification
1844 * has occurred about Multicast Groups that might interest them.
1845 */
1846 struct ibtl_sm_notice {
1847 ibt_clnt_hdl_t np_ibt_hdl;
1848 ib_gid_t np_sgid;
1849 ibt_subnet_event_code_t np_code;
1850 ibt_subnet_event_t np_event;
1851 };
1852
1853 static void
1854 ibtl_sm_notice_task(void *arg)
1855 {
1856 struct ibtl_sm_notice *noticep = (struct ibtl_sm_notice *)arg;
1857 ibt_clnt_hdl_t ibt_hdl = noticep->np_ibt_hdl;
1858 ibt_sm_notice_handler_t sm_notice_handler;
1859
1860 sm_notice_handler = ibt_hdl->clnt_sm_trap_handler;
1861 if (sm_notice_handler != NULL)
1862 sm_notice_handler(ibt_hdl->clnt_sm_trap_handler_arg,
1863 noticep->np_sgid, noticep->np_code, ¬icep->np_event);
1864 kmem_free(noticep, sizeof (*noticep));
1865 ibtl_dec_clnt_async_cnt(ibt_hdl);
1866 }
1867
1868 /*
1869 * Inform the client that MCG notices are not working at this time.
1870 */
1871 void
1872 ibtl_cm_sm_notice_init_failure(ibtl_cm_sm_init_fail_t *ifail)
1873 {
1874 ibt_clnt_hdl_t ibt_hdl = ifail->smf_ibt_hdl;
1875 struct ibtl_sm_notice *noticep;
1876 ib_gid_t *sgidp = &ifail->smf_sgid[0];
1877 int i;
1878
1879 for (i = 0; i < ifail->smf_num_sgids; i++) {
1880 _NOTE(NO_COMPETING_THREADS_NOW)
1881 noticep = kmem_zalloc(sizeof (*noticep), KM_SLEEP);
1882 noticep->np_ibt_hdl = ibt_hdl;
1883 noticep->np_sgid = *sgidp++;
1884 noticep->np_code = IBT_SM_EVENT_UNAVAILABLE;
1885 #ifndef lint
1886 _NOTE(COMPETING_THREADS_NOW)
1887 #endif
1888 ibtl_inc_clnt_async_cnt(ibt_hdl);
1889 (void) taskq_dispatch(ibtl_async_taskq,
1890 ibtl_sm_notice_task, noticep, TQ_SLEEP);
1891 }
1892 }
1893
1894 /*
1895 * Inform all clients of the event.
1896 */
1897 void
1898 ibtl_cm_sm_notice_handler(ib_gid_t sgid, ibt_subnet_event_code_t code,
1899 ibt_subnet_event_t *event)
1900 {
1901 _NOTE(NO_COMPETING_THREADS_NOW)
1902 struct ibtl_sm_notice *noticep;
1903 ibtl_clnt_t *clntp;
1904
1905 mutex_enter(&ibtl_clnt_list_mutex);
1906 clntp = ibtl_clnt_list;
1907 while (clntp != NULL) {
1908 if (clntp->clnt_sm_trap_handler) {
1909 noticep = kmem_zalloc(sizeof (*noticep), KM_SLEEP);
1910 noticep->np_ibt_hdl = clntp;
1911 noticep->np_sgid = sgid;
1912 noticep->np_code = code;
1913 noticep->np_event = *event;
1914 ++clntp->clnt_async_cnt;
1915 (void) taskq_dispatch(ibtl_async_taskq,
1916 ibtl_sm_notice_task, noticep, TQ_SLEEP);
1917 }
1918 clntp = clntp->clnt_list_link;
1919 }
1920 mutex_exit(&ibtl_clnt_list_mutex);
1921 #ifndef lint
1922 _NOTE(COMPETING_THREADS_NOW)
1923 #endif
1924 }
1925
1926 /*
1927 * Record the handler for this client.
1928 */
1929 void
1930 ibtl_cm_set_sm_notice_handler(ibt_clnt_hdl_t ibt_hdl,
1931 ibt_sm_notice_handler_t sm_notice_handler, void *private)
1932 {
1933 _NOTE(NO_COMPETING_THREADS_NOW)
1934 ibt_hdl->clnt_sm_trap_handler = sm_notice_handler;
1935 ibt_hdl->clnt_sm_trap_handler_arg = private;
1936 #ifndef lint
1937 _NOTE(COMPETING_THREADS_NOW)
1938 #endif
1939 }
1940
1941
1942 /*
1943 * ibtl_another_cq_handler_in_thread()
1944 *
1945 * Conditionally increase the number of cq_threads.
1946 * The number of threads grows, based on the number of cqs using threads.
1947 *
1948 * The table below controls the number of threads as follows:
1949 *
1950 * Number of CQs Number of cq_threads
1951 * 0 0
1952 * 1 1
1953 * 2-3 2
1954 * 4-5 3
1955 * 6-9 4
1956 * 10-15 5
1957 * 16-23 6
1958 * 24-31 7
1959 * 32+ 8
1960 */
1961
1962 #define IBTL_CQ_MAXTHREADS 8
1963 static uint8_t ibtl_cq_scaling[IBTL_CQ_MAXTHREADS] = {
1964 1, 2, 4, 6, 10, 16, 24, 32
1965 };
1966
1967 static kt_did_t ibtl_cq_did[IBTL_CQ_MAXTHREADS];
1968
1969 void
1970 ibtl_another_cq_handler_in_thread(void)
1971 {
1972 kthread_t *t;
1973 int my_idx;
1974
1975 mutex_enter(&ibtl_cq_mutex);
1976 if ((ibtl_cq_threads == IBTL_CQ_MAXTHREADS) ||
1977 (++ibtl_cqs_using_threads < ibtl_cq_scaling[ibtl_cq_threads])) {
1978 mutex_exit(&ibtl_cq_mutex);
1979 return;
1980 }
1981 my_idx = ibtl_cq_threads++;
1982 mutex_exit(&ibtl_cq_mutex);
1983 t = thread_create(NULL, 0, ibtl_cq_thread, NULL, 0, &p0, TS_RUN,
1984 ibtl_pri - 1);
1985 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibtl_cq_did))
1986 ibtl_cq_did[my_idx] = t->t_did; /* save for thread_join() */
1987 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibtl_cq_did))
1988 }
1989
1990 void
1991 ibtl_thread_init(void)
1992 {
1993 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_init()");
1994
1995 mutex_init(&ibtl_async_mutex, NULL, MUTEX_DEFAULT, NULL);
1996 cv_init(&ibtl_async_cv, NULL, CV_DEFAULT, NULL);
1997 cv_init(&ibtl_clnt_cv, NULL, CV_DEFAULT, NULL);
1998
1999 mutex_init(&ibtl_cq_mutex, NULL, MUTEX_DEFAULT, NULL);
2000 cv_init(&ibtl_cq_cv, NULL, CV_DEFAULT, NULL);
2001 }
2002
2003 void
2004 ibtl_thread_init2(void)
2005 {
2006 int i;
2007 static int initted = 0;
2008 kthread_t *t;
2009
2010 mutex_enter(&ibtl_async_mutex);
2011 if (initted == 1) {
2012 mutex_exit(&ibtl_async_mutex);
2013 return;
2014 }
2015 initted = 1;
2016 mutex_exit(&ibtl_async_mutex);
2017 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibtl_async_did))
2018 ibtl_async_did = kmem_zalloc(ibtl_async_thread_init * sizeof (kt_did_t),
2019 KM_SLEEP);
2020
2021 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_init2()");
2022
2023 for (i = 0; i < ibtl_async_thread_init; i++) {
2024 t = thread_create(NULL, 0, ibtl_async_thread, NULL, 0, &p0,
2025 TS_RUN, ibtl_pri - 1);
2026 ibtl_async_did[i] = t->t_did; /* thread_join() */
2027 }
2028 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibtl_async_did))
2029 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibtl_cq_threads))
2030 for (i = 0; i < ibtl_cq_threads; i++) {
2031 t = thread_create(NULL, 0, ibtl_cq_thread, NULL, 0, &p0,
2032 TS_RUN, ibtl_pri - 1);
2033 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibtl_cq_did))
2034 ibtl_cq_did[i] = t->t_did; /* save for thread_join() */
2035 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibtl_cq_did))
2036 }
2037 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibtl_cq_threads))
2038 }
2039
2040 void
2041 ibtl_thread_fini(void)
2042 {
2043 int i;
2044
2045 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_fini()");
2046
2047 /* undo the work done by ibtl_thread_init() */
2048
2049 mutex_enter(&ibtl_cq_mutex);
2050 ibtl_cq_thread_exit = IBTL_THREAD_EXIT;
2051 cv_broadcast(&ibtl_cq_cv);
2052 mutex_exit(&ibtl_cq_mutex);
2053
2054 mutex_enter(&ibtl_async_mutex);
2055 ibtl_async_thread_exit = IBTL_THREAD_EXIT;
2056 cv_broadcast(&ibtl_async_cv);
2057 mutex_exit(&ibtl_async_mutex);
2058
2059 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibtl_cq_threads))
2060 for (i = 0; i < ibtl_cq_threads; i++)
2061 thread_join(ibtl_cq_did[i]);
2062 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibtl_cq_threads))
2063
2064 if (ibtl_async_did) {
2065 for (i = 0; i < ibtl_async_thread_init; i++)
2066 thread_join(ibtl_async_did[i]);
2067
2068 kmem_free(ibtl_async_did,
2069 ibtl_async_thread_init * sizeof (kt_did_t));
2070 }
2071 mutex_destroy(&ibtl_cq_mutex);
2072 cv_destroy(&ibtl_cq_cv);
2073
2074 mutex_destroy(&ibtl_async_mutex);
2075 cv_destroy(&ibtl_async_cv);
2076 cv_destroy(&ibtl_clnt_cv);
2077 }
2078
2079 /* ARGSUSED */
2080 ibt_status_t ibtl_dummy_node_info_cb(ib_guid_t hca_guid, uint8_t port,
2081 ib_lid_t lid, ibt_node_info_t *node_info)
2082 {
2083 return (IBT_SUCCESS);
2084 }