1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/ib/ibtl/impl/ibtl.h>
26 #include <sys/ib/ibtl/impl/ibtl_cm.h>
27 #include <sys/taskq.h>
28 #include <sys/disp.h>
29 #include <sys/callb.h>
30 #include <sys/proc.h>
31
32 /*
33 * ibtl_handlers.c
34 */
35
36 /*
37 * What's in this file?
38 *
39 * This file started as an implementation of Asynchronous Event/Error
40 * handling and Completion Queue handling. As the implementation
41 * evolved, code has been added for other ibc_* interfaces (resume,
42 * predetach, etc.) that use the same mechanisms as used for asyncs.
43 *
44 * Async and CQ handling at interrupt level.
45 *
46 * CQ handling is normally done at interrupt level using the CQ callback
47 * handler to call the appropriate IBT Client (owner of the CQ). For
48 * clients that would prefer a fully flexible non-interrupt context to
49 * do their CQ handling, a CQ can be created so that its handler is
50 * called from a non-interrupt thread. CQ handling is done frequently
51 * whereas Async handling is expected to occur very infrequently.
52 *
53 * Async handling is done by marking (or'ing in of an async_code of) the
54 * pertinent IBTL data structure, and then notifying the async_thread(s)
55 * that the data structure has async work to be done. The notification
56 * occurs by linking the data structure through its async_link onto a
57 * list of like data structures and waking up an async_thread. This
58 * list append is not done if there is already async work pending on
59 * this data structure (IBTL_ASYNC_PENDING).
60 *
61 * Async Mutex and CQ Mutex
62 *
63 * The global ibtl_async_mutex is "the" mutex used to control access
64 * to all the data needed by ibc_async_handler. All the threads that
65 * use this mutex are written so that the mutex is held for very short
66 * periods of time, and never held while making calls to functions
67 * that may block.
68 *
69 * The global ibtl_cq_mutex is used similarly by ibc_cq_handler and
70 * the ibtl_cq_thread(s).
71 *
72 * Mutex hierarchy
73 *
74 * The ibtl_clnt_list_mutex is above the ibtl_async_mutex.
75 * ibtl_clnt_list_mutex protects all of the various lists.
76 * The ibtl_async_mutex is below this in the hierarchy.
77 *
78 * The ibtl_cq_mutex is independent of the above mutexes.
79 *
80 * Threads
81 *
82 * There are "ibtl_cq_threads" number of threads created for handling
83 * Completion Queues in threads. If this feature really gets used,
84 * then we will want to do some suitable tuning. Similarly, we may
85 * want to tune the number of "ibtl_async_thread_init".
86 *
87 * The function ibtl_cq_thread is the main loop for handling a CQ in a
88 * thread. There can be multiple threads executing this same code.
89 * The code sleeps when there is no work to be done (list is empty),
90 * otherwise it pulls the first CQ structure off the list and performs
91 * the CQ handler callback to the client. After that returns, a check
92 * is made, and if another ibc_cq_handler call was made for this CQ,
93 * the client is called again.
94 *
95 * The function ibtl_async_thread is the main loop for handling async
96 * events/errors. There can be multiple threads executing this same code.
97 * The code sleeps when there is no work to be done (lists are empty),
98 * otherwise it pulls the first structure off one of the lists and
99 * performs the async callback(s) to the client(s). Note that HCA
100 * async handling is done by calling each of the clients using the HCA.
101 * When the async handling completes, the data structure having the async
102 * event/error is checked for more work before it's considered "done".
103 *
104 * Taskq
105 *
106 * The async_taskq is used here for allowing async handler callbacks to
107 * occur simultaneously to multiple clients of an HCA. This taskq could
108 * be used for other purposes, e.g., if all the async_threads are in
109 * use, but this is deemed as overkill since asyncs should occur rarely.
110 */
111
112 /* Globals */
113 static char ibtf_handlers[] = "ibtl_handlers";
114
115 /* priority for IBTL threads (async, cq, and taskq) */
116 static pri_t ibtl_pri = MAXCLSYSPRI - 1; /* maybe override in /etc/system */
117
118 /* taskq used for HCA asyncs */
119 #define ibtl_async_taskq system_taskq
120
121 /* data for async handling by threads */
122 static kmutex_t ibtl_async_mutex; /* protects most *_async_* data */
123 static kcondvar_t ibtl_async_cv; /* async_threads wait on this */
124 static kcondvar_t ibtl_clnt_cv; /* ibt_detach might wait on this */
125 static void ibtl_dec_clnt_async_cnt(ibtl_clnt_t *clntp);
126 static void ibtl_inc_clnt_async_cnt(ibtl_clnt_t *clntp);
127
128 static kt_did_t *ibtl_async_did; /* for thread_join() */
129 int ibtl_async_thread_init = 4; /* total # of async_threads to create */
130 static int ibtl_async_thread_exit = 0; /* set if/when thread(s) should exit */
131
132 /* async lists for various structures */
133 static ibtl_hca_devinfo_t *ibtl_async_hca_list_start, *ibtl_async_hca_list_end;
134 static ibtl_eec_t *ibtl_async_eec_list_start, *ibtl_async_eec_list_end;
135 static ibtl_qp_t *ibtl_async_qp_list_start, *ibtl_async_qp_list_end;
136 static ibtl_cq_t *ibtl_async_cq_list_start, *ibtl_async_cq_list_end;
137 static ibtl_srq_t *ibtl_async_srq_list_start, *ibtl_async_srq_list_end;
138
139 /* data for CQ completion handling by threads */
140 static kmutex_t ibtl_cq_mutex; /* protects the cv and the list below */
141 static kcondvar_t ibtl_cq_cv;
142 static ibtl_cq_t *ibtl_cq_list_start, *ibtl_cq_list_end;
143
144 static int ibtl_cq_threads = 0; /* total # of cq threads */
145 static int ibtl_cqs_using_threads = 0; /* total # of cqs using threads */
146 static int ibtl_cq_thread_exit = 0; /* set if/when thread(s) should exit */
147
148 /* value used to tell IBTL threads to exit */
149 #define IBTL_THREAD_EXIT 0x1b7fdead /* IBTF DEAD */
150 /* Cisco Topspin Vendor ID for Rereg hack */
151 #define IBT_VENDOR_CISCO 0x05ad
152
153 int ibtl_eec_not_supported = 1;
154
155 char *ibtl_last_client_name; /* may help debugging */
156 typedef ibt_status_t (*ibtl_node_info_cb_t)(ib_guid_t, uint8_t, ib_lid_t,
157 ibt_node_info_t *);
158
159 ibtl_node_info_cb_t ibtl_node_info_cb;
160
161 void
162 ibtl_cm_set_node_info_cb(ibt_status_t (*node_info_cb)(ib_guid_t, uint8_t,
163 ib_lid_t, ibt_node_info_t *))
164 {
165 mutex_enter(&ibtl_clnt_list_mutex);
166 ibtl_node_info_cb = node_info_cb;
167 mutex_exit(&ibtl_clnt_list_mutex);
168 }
169
170 /*
171 * ibc_async_handler()
172 *
173 * Asynchronous Event/Error Handler.
174 *
175 * This is the function called HCA drivers to post various async
176 * event and errors mention in the IB architecture spec. See
177 * ibtl_types.h for additional details of this.
178 *
179 * This function marks the pertinent IBTF object with the async_code,
180 * and queues the object for handling by an ibtl_async_thread. If
181 * the object is NOT already marked for async processing, it is added
182 * to the associated list for that type of object, and an
183 * ibtl_async_thread is signaled to finish the async work.
184 */
185 void
186 ibc_async_handler(ibc_clnt_hdl_t hca_devp, ibt_async_code_t code,
187 ibc_async_event_t *event_p)
188 {
189 ibtl_qp_t *ibtl_qp;
190 ibtl_cq_t *ibtl_cq;
191 ibtl_srq_t *ibtl_srq;
192 ibtl_eec_t *ibtl_eec;
193 uint8_t port_minus1;
194
195 ibtl_async_port_event_t *portp;
196
197 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler(%p, 0x%x, %p)",
198 hca_devp, code, event_p);
199
200 mutex_enter(&ibtl_async_mutex);
201
202 switch (code) {
203 case IBT_EVENT_PATH_MIGRATED_QP:
204 case IBT_EVENT_SQD:
205 case IBT_ERROR_CATASTROPHIC_QP:
206 case IBT_ERROR_PATH_MIGRATE_REQ_QP:
207 case IBT_EVENT_COM_EST_QP:
208 case IBT_ERROR_INVALID_REQUEST_QP:
209 case IBT_ERROR_ACCESS_VIOLATION_QP:
210 case IBT_EVENT_EMPTY_QP:
211 case IBT_FEXCH_ERROR:
212 ibtl_qp = event_p->ev_qp_hdl;
213 if (ibtl_qp == NULL) {
214 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
215 "bad qp handle");
216 break;
217 }
218 switch (code) {
219 case IBT_ERROR_CATASTROPHIC_QP:
220 ibtl_qp->qp_cat_fma_ena = event_p->ev_fma_ena; break;
221 case IBT_ERROR_PATH_MIGRATE_REQ_QP:
222 ibtl_qp->qp_pth_fma_ena = event_p->ev_fma_ena; break;
223 case IBT_ERROR_INVALID_REQUEST_QP:
224 ibtl_qp->qp_inv_fma_ena = event_p->ev_fma_ena; break;
225 case IBT_ERROR_ACCESS_VIOLATION_QP:
226 ibtl_qp->qp_acc_fma_ena = event_p->ev_fma_ena; break;
227 }
228
229 ibtl_qp->qp_async_codes |= code;
230 if ((ibtl_qp->qp_async_flags & IBTL_ASYNC_PENDING) == 0) {
231 ibtl_qp->qp_async_flags |= IBTL_ASYNC_PENDING;
232 ibtl_qp->qp_async_link = NULL;
233 if (ibtl_async_qp_list_end == NULL)
234 ibtl_async_qp_list_start = ibtl_qp;
235 else
236 ibtl_async_qp_list_end->qp_async_link = ibtl_qp;
237 ibtl_async_qp_list_end = ibtl_qp;
238 cv_signal(&ibtl_async_cv);
239 }
240 break;
241
242 case IBT_ERROR_CQ:
243 ibtl_cq = event_p->ev_cq_hdl;
244 if (ibtl_cq == NULL) {
245 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
246 "bad cq handle");
247 break;
248 }
249 ibtl_cq->cq_async_codes |= code;
250 ibtl_cq->cq_fma_ena = event_p->ev_fma_ena;
251 if ((ibtl_cq->cq_async_flags & IBTL_ASYNC_PENDING) == 0) {
252 ibtl_cq->cq_async_flags |= IBTL_ASYNC_PENDING;
253 ibtl_cq->cq_async_link = NULL;
254 if (ibtl_async_cq_list_end == NULL)
255 ibtl_async_cq_list_start = ibtl_cq;
256 else
257 ibtl_async_cq_list_end->cq_async_link = ibtl_cq;
258 ibtl_async_cq_list_end = ibtl_cq;
259 cv_signal(&ibtl_async_cv);
260 }
261 break;
262
263 case IBT_ERROR_CATASTROPHIC_SRQ:
264 case IBT_EVENT_LIMIT_REACHED_SRQ:
265 ibtl_srq = event_p->ev_srq_hdl;
266 if (ibtl_srq == NULL) {
267 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
268 "bad srq handle");
269 break;
270 }
271 ibtl_srq->srq_async_codes |= code;
272 ibtl_srq->srq_fma_ena = event_p->ev_fma_ena;
273 if ((ibtl_srq->srq_async_flags & IBTL_ASYNC_PENDING) == 0) {
274 ibtl_srq->srq_async_flags |= IBTL_ASYNC_PENDING;
275 ibtl_srq->srq_async_link = NULL;
276 if (ibtl_async_srq_list_end == NULL)
277 ibtl_async_srq_list_start = ibtl_srq;
278 else
279 ibtl_async_srq_list_end->srq_async_link =
280 ibtl_srq;
281 ibtl_async_srq_list_end = ibtl_srq;
282 cv_signal(&ibtl_async_cv);
283 }
284 break;
285
286 case IBT_EVENT_PATH_MIGRATED_EEC:
287 case IBT_ERROR_PATH_MIGRATE_REQ_EEC:
288 case IBT_ERROR_CATASTROPHIC_EEC:
289 case IBT_EVENT_COM_EST_EEC:
290 if (ibtl_eec_not_supported) {
291 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
292 "EEC events are disabled.");
293 break;
294 }
295 ibtl_eec = event_p->ev_eec_hdl;
296 if (ibtl_eec == NULL) {
297 IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler: "
298 "bad eec handle");
299 break;
300 }
301 switch (code) {
302 case IBT_ERROR_PATH_MIGRATE_REQ_EEC:
303 ibtl_eec->eec_pth_fma_ena = event_p->ev_fma_ena; break;
304 case IBT_ERROR_CATASTROPHIC_EEC:
305 ibtl_eec->eec_cat_fma_ena = event_p->ev_fma_ena; break;
306 }
307 ibtl_eec->eec_async_codes |= code;
308 if ((ibtl_eec->eec_async_flags & IBTL_ASYNC_PENDING) == 0) {
309 ibtl_eec->eec_async_flags |= IBTL_ASYNC_PENDING;
310 ibtl_eec->eec_async_link = NULL;
311 if (ibtl_async_eec_list_end == NULL)
312 ibtl_async_eec_list_start = ibtl_eec;
313 else
314 ibtl_async_eec_list_end->eec_async_link =
315 ibtl_eec;
316 ibtl_async_eec_list_end = ibtl_eec;
317 cv_signal(&ibtl_async_cv);
318 }
319 break;
320
321 case IBT_ERROR_LOCAL_CATASTROPHIC:
322 hca_devp->hd_async_codes |= code;
323 hca_devp->hd_fma_ena = event_p->ev_fma_ena;
324 /* FALLTHROUGH */
325
326 case IBT_EVENT_PORT_UP:
327 case IBT_PORT_CHANGE_EVENT:
328 case IBT_CLNT_REREG_EVENT:
329 case IBT_ERROR_PORT_DOWN:
330 if ((code & IBT_PORT_EVENTS) != 0) {
331 if ((port_minus1 = event_p->ev_port - 1) >=
332 hca_devp->hd_hca_attr->hca_nports) {
333 IBTF_DPRINTF_L2(ibtf_handlers,
334 "ibc_async_handler: bad port #: %d",
335 event_p->ev_port);
336 break;
337 }
338 portp = &hca_devp->hd_async_port[port_minus1];
339 if (code == IBT_EVENT_PORT_UP) {
340 /*
341 * The port is just coming UP we can't have any
342 * valid older events.
343 */
344 portp->status = IBTL_HCA_PORT_UP;
345 } else if (code == IBT_ERROR_PORT_DOWN) {
346 /*
347 * The port is going DOWN older events don't
348 * count.
349 */
350 portp->status = IBTL_HCA_PORT_DOWN;
351 } else if (code == IBT_PORT_CHANGE_EVENT) {
352 /*
353 * For port UP and DOWN events only the latest
354 * event counts. If we get a UP after DOWN it
355 * is sufficient to send just UP and vice versa.
356 * In the case of port CHANGE event it is valid
357 * only when the port is UP already but if we
358 * receive it after UP but before UP is
359 * delivered we still need to deliver CHANGE
360 * after we deliver UP event.
361 *
362 * We will not get a CHANGE event when the port
363 * is down or DOWN event is pending.
364 */
365 portp->flags |= event_p->ev_port_flags;
366 portp->status |= IBTL_HCA_PORT_CHG;
367 } else if (code == IBT_CLNT_REREG_EVENT) {
368 /*
369 * SM has requested a re-register of
370 * subscription to SM events notification.
371 */
372 portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
373 }
374
375 hca_devp->hd_async_codes |= code;
376 }
377
378 if ((hca_devp->hd_async_flags & IBTL_ASYNC_PENDING) == 0) {
379 hca_devp->hd_async_flags |= IBTL_ASYNC_PENDING;
380 hca_devp->hd_async_link = NULL;
381 if (ibtl_async_hca_list_end == NULL)
382 ibtl_async_hca_list_start = hca_devp;
383 else
384 ibtl_async_hca_list_end->hd_async_link =
385 hca_devp;
386 ibtl_async_hca_list_end = hca_devp;
387 cv_signal(&ibtl_async_cv);
388 }
389
390 break;
391
392 default:
393 IBTF_DPRINTF_L1(ibtf_handlers, "ibc_async_handler: "
394 "invalid code (0x%x)", code);
395 }
396
397 mutex_exit(&ibtl_async_mutex);
398 }
399
400
401 /* Finally, make the async call to the client. */
402
403 static void
404 ibtl_async_client_call(ibtl_hca_t *ibt_hca, ibt_async_code_t code,
405 ibt_async_event_t *event_p)
406 {
407 ibtl_clnt_t *clntp;
408 void *client_private;
409 ibt_async_handler_t async_handler;
410 char *client_name;
411
412 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call(%p, 0x%x, %p)",
413 ibt_hca, code, event_p);
414
415 clntp = ibt_hca->ha_clnt_devp;
416
417 /* Record who is being called (just a debugging aid) */
418 ibtl_last_client_name = client_name = clntp->clnt_name;
419
420 client_private = clntp->clnt_private;
421 async_handler = clntp->clnt_modinfop->mi_async_handler;
422
423 if (code & (IBT_EVENT_COM_EST_QP | IBT_EVENT_COM_EST_EEC)) {
424 mutex_enter(&ibtl_clnt_list_mutex);
425 async_handler = ibtl_cm_async_handler;
426 client_private = ibtl_cm_clnt_private;
427 mutex_exit(&ibtl_clnt_list_mutex);
428 ibt_hca = NULL;
429 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
430 "calling CM for COM_EST");
431 } else {
432 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
433 "calling client '%s'", client_name);
434 }
435 if (async_handler != NULL)
436 async_handler(client_private, ibt_hca, code, event_p);
437 else
438 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_async_client_call: "
439 "client '%s' has no async handler", client_name);
440 }
441
442 /*
443 * Inform CM or DM about HCA events.
444 *
445 * We use taskqs to allow simultaneous notification, with sleeping.
446 * Since taskqs only allow one argument, we define a structure
447 * because we need to pass in more than one argument.
448 */
449
450 struct ibtl_mgr_s {
451 ibtl_hca_devinfo_t *mgr_hca_devp;
452 ibt_async_handler_t mgr_async_handler;
453 void *mgr_clnt_private;
454 };
455
456 /*
457 * Asyncs of HCA level events for CM and DM. Call CM or DM and tell them
458 * about the HCA for the event recorded in the ibtl_hca_devinfo_t.
459 */
460 static void
461 ibtl_do_mgr_async_task(void *arg)
462 {
463 struct ibtl_mgr_s *mgrp = (struct ibtl_mgr_s *)arg;
464 ibtl_hca_devinfo_t *hca_devp = mgrp->mgr_hca_devp;
465
466 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_mgr_async_task(0x%x)",
467 hca_devp->hd_async_code);
468
469 mgrp->mgr_async_handler(mgrp->mgr_clnt_private, NULL,
470 hca_devp->hd_async_code, &hca_devp->hd_async_event);
471 kmem_free(mgrp, sizeof (*mgrp));
472
473 mutex_enter(&ibtl_clnt_list_mutex);
474 if (--hca_devp->hd_async_task_cnt == 0)
475 cv_signal(&hca_devp->hd_async_task_cv);
476 mutex_exit(&ibtl_clnt_list_mutex);
477 }
478
479 static void
480 ibt_cisco_embedded_sm_rereg_fix(void *arg)
481 {
482 struct ibtl_mgr_s *mgrp = arg;
483 ibtl_hca_devinfo_t *hca_devp;
484 ibt_node_info_t node_info;
485 ibt_status_t ibt_status;
486 ibtl_async_port_event_t *portp;
487 ib_lid_t sm_lid;
488 ib_guid_t hca_guid;
489 ibt_async_event_t *event_p;
490 ibt_hca_portinfo_t *pinfop;
491 uint8_t port;
492
493 hca_devp = mgrp->mgr_hca_devp;
494
495 mutex_enter(&ibtl_clnt_list_mutex);
496 event_p = &hca_devp->hd_async_event;
497 port = event_p->ev_port;
498 portp = &hca_devp->hd_async_port[port - 1];
499 pinfop = &hca_devp->hd_portinfop[port - 1];
500 sm_lid = pinfop->p_sm_lid;
501 hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
502 mutex_exit(&ibtl_clnt_list_mutex);
503
504 ibt_status = ((ibtl_node_info_cb_t)mgrp->mgr_async_handler)(hca_guid,
505 port, sm_lid, &node_info);
506 if (ibt_status == IBT_SUCCESS) {
507 if ((node_info.n_vendor_id == IBT_VENDOR_CISCO) &&
508 (node_info.n_node_type == IBT_NODE_TYPE_SWITCH)) {
509 mutex_enter(&ibtl_async_mutex);
510 portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
511 hca_devp->hd_async_codes |= IBT_CLNT_REREG_EVENT;
512 mutex_exit(&ibtl_async_mutex);
513 }
514 }
515 kmem_free(mgrp, sizeof (*mgrp));
516
517 mutex_enter(&ibtl_clnt_list_mutex);
518 if (--hca_devp->hd_async_task_cnt == 0)
519 cv_signal(&hca_devp->hd_async_task_cv);
520 mutex_exit(&ibtl_clnt_list_mutex);
521 }
522
523 static void
524 ibtl_cm_get_node_info(ibtl_hca_devinfo_t *hca_devp,
525 ibt_async_handler_t async_handler)
526 {
527 struct ibtl_mgr_s *mgrp;
528
529 if (async_handler == NULL)
530 return;
531
532 mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
533 mgrp->mgr_hca_devp = hca_devp;
534 mgrp->mgr_async_handler = async_handler;
535 mgrp->mgr_clnt_private = NULL;
536 hca_devp->hd_async_task_cnt++;
537
538 (void) taskq_dispatch(ibtl_async_taskq,
539 ibt_cisco_embedded_sm_rereg_fix, mgrp, TQ_SLEEP);
540 }
541
542 static void
543 ibtl_tell_mgr(ibtl_hca_devinfo_t *hca_devp, ibt_async_handler_t async_handler,
544 void *clnt_private)
545 {
546 struct ibtl_mgr_s *mgrp;
547
548 if (async_handler == NULL)
549 return;
550
551 mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
552 mgrp->mgr_hca_devp = hca_devp;
553 mgrp->mgr_async_handler = async_handler;
554 mgrp->mgr_clnt_private = clnt_private;
555 hca_devp->hd_async_task_cnt++;
556
557 (void) taskq_dispatch(ibtl_async_taskq, ibtl_do_mgr_async_task, mgrp,
558 TQ_SLEEP);
559 }
560
561 /*
562 * Per client-device asyncs for HCA level events. Call each client that is
563 * using the HCA for the event recorded in the ibtl_hca_devinfo_t.
564 */
565 static void
566 ibtl_hca_client_async_task(void *arg)
567 {
568 ibtl_hca_t *ibt_hca = (ibtl_hca_t *)arg;
569 ibtl_hca_devinfo_t *hca_devp = ibt_hca->ha_hca_devp;
570 ibtl_clnt_t *clntp = ibt_hca->ha_clnt_devp;
571 ibt_async_event_t async_event;
572
573 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_hca_client_async_task(%p, 0x%x)",
574 ibt_hca, hca_devp->hd_async_code);
575
576 bcopy(&hca_devp->hd_async_event, &async_event, sizeof (async_event));
577 ibtl_async_client_call(ibt_hca, hca_devp->hd_async_code, &async_event);
578
579 mutex_enter(&ibtl_async_mutex);
580 if (--ibt_hca->ha_async_cnt == 0 &&
581 (ibt_hca->ha_async_flags & IBTL_ASYNC_FREE_OBJECT)) {
582 mutex_exit(&ibtl_async_mutex);
583 kmem_free(ibt_hca, sizeof (ibtl_hca_t));
584 } else
585 mutex_exit(&ibtl_async_mutex);
586
587 mutex_enter(&ibtl_clnt_list_mutex);
588 if (--hca_devp->hd_async_task_cnt == 0)
589 cv_signal(&hca_devp->hd_async_task_cv);
590 if (--clntp->clnt_async_cnt == 0)
591 cv_broadcast(&ibtl_clnt_cv);
592
593 mutex_exit(&ibtl_clnt_list_mutex);
594 }
595
596 /*
597 * Asyncs for HCA level events.
598 *
599 * The function continues to run until there are no more async
600 * events/errors for this HCA. An event is chosen for dispatch
601 * to all clients of this HCA. This thread dispatches them via
602 * the ibtl_async_taskq, then sleeps until all tasks are done.
603 *
604 * This thread records the async_code and async_event in the
605 * ibtl_hca_devinfo_t for all client taskq threads to reference.
606 *
607 * This is called from an async or taskq thread with ibtl_async_mutex held.
608 */
609 static void
610 ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
611 {
612 ibtl_hca_t *ibt_hca;
613 ibt_async_event_t *eventp;
614 ibt_async_code_t code;
615 ibtl_async_port_status_t temp;
616 uint8_t nports;
617 uint8_t port_minus1;
618 ibtl_async_port_event_t *portp;
619
620 mutex_exit(&ibtl_async_mutex);
621
622 mutex_enter(&ibtl_clnt_list_mutex);
623 while (hca_devp->hd_async_busy)
624 cv_wait(&hca_devp->hd_async_busy_cv, &ibtl_clnt_list_mutex);
625 hca_devp->hd_async_busy = 1;
626 mutex_enter(&ibtl_async_mutex);
627
628 bzero(&hca_devp->hd_async_event, sizeof (hca_devp->hd_async_event));
629 for (;;) {
630
631 hca_devp->hd_async_event.ev_fma_ena = 0;
632
633 code = hca_devp->hd_async_codes;
634 if (code & IBT_ERROR_LOCAL_CATASTROPHIC) {
635 code = IBT_ERROR_LOCAL_CATASTROPHIC;
636 hca_devp->hd_async_event.ev_fma_ena =
637 hca_devp->hd_fma_ena;
638 } else if (code & IBT_ERROR_PORT_DOWN) {
639 code = IBT_ERROR_PORT_DOWN;
640 temp = IBTL_HCA_PORT_DOWN;
641 } else if (code & IBT_EVENT_PORT_UP) {
642 code = IBT_EVENT_PORT_UP;
643 temp = IBTL_HCA_PORT_UP;
644 } else if (code & IBT_PORT_CHANGE_EVENT) {
645 code = IBT_PORT_CHANGE_EVENT;
646 temp = IBTL_HCA_PORT_CHG;
647 } else if (code & IBT_CLNT_REREG_EVENT) {
648 code = IBT_CLNT_REREG_EVENT;
649 temp = IBTL_HCA_PORT_ASYNC_CLNT_REREG;
650 } else {
651 hca_devp->hd_async_codes = 0;
652 code = 0;
653 }
654
655 if (code == 0) {
656 hca_devp->hd_async_flags &= ~IBTL_ASYNC_PENDING;
657 break;
658 }
659 hca_devp->hd_async_codes &= ~code;
660
661 /* PORT_UP, PORT_CHANGE, PORT_DOWN or ASYNC_REREG */
662 if ((code & IBT_PORT_EVENTS) != 0) {
663 portp = hca_devp->hd_async_port;
664 nports = hca_devp->hd_hca_attr->hca_nports;
665 for (port_minus1 = 0; port_minus1 < nports;
666 port_minus1++) {
667 /*
668 * Matching event in this port, let's go handle
669 * it.
670 */
671 if ((portp[port_minus1].status & temp) != 0)
672 break;
673 }
674 if (port_minus1 >= nports) {
675 /* we checked again, but found nothing */
676 continue;
677 }
678 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_do_hca_asyncs: "
679 "async: port# %x code %x", port_minus1 + 1, code);
680 /* mark it to check for other ports after we're done */
681 hca_devp->hd_async_codes |= code;
682
683 /*
684 * Copy the event information into hca_devp and clear
685 * event information from the per port data.
686 */
687 hca_devp->hd_async_event.ev_port = port_minus1 + 1;
688 if (temp == IBTL_HCA_PORT_CHG) {
689 hca_devp->hd_async_event.ev_port_flags =
690 hca_devp->hd_async_port[port_minus1].flags;
691 hca_devp->hd_async_port[port_minus1].flags = 0;
692 }
693 hca_devp->hd_async_port[port_minus1].status &= ~temp;
694
695 mutex_exit(&ibtl_async_mutex);
696 ibtl_reinit_hca_portinfo(hca_devp, port_minus1 + 1);
697 mutex_enter(&ibtl_async_mutex);
698 eventp = &hca_devp->hd_async_event;
699 eventp->ev_hca_guid =
700 hca_devp->hd_hca_attr->hca_node_guid;
701 }
702
703 hca_devp->hd_async_code = code;
704 hca_devp->hd_async_event.ev_hca_guid =
705 hca_devp->hd_hca_attr->hca_node_guid;
706 mutex_exit(&ibtl_async_mutex);
707
708 /*
709 * Make sure to inform CM, DM, and IBMA if we know of them.
710 * Also, make sure not to inform them a second time, which
711 * would occur if they have the HCA open.
712 */
713
714 if (ibtl_ibma_async_handler)
715 ibtl_tell_mgr(hca_devp, ibtl_ibma_async_handler,
716 ibtl_ibma_clnt_private);
717 /* wait for all tasks to complete */
718 while (hca_devp->hd_async_task_cnt != 0)
719 cv_wait(&hca_devp->hd_async_task_cv,
720 &ibtl_clnt_list_mutex);
721
722 /*
723 * Hack Alert:
724 * The ibmf handler would have updated the Master SM LID if it
725 * was SM LID change event. Now lets check if the new Master SM
726 * is a Embedded Cisco Topspin SM.
727 */
728 if ((code == IBT_PORT_CHANGE_EVENT) &&
729 eventp->ev_port_flags & IBT_PORT_CHANGE_SM_LID)
730 ibtl_cm_get_node_info(hca_devp,
731 (ibt_async_handler_t)ibtl_node_info_cb);
732 /* wait for node info task to complete */
733 while (hca_devp->hd_async_task_cnt != 0)
734 cv_wait(&hca_devp->hd_async_task_cv,
735 &ibtl_clnt_list_mutex);
736
737 if (ibtl_dm_async_handler)
738 ibtl_tell_mgr(hca_devp, ibtl_dm_async_handler,
739 ibtl_dm_clnt_private);
740 if (ibtl_cm_async_handler)
741 ibtl_tell_mgr(hca_devp, ibtl_cm_async_handler,
742 ibtl_cm_clnt_private);
743 /* wait for all tasks to complete */
744 while (hca_devp->hd_async_task_cnt != 0)
745 cv_wait(&hca_devp->hd_async_task_cv,
746 &ibtl_clnt_list_mutex);
747
748 for (ibt_hca = hca_devp->hd_clnt_list;
749 ibt_hca != NULL;
750 ibt_hca = ibt_hca->ha_clnt_link) {
751
752 /* Managers are handled above */
753 if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
754 ibtl_cm_async_handler)
755 continue;
756 if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
757 ibtl_dm_async_handler)
758 continue;
759 if (IBTL_HCA2MODI_P(ibt_hca)->mi_async_handler ==
760 ibtl_ibma_async_handler)
761 continue;
762 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
763
764 mutex_enter(&ibtl_async_mutex);
765 ibt_hca->ha_async_cnt++;
766 mutex_exit(&ibtl_async_mutex);
767 hca_devp->hd_async_task_cnt++;
768 (void) taskq_dispatch(ibtl_async_taskq,
769 ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
770 }
771
772 /* wait for all tasks to complete */
773 while (hca_devp->hd_async_task_cnt != 0)
774 cv_wait(&hca_devp->hd_async_task_cv,
775 &ibtl_clnt_list_mutex);
776
777 mutex_enter(&ibtl_async_mutex);
778 }
779 hca_devp->hd_async_code = 0;
780 hca_devp->hd_async_busy = 0;
781 cv_broadcast(&hca_devp->hd_async_busy_cv);
782 mutex_exit(&ibtl_clnt_list_mutex);
783 }
784
785 /*
786 * Asyncs for QP objects.
787 *
788 * The function continues to run until there are no more async
789 * events/errors for this object.
790 */
791 static void
792 ibtl_do_qp_asyncs(ibtl_qp_t *ibtl_qp)
793 {
794 ibt_async_code_t code;
795 ibt_async_event_t async_event;
796
797 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
798 bzero(&async_event, sizeof (async_event));
799 async_event.ev_chan_hdl = IBTL_QP2CHAN(ibtl_qp);
800
801 while ((code = ibtl_qp->qp_async_codes) != 0) {
802 async_event.ev_fma_ena = 0;
803 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_FREE_OBJECT)
804 code = 0; /* fallthrough to "kmem_free" */
805 else if (code & IBT_ERROR_CATASTROPHIC_QP) {
806 code = IBT_ERROR_CATASTROPHIC_QP;
807 async_event.ev_fma_ena = ibtl_qp->qp_cat_fma_ena;
808 } else if (code & IBT_ERROR_INVALID_REQUEST_QP) {
809 code = IBT_ERROR_INVALID_REQUEST_QP;
810 async_event.ev_fma_ena = ibtl_qp->qp_inv_fma_ena;
811 } else if (code & IBT_ERROR_ACCESS_VIOLATION_QP) {
812 code = IBT_ERROR_ACCESS_VIOLATION_QP;
813 async_event.ev_fma_ena = ibtl_qp->qp_acc_fma_ena;
814 } else if (code & IBT_ERROR_PATH_MIGRATE_REQ_QP) {
815 code = IBT_ERROR_PATH_MIGRATE_REQ_QP;
816 async_event.ev_fma_ena = ibtl_qp->qp_pth_fma_ena;
817 } else if (code & IBT_EVENT_PATH_MIGRATED_QP)
818 code = IBT_EVENT_PATH_MIGRATED_QP;
819 else if (code & IBT_EVENT_SQD)
820 code = IBT_EVENT_SQD;
821 else if (code & IBT_EVENT_COM_EST_QP)
822 code = IBT_EVENT_COM_EST_QP;
823 else if (code & IBT_EVENT_EMPTY_QP)
824 code = IBT_EVENT_EMPTY_QP;
825 else {
826 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_qp_asyncs: "
827 "async: unexpected QP async code 0x%x", code);
828 ibtl_qp->qp_async_codes = 0;
829 code = 0;
830 }
831 ibtl_qp->qp_async_codes &= ~code;
832
833 if (code) {
834 mutex_exit(&ibtl_async_mutex);
835 ibtl_async_client_call(ibtl_qp->qp_hca,
836 code, &async_event);
837 mutex_enter(&ibtl_async_mutex);
838 }
839
840 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_FREE_OBJECT) {
841 mutex_exit(&ibtl_async_mutex);
842 cv_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_cv);
843 mutex_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_mutex);
844 kmem_free(IBTL_QP2CHAN(ibtl_qp),
845 sizeof (ibtl_channel_t));
846 mutex_enter(&ibtl_async_mutex);
847 return;
848 }
849 }
850 ibtl_qp->qp_async_flags &= ~IBTL_ASYNC_PENDING;
851 }
852
853 /*
854 * Asyncs for SRQ objects.
855 *
856 * The function continues to run until there are no more async
857 * events/errors for this object.
858 */
859 static void
860 ibtl_do_srq_asyncs(ibtl_srq_t *ibtl_srq)
861 {
862 ibt_async_code_t code;
863 ibt_async_event_t async_event;
864
865 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
866 bzero(&async_event, sizeof (async_event));
867 async_event.ev_srq_hdl = ibtl_srq;
868 async_event.ev_fma_ena = ibtl_srq->srq_fma_ena;
869
870 while ((code = ibtl_srq->srq_async_codes) != 0) {
871 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_FREE_OBJECT)
872 code = 0; /* fallthrough to "kmem_free" */
873 else if (code & IBT_ERROR_CATASTROPHIC_SRQ)
874 code = IBT_ERROR_CATASTROPHIC_SRQ;
875 else if (code & IBT_EVENT_LIMIT_REACHED_SRQ)
876 code = IBT_EVENT_LIMIT_REACHED_SRQ;
877 else {
878 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_srq_asyncs: "
879 "async: unexpected SRQ async code 0x%x", code);
880 ibtl_srq->srq_async_codes = 0;
881 code = 0;
882 }
883 ibtl_srq->srq_async_codes &= ~code;
884
885 if (code) {
886 mutex_exit(&ibtl_async_mutex);
887 ibtl_async_client_call(ibtl_srq->srq_hca,
888 code, &async_event);
889 mutex_enter(&ibtl_async_mutex);
890 }
891
892 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_FREE_OBJECT) {
893 mutex_exit(&ibtl_async_mutex);
894 kmem_free(ibtl_srq, sizeof (struct ibtl_srq_s));
895 mutex_enter(&ibtl_async_mutex);
896 return;
897 }
898 }
899 ibtl_srq->srq_async_flags &= ~IBTL_ASYNC_PENDING;
900 }
901
902 /*
903 * Asyncs for CQ objects.
904 *
905 * The function continues to run until there are no more async
906 * events/errors for this object.
907 */
908 static void
909 ibtl_do_cq_asyncs(ibtl_cq_t *ibtl_cq)
910 {
911 ibt_async_code_t code;
912 ibt_async_event_t async_event;
913
914 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
915 bzero(&async_event, sizeof (async_event));
916 async_event.ev_cq_hdl = ibtl_cq;
917 async_event.ev_fma_ena = ibtl_cq->cq_fma_ena;
918
919 while ((code = ibtl_cq->cq_async_codes) != 0) {
920 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_FREE_OBJECT)
921 code = 0; /* fallthrough to "kmem_free" */
922 else if (code & IBT_ERROR_CQ)
923 code = IBT_ERROR_CQ;
924 else {
925 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_cq_asyncs: "
926 "async: unexpected CQ async code 0x%x", code);
927 ibtl_cq->cq_async_codes = 0;
928 code = 0;
929 }
930 ibtl_cq->cq_async_codes &= ~code;
931
932 if (code) {
933 mutex_exit(&ibtl_async_mutex);
934 ibtl_async_client_call(ibtl_cq->cq_hca,
935 code, &async_event);
936 mutex_enter(&ibtl_async_mutex);
937 }
938
939 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_FREE_OBJECT) {
940 mutex_exit(&ibtl_async_mutex);
941 mutex_destroy(&ibtl_cq->cq_mutex);
942 kmem_free(ibtl_cq, sizeof (struct ibtl_cq_s));
943 mutex_enter(&ibtl_async_mutex);
944 return;
945 }
946 }
947 ibtl_cq->cq_async_flags &= ~IBTL_ASYNC_PENDING;
948 }
949
950 /*
951 * Asyncs for EEC objects.
952 *
953 * The function continues to run until there are no more async
954 * events/errors for this object.
955 */
956 static void
957 ibtl_do_eec_asyncs(ibtl_eec_t *ibtl_eec)
958 {
959 ibt_async_code_t code;
960 ibt_async_event_t async_event;
961
962 ASSERT(MUTEX_HELD(&ibtl_async_mutex));
963 bzero(&async_event, sizeof (async_event));
964 async_event.ev_chan_hdl = ibtl_eec->eec_channel;
965
966 while ((code = ibtl_eec->eec_async_codes) != 0) {
967 async_event.ev_fma_ena = 0;
968 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_FREE_OBJECT)
969 code = 0; /* fallthrough to "kmem_free" */
970 else if (code & IBT_ERROR_CATASTROPHIC_EEC) {
971 code = IBT_ERROR_CATASTROPHIC_CHAN;
972 async_event.ev_fma_ena = ibtl_eec->eec_cat_fma_ena;
973 } else if (code & IBT_ERROR_PATH_MIGRATE_REQ_EEC) {
974 code = IBT_ERROR_PATH_MIGRATE_REQ;
975 async_event.ev_fma_ena = ibtl_eec->eec_pth_fma_ena;
976 } else if (code & IBT_EVENT_PATH_MIGRATED_EEC)
977 code = IBT_EVENT_PATH_MIGRATED;
978 else if (code & IBT_EVENT_COM_EST_EEC)
979 code = IBT_EVENT_COM_EST;
980 else {
981 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_do_eec_asyncs: "
982 "async: unexpected code 0x%x", code);
983 ibtl_eec->eec_async_codes = 0;
984 code = 0;
985 }
986 ibtl_eec->eec_async_codes &= ~code;
987
988 if (code) {
989 mutex_exit(&ibtl_async_mutex);
990 ibtl_async_client_call(ibtl_eec->eec_hca,
991 code, &async_event);
992 mutex_enter(&ibtl_async_mutex);
993 }
994
995 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_FREE_OBJECT) {
996 mutex_exit(&ibtl_async_mutex);
997 kmem_free(ibtl_eec, sizeof (struct ibtl_eec_s));
998 mutex_enter(&ibtl_async_mutex);
999 return;
1000 }
1001 }
1002 ibtl_eec->eec_async_flags &= ~IBTL_ASYNC_PENDING;
1003 }
1004
1005 /*
1006 * Loop forever, calling async_handlers until all of the async lists
1007 * are empty.
1008 */
1009
1010 static void
1011 ibtl_async_thread(void)
1012 {
1013 kmutex_t cpr_mutex;
1014 callb_cpr_t cprinfo;
1015
1016 mutex_init(&cpr_mutex, NULL, MUTEX_DRIVER, NULL);
1017 CALLB_CPR_INIT(&cprinfo, &cpr_mutex, callb_generic_cpr,
1018 "ibtl_async_thread");
1019
1020 mutex_enter(&ibtl_async_mutex);
1021
1022 for (;;) {
1023 if (ibtl_async_hca_list_start) {
1024 ibtl_hca_devinfo_t *hca_devp;
1025
1026 /* remove first entry from list */
1027 hca_devp = ibtl_async_hca_list_start;
1028 ibtl_async_hca_list_start = hca_devp->hd_async_link;
1029 hca_devp->hd_async_link = NULL;
1030 if (ibtl_async_hca_list_start == NULL)
1031 ibtl_async_hca_list_end = NULL;
1032
1033 ibtl_do_hca_asyncs(hca_devp);
1034
1035 } else if (ibtl_async_qp_list_start) {
1036 ibtl_qp_t *ibtl_qp;
1037
1038 /* remove from list */
1039 ibtl_qp = ibtl_async_qp_list_start;
1040 ibtl_async_qp_list_start = ibtl_qp->qp_async_link;
1041 ibtl_qp->qp_async_link = NULL;
1042 if (ibtl_async_qp_list_start == NULL)
1043 ibtl_async_qp_list_end = NULL;
1044
1045 ibtl_do_qp_asyncs(ibtl_qp);
1046
1047 } else if (ibtl_async_srq_list_start) {
1048 ibtl_srq_t *ibtl_srq;
1049
1050 /* remove from list */
1051 ibtl_srq = ibtl_async_srq_list_start;
1052 ibtl_async_srq_list_start = ibtl_srq->srq_async_link;
1053 ibtl_srq->srq_async_link = NULL;
1054 if (ibtl_async_srq_list_start == NULL)
1055 ibtl_async_srq_list_end = NULL;
1056
1057 ibtl_do_srq_asyncs(ibtl_srq);
1058
1059 } else if (ibtl_async_eec_list_start) {
1060 ibtl_eec_t *ibtl_eec;
1061
1062 /* remove from list */
1063 ibtl_eec = ibtl_async_eec_list_start;
1064 ibtl_async_eec_list_start = ibtl_eec->eec_async_link;
1065 ibtl_eec->eec_async_link = NULL;
1066 if (ibtl_async_eec_list_start == NULL)
1067 ibtl_async_eec_list_end = NULL;
1068
1069 ibtl_do_eec_asyncs(ibtl_eec);
1070
1071 } else if (ibtl_async_cq_list_start) {
1072 ibtl_cq_t *ibtl_cq;
1073
1074 /* remove from list */
1075 ibtl_cq = ibtl_async_cq_list_start;
1076 ibtl_async_cq_list_start = ibtl_cq->cq_async_link;
1077 ibtl_cq->cq_async_link = NULL;
1078 if (ibtl_async_cq_list_start == NULL)
1079 ibtl_async_cq_list_end = NULL;
1080
1081 ibtl_do_cq_asyncs(ibtl_cq);
1082
1083 } else {
1084 if (ibtl_async_thread_exit == IBTL_THREAD_EXIT)
1085 break;
1086 mutex_enter(&cpr_mutex);
1087 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1088 mutex_exit(&cpr_mutex);
1089
1090 cv_wait(&ibtl_async_cv, &ibtl_async_mutex);
1091
1092 mutex_exit(&ibtl_async_mutex);
1093 mutex_enter(&cpr_mutex);
1094 CALLB_CPR_SAFE_END(&cprinfo, &cpr_mutex);
1095 mutex_exit(&cpr_mutex);
1096 mutex_enter(&ibtl_async_mutex);
1097 }
1098 }
1099
1100 mutex_exit(&ibtl_async_mutex);
1101
1102 mutex_enter(&cpr_mutex);
1103 CALLB_CPR_EXIT(&cprinfo);
1104 mutex_destroy(&cpr_mutex);
1105 }
1106
1107
1108 void
1109 ibtl_free_qp_async_check(ibtl_qp_t *ibtl_qp)
1110 {
1111 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_qp_async_check(%p)", ibtl_qp);
1112
1113 mutex_enter(&ibtl_async_mutex);
1114
1115 /*
1116 * If there is an active async, mark this object to be freed
1117 * by the async_thread when it's done.
1118 */
1119 if (ibtl_qp->qp_async_flags & IBTL_ASYNC_PENDING) {
1120 ibtl_qp->qp_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1121 mutex_exit(&ibtl_async_mutex);
1122 } else { /* free the object now */
1123 mutex_exit(&ibtl_async_mutex);
1124 cv_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_cv);
1125 mutex_destroy(&(IBTL_QP2CHAN(ibtl_qp))->ch_cm_mutex);
1126 kmem_free(IBTL_QP2CHAN(ibtl_qp), sizeof (ibtl_channel_t));
1127 }
1128 }
1129
1130 void
1131 ibtl_free_cq_async_check(ibtl_cq_t *ibtl_cq)
1132 {
1133 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_cq_async_check(%p)", ibtl_cq);
1134
1135 mutex_enter(&ibtl_async_mutex);
1136
1137 /* if there is an active async, mark this object to be freed */
1138 if (ibtl_cq->cq_async_flags & IBTL_ASYNC_PENDING) {
1139 ibtl_cq->cq_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1140 mutex_exit(&ibtl_async_mutex);
1141 } else { /* free the object now */
1142 mutex_exit(&ibtl_async_mutex);
1143 mutex_destroy(&ibtl_cq->cq_mutex);
1144 kmem_free(ibtl_cq, sizeof (struct ibtl_cq_s));
1145 }
1146 }
1147
1148 void
1149 ibtl_free_srq_async_check(ibtl_srq_t *ibtl_srq)
1150 {
1151 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_srq_async_check(%p)",
1152 ibtl_srq);
1153
1154 mutex_enter(&ibtl_async_mutex);
1155
1156 /* if there is an active async, mark this object to be freed */
1157 if (ibtl_srq->srq_async_flags & IBTL_ASYNC_PENDING) {
1158 ibtl_srq->srq_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1159 mutex_exit(&ibtl_async_mutex);
1160 } else { /* free the object now */
1161 mutex_exit(&ibtl_async_mutex);
1162 kmem_free(ibtl_srq, sizeof (struct ibtl_srq_s));
1163 }
1164 }
1165
1166 void
1167 ibtl_free_eec_async_check(ibtl_eec_t *ibtl_eec)
1168 {
1169 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_eec_async_check(%p)",
1170 ibtl_eec);
1171
1172 mutex_enter(&ibtl_async_mutex);
1173
1174 /* if there is an active async, mark this object to be freed */
1175 if (ibtl_eec->eec_async_flags & IBTL_ASYNC_PENDING) {
1176 ibtl_eec->eec_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1177 mutex_exit(&ibtl_async_mutex);
1178 } else { /* free the object now */
1179 mutex_exit(&ibtl_async_mutex);
1180 kmem_free(ibtl_eec, sizeof (struct ibtl_eec_s));
1181 }
1182 }
1183
1184 /*
1185 * This function differs from above in that we assume this is called
1186 * from non-interrupt context, and never called from the async_thread.
1187 */
1188
1189 void
1190 ibtl_free_hca_async_check(ibtl_hca_t *ibt_hca)
1191 {
1192 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_hca_async_check(%p)",
1193 ibt_hca);
1194
1195 mutex_enter(&ibtl_async_mutex);
1196
1197 /* if there is an active async, mark this object to be freed */
1198 if (ibt_hca->ha_async_cnt > 0) {
1199 ibt_hca->ha_async_flags |= IBTL_ASYNC_FREE_OBJECT;
1200 mutex_exit(&ibtl_async_mutex);
1201 } else { /* free the object now */
1202 mutex_exit(&ibtl_async_mutex);
1203 kmem_free(ibt_hca, sizeof (ibtl_hca_t));
1204 }
1205 }
1206
1207 /*
1208 * Completion Queue Handling.
1209 *
1210 * A completion queue can be handled through a simple callback
1211 * at interrupt level, or it may be queued for an ibtl_cq_thread
1212 * to handle. The latter is chosen during ibt_alloc_cq when the
1213 * IBTF_CQ_HANDLER_IN_THREAD is specified.
1214 */
1215
1216 static void
1217 ibtl_cq_handler_call(ibtl_cq_t *ibtl_cq)
1218 {
1219 ibt_cq_handler_t cq_handler;
1220 void *arg;
1221
1222 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_cq_handler_call(%p)", ibtl_cq);
1223
1224 cq_handler = ibtl_cq->cq_comp_handler;
1225 arg = ibtl_cq->cq_arg;
1226 if (cq_handler != NULL)
1227 cq_handler(ibtl_cq, arg);
1228 else
1229 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_cq_handler_call: "
1230 "no cq_handler for cq %p", ibtl_cq);
1231 }
1232
1233 /*
1234 * Before ibt_free_cq can continue, we need to ensure no more cq_handler
1235 * callbacks can occur. When we get the mutex, we know there are no
1236 * outstanding cq_handler callbacks. We set the cq_handler to NULL to
1237 * prohibit future callbacks.
1238 */
1239 void
1240 ibtl_free_cq_check(ibtl_cq_t *ibtl_cq)
1241 {
1242 mutex_enter(&ibtl_cq->cq_mutex);
1243 ibtl_cq->cq_comp_handler = NULL;
1244 mutex_exit(&ibtl_cq->cq_mutex);
1245 if (ibtl_cq->cq_in_thread) {
1246 mutex_enter(&ibtl_cq_mutex);
1247 --ibtl_cqs_using_threads;
1248 while (ibtl_cq->cq_impl_flags & IBTL_CQ_PENDING) {
1249 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_CALL_CLIENT;
1250 ibtl_cq->cq_impl_flags |= IBTL_CQ_FREE;
1251 cv_wait(&ibtl_cq_cv, &ibtl_cq_mutex);
1252 }
1253 mutex_exit(&ibtl_cq_mutex);
1254 }
1255 }
1256
1257 /*
1258 * Loop forever, calling cq_handlers until the cq list
1259 * is empty.
1260 */
1261
1262 static void
1263 ibtl_cq_thread(void)
1264 {
1265 kmutex_t cpr_mutex;
1266 callb_cpr_t cprinfo;
1267
1268 mutex_init(&cpr_mutex, NULL, MUTEX_DRIVER, NULL);
1269 CALLB_CPR_INIT(&cprinfo, &cpr_mutex, callb_generic_cpr,
1270 "ibtl_cq_thread");
1271
1272 mutex_enter(&ibtl_cq_mutex);
1273
1274 for (;;) {
1275 if (ibtl_cq_list_start) {
1276 ibtl_cq_t *ibtl_cq;
1277
1278 ibtl_cq = ibtl_cq_list_start;
1279 ibtl_cq_list_start = ibtl_cq->cq_link;
1280 ibtl_cq->cq_link = NULL;
1281 if (ibtl_cq == ibtl_cq_list_end)
1282 ibtl_cq_list_end = NULL;
1283
1284 while (ibtl_cq->cq_impl_flags & IBTL_CQ_CALL_CLIENT) {
1285 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_CALL_CLIENT;
1286 mutex_exit(&ibtl_cq_mutex);
1287 ibtl_cq_handler_call(ibtl_cq);
1288 mutex_enter(&ibtl_cq_mutex);
1289 }
1290 ibtl_cq->cq_impl_flags &= ~IBTL_CQ_PENDING;
1291 if (ibtl_cq->cq_impl_flags & IBTL_CQ_FREE)
1292 cv_broadcast(&ibtl_cq_cv);
1293 } else {
1294 if (ibtl_cq_thread_exit == IBTL_THREAD_EXIT)
1295 break;
1296 mutex_enter(&cpr_mutex);
1297 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1298 mutex_exit(&cpr_mutex);
1299
1300 cv_wait(&ibtl_cq_cv, &ibtl_cq_mutex);
1301
1302 mutex_exit(&ibtl_cq_mutex);
1303 mutex_enter(&cpr_mutex);
1304 CALLB_CPR_SAFE_END(&cprinfo, &cpr_mutex);
1305 mutex_exit(&cpr_mutex);
1306 mutex_enter(&ibtl_cq_mutex);
1307 }
1308 }
1309
1310 mutex_exit(&ibtl_cq_mutex);
1311 mutex_enter(&cpr_mutex);
1312 CALLB_CPR_EXIT(&cprinfo);
1313 mutex_destroy(&cpr_mutex);
1314 }
1315
1316
1317 /*
1318 * ibc_cq_handler()
1319 *
1320 * Completion Queue Notification Handler.
1321 *
1322 */
1323 /*ARGSUSED*/
1324 void
1325 ibc_cq_handler(ibc_clnt_hdl_t ibc_hdl, ibt_cq_hdl_t ibtl_cq)
1326 {
1327 IBTF_DPRINTF_L4(ibtf_handlers, "ibc_cq_handler(%p, %p)",
1328 ibc_hdl, ibtl_cq);
1329
1330 if (ibtl_cq->cq_in_thread) {
1331 mutex_enter(&ibtl_cq_mutex);
1332 ibtl_cq->cq_impl_flags |= IBTL_CQ_CALL_CLIENT;
1333 if ((ibtl_cq->cq_impl_flags & IBTL_CQ_PENDING) == 0) {
1334 ibtl_cq->cq_impl_flags |= IBTL_CQ_PENDING;
1335 ibtl_cq->cq_link = NULL;
1336 if (ibtl_cq_list_end == NULL)
1337 ibtl_cq_list_start = ibtl_cq;
1338 else
1339 ibtl_cq_list_end->cq_link = ibtl_cq;
1340 ibtl_cq_list_end = ibtl_cq;
1341 cv_signal(&ibtl_cq_cv);
1342 }
1343 mutex_exit(&ibtl_cq_mutex);
1344 return;
1345 } else
1346 ibtl_cq_handler_call(ibtl_cq);
1347 }
1348
1349
1350 /*
1351 * ibt_enable_cq_notify()
1352 * Enable Notification requests on the specified CQ.
1353 *
1354 * ibt_cq The CQ handle.
1355 *
1356 * notify_type Enable notifications for all (IBT_NEXT_COMPLETION)
1357 * completions, or the next Solicited completion
1358 * (IBT_NEXT_SOLICITED) only.
1359 *
1360 * Completion notifications are disabled by setting the completion
1361 * handler to NULL by calling ibt_set_cq_handler().
1362 */
1363 ibt_status_t
1364 ibt_enable_cq_notify(ibt_cq_hdl_t ibtl_cq, ibt_cq_notify_flags_t notify_type)
1365 {
1366 IBTF_DPRINTF_L3(ibtf_handlers, "ibt_enable_cq_notify(%p, %d)",
1367 ibtl_cq, notify_type);
1368
1369 return (IBTL_CQ2CIHCAOPS_P(ibtl_cq)->ibc_notify_cq(
1370 IBTL_CQ2CIHCA(ibtl_cq), ibtl_cq->cq_ibc_cq_hdl, notify_type));
1371 }
1372
1373
1374 /*
1375 * ibt_set_cq_handler()
1376 * Register a work request completion handler with the IBTF.
1377 *
1378 * ibt_cq The CQ handle.
1379 *
1380 * completion_handler The completion handler.
1381 *
1382 * arg The IBTF client private argument to be passed
1383 * back to the client when calling the CQ
1384 * completion handler.
1385 *
1386 * Completion notifications are disabled by setting the completion
1387 * handler to NULL. When setting the handler to NULL, no additional
1388 * calls to the previous CQ handler will be initiated, but there may
1389 * be one in progress.
1390 *
1391 * This function does not otherwise change the state of previous
1392 * calls to ibt_enable_cq_notify().
1393 */
1394 void
1395 ibt_set_cq_handler(ibt_cq_hdl_t ibtl_cq, ibt_cq_handler_t completion_handler,
1396 void *arg)
1397 {
1398 IBTF_DPRINTF_L3(ibtf_handlers, "ibt_set_cq_handler(%p, %p, %p)",
1399 ibtl_cq, completion_handler, arg);
1400
1401 ibtl_cq->cq_comp_handler = completion_handler;
1402 ibtl_cq->cq_arg = arg;
1403 }
1404
1405
1406 /*
1407 * Inform IBT clients about New HCAs.
1408 *
1409 * We use taskqs to allow simultaneous notification, with sleeping.
1410 * Since taskqs only allow one argument, we define a structure
1411 * because we need to pass in two arguments.
1412 */
1413
1414 struct ibtl_new_hca_s {
1415 ibtl_clnt_t *nh_clntp;
1416 ibtl_hca_devinfo_t *nh_hca_devp;
1417 ibt_async_code_t nh_code;
1418 };
1419
1420 static void
1421 ibtl_tell_client_about_new_hca(void *arg)
1422 {
1423 struct ibtl_new_hca_s *new_hcap = (struct ibtl_new_hca_s *)arg;
1424 ibtl_clnt_t *clntp = new_hcap->nh_clntp;
1425 ibt_async_event_t async_event;
1426 ibtl_hca_devinfo_t *hca_devp = new_hcap->nh_hca_devp;
1427
1428 bzero(&async_event, sizeof (async_event));
1429 async_event.ev_hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
1430 clntp->clnt_modinfop->mi_async_handler(
1431 clntp->clnt_private, NULL, new_hcap->nh_code, &async_event);
1432 kmem_free(new_hcap, sizeof (*new_hcap));
1433 mutex_enter(&ibtl_clnt_list_mutex);
1434 if (--hca_devp->hd_async_task_cnt == 0)
1435 cv_signal(&hca_devp->hd_async_task_cv);
1436 if (--clntp->clnt_async_cnt == 0)
1437 cv_broadcast(&ibtl_clnt_cv);
1438 mutex_exit(&ibtl_clnt_list_mutex);
1439 }
1440
1441 /*
1442 * ibtl_announce_new_hca:
1443 *
1444 * o First attach these clients in the given order
1445 * IBMA
1446 * IBCM
1447 *
1448 * o Next attach all other clients in parallel.
1449 *
1450 * NOTE: Use the taskq to simultaneously notify all clients of the new HCA.
1451 * Retval from clients is ignored.
1452 */
1453 void
1454 ibtl_announce_new_hca(ibtl_hca_devinfo_t *hca_devp)
1455 {
1456 ibtl_clnt_t *clntp;
1457 struct ibtl_new_hca_s *new_hcap;
1458
1459 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_announce_new_hca(%p, %llX)",
1460 hca_devp, hca_devp->hd_hca_attr->hca_node_guid);
1461
1462 mutex_enter(&ibtl_clnt_list_mutex);
1463
1464 clntp = ibtl_clnt_list;
1465 while (clntp != NULL) {
1466 if (clntp->clnt_modinfop->mi_clnt_class == IBT_IBMA) {
1467 IBTF_DPRINTF_L4(ibtf_handlers,
1468 "ibtl_announce_new_hca: calling IBMF");
1469 if (clntp->clnt_modinfop->mi_async_handler) {
1470 new_hcap = kmem_alloc(sizeof (*new_hcap),
1471 KM_SLEEP);
1472 new_hcap->nh_clntp = clntp;
1473 new_hcap->nh_hca_devp = hca_devp;
1474 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1475 clntp->clnt_async_cnt++;
1476 hca_devp->hd_async_task_cnt++;
1477
1478 (void) taskq_dispatch(ibtl_async_taskq,
1479 ibtl_tell_client_about_new_hca, new_hcap,
1480 TQ_SLEEP);
1481 }
1482 break;
1483 }
1484 clntp = clntp->clnt_list_link;
1485 }
1486 if (clntp != NULL)
1487 while (clntp->clnt_async_cnt > 0)
1488 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1489 clntp = ibtl_clnt_list;
1490 while (clntp != NULL) {
1491 if (clntp->clnt_modinfop->mi_clnt_class == IBT_DM) {
1492 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_announce_new_hca: "
1493 "calling %s", clntp->clnt_modinfop->mi_clnt_name);
1494 if (clntp->clnt_modinfop->mi_async_handler) {
1495 new_hcap = kmem_alloc(sizeof (*new_hcap),
1496 KM_SLEEP);
1497 new_hcap->nh_clntp = clntp;
1498 new_hcap->nh_hca_devp = hca_devp;
1499 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1500 clntp->clnt_async_cnt++;
1501 hca_devp->hd_async_task_cnt++;
1502
1503 mutex_exit(&ibtl_clnt_list_mutex);
1504 (void) ibtl_tell_client_about_new_hca(
1505 new_hcap);
1506 mutex_enter(&ibtl_clnt_list_mutex);
1507 }
1508 break;
1509 }
1510 clntp = clntp->clnt_list_link;
1511 }
1512
1513 clntp = ibtl_clnt_list;
1514 while (clntp != NULL) {
1515 if (clntp->clnt_modinfop->mi_clnt_class == IBT_CM) {
1516 IBTF_DPRINTF_L4(ibtf_handlers, "ibtl_announce_new_hca: "
1517 "calling %s", clntp->clnt_modinfop->mi_clnt_name);
1518 if (clntp->clnt_modinfop->mi_async_handler) {
1519 new_hcap = kmem_alloc(sizeof (*new_hcap),
1520 KM_SLEEP);
1521 new_hcap->nh_clntp = clntp;
1522 new_hcap->nh_hca_devp = hca_devp;
1523 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1524 clntp->clnt_async_cnt++;
1525 hca_devp->hd_async_task_cnt++;
1526
1527 (void) taskq_dispatch(ibtl_async_taskq,
1528 ibtl_tell_client_about_new_hca, new_hcap,
1529 TQ_SLEEP);
1530 }
1531 break;
1532 }
1533 clntp = clntp->clnt_list_link;
1534 }
1535 if (clntp != NULL)
1536 while (clntp->clnt_async_cnt > 0)
1537 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1538 clntp = ibtl_clnt_list;
1539 while (clntp != NULL) {
1540 if ((clntp->clnt_modinfop->mi_clnt_class != IBT_DM) &&
1541 (clntp->clnt_modinfop->mi_clnt_class != IBT_CM) &&
1542 (clntp->clnt_modinfop->mi_clnt_class != IBT_IBMA)) {
1543 IBTF_DPRINTF_L4(ibtf_handlers,
1544 "ibtl_announce_new_hca: Calling %s ",
1545 clntp->clnt_modinfop->mi_clnt_name);
1546 if (clntp->clnt_modinfop->mi_async_handler) {
1547 new_hcap = kmem_alloc(sizeof (*new_hcap),
1548 KM_SLEEP);
1549 new_hcap->nh_clntp = clntp;
1550 new_hcap->nh_hca_devp = hca_devp;
1551 new_hcap->nh_code = IBT_HCA_ATTACH_EVENT;
1552 clntp->clnt_async_cnt++;
1553 hca_devp->hd_async_task_cnt++;
1554
1555 (void) taskq_dispatch(ibtl_async_taskq,
1556 ibtl_tell_client_about_new_hca, new_hcap,
1557 TQ_SLEEP);
1558 }
1559 }
1560 clntp = clntp->clnt_list_link;
1561 }
1562
1563 /* wait for all tasks to complete */
1564 while (hca_devp->hd_async_task_cnt != 0)
1565 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1566
1567 /* wakeup thread that may be waiting to send an HCA async */
1568 ASSERT(hca_devp->hd_async_busy == 1);
1569 hca_devp->hd_async_busy = 0;
1570 cv_broadcast(&hca_devp->hd_async_busy_cv);
1571 mutex_exit(&ibtl_clnt_list_mutex);
1572 }
1573
1574 /*
1575 * ibtl_detach_all_clients:
1576 *
1577 * Return value - 0 for Success, 1 for Failure
1578 *
1579 * o First detach general clients.
1580 *
1581 * o Next detach these clients
1582 * IBCM
1583 * IBDM
1584 *
1585 * o Finally, detach this client
1586 * IBMA
1587 */
1588 int
1589 ibtl_detach_all_clients(ibtl_hca_devinfo_t *hca_devp)
1590 {
1591 ib_guid_t hcaguid = hca_devp->hd_hca_attr->hca_node_guid;
1592 ibtl_hca_t *ibt_hca;
1593 ibtl_clnt_t *clntp;
1594 int retval;
1595
1596 IBTF_DPRINTF_L2(ibtf_handlers, "ibtl_detach_all_clients(%llX)",
1597 hcaguid);
1598
1599 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex));
1600
1601 while (hca_devp->hd_async_busy)
1602 cv_wait(&hca_devp->hd_async_busy_cv, &ibtl_clnt_list_mutex);
1603 hca_devp->hd_async_busy = 1;
1604
1605 /* First inform general clients asynchronously */
1606 hca_devp->hd_async_event.ev_hca_guid = hcaguid;
1607 hca_devp->hd_async_event.ev_fma_ena = 0;
1608 hca_devp->hd_async_event.ev_chan_hdl = NULL;
1609 hca_devp->hd_async_event.ev_cq_hdl = NULL;
1610 hca_devp->hd_async_code = IBT_HCA_DETACH_EVENT;
1611
1612 ibt_hca = hca_devp->hd_clnt_list;
1613 while (ibt_hca != NULL) {
1614 clntp = ibt_hca->ha_clnt_devp;
1615 if (IBTL_GENERIC_CLIENT(clntp)) {
1616 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1617 mutex_enter(&ibtl_async_mutex);
1618 ibt_hca->ha_async_cnt++;
1619 mutex_exit(&ibtl_async_mutex);
1620 hca_devp->hd_async_task_cnt++;
1621
1622 (void) taskq_dispatch(ibtl_async_taskq,
1623 ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
1624 }
1625 ibt_hca = ibt_hca->ha_clnt_link;
1626 }
1627
1628 /* wait for all clients to complete */
1629 while (hca_devp->hd_async_task_cnt != 0) {
1630 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1631 }
1632 /* Go thru the clients and check if any have not closed this HCA. */
1633 retval = 0;
1634 ibt_hca = hca_devp->hd_clnt_list;
1635 while (ibt_hca != NULL) {
1636 clntp = ibt_hca->ha_clnt_devp;
1637 if (IBTL_GENERIC_CLIENT(clntp)) {
1638 IBTF_DPRINTF_L2(ibtf_handlers,
1639 "ibtl_detach_all_clients: "
1640 "client '%s' failed to close the HCA.",
1641 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1642 retval = 1;
1643 }
1644 ibt_hca = ibt_hca->ha_clnt_link;
1645 }
1646 if (retval == 1)
1647 goto bailout;
1648
1649 /* Next inform IBDM asynchronously */
1650 ibt_hca = hca_devp->hd_clnt_list;
1651 while (ibt_hca != NULL) {
1652 clntp = ibt_hca->ha_clnt_devp;
1653 if (clntp->clnt_modinfop->mi_clnt_class == IBT_DM) {
1654 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1655 mutex_enter(&ibtl_async_mutex);
1656 ibt_hca->ha_async_cnt++;
1657 mutex_exit(&ibtl_async_mutex);
1658 hca_devp->hd_async_task_cnt++;
1659
1660 mutex_exit(&ibtl_clnt_list_mutex);
1661 ibtl_hca_client_async_task(ibt_hca);
1662 mutex_enter(&ibtl_clnt_list_mutex);
1663 break;
1664 }
1665 ibt_hca = ibt_hca->ha_clnt_link;
1666 }
1667
1668 /*
1669 * Next inform IBCM.
1670 * As IBCM doesn't perform ibt_open_hca(), IBCM will not be
1671 * accessible via hca_devp->hd_clnt_list.
1672 * ibtl_cm_async_handler will NOT be NULL, if IBCM is registered.
1673 */
1674 if (ibtl_cm_async_handler) {
1675 ibtl_tell_mgr(hca_devp, ibtl_cm_async_handler,
1676 ibtl_cm_clnt_private);
1677
1678 /* wait for all tasks to complete */
1679 while (hca_devp->hd_async_task_cnt != 0)
1680 cv_wait(&hca_devp->hd_async_task_cv,
1681 &ibtl_clnt_list_mutex);
1682 }
1683
1684 /* Go thru the clients and check if any have not closed this HCA. */
1685 retval = 0;
1686 ibt_hca = hca_devp->hd_clnt_list;
1687 while (ibt_hca != NULL) {
1688 clntp = ibt_hca->ha_clnt_devp;
1689 if (clntp->clnt_modinfop->mi_clnt_class != IBT_IBMA) {
1690 IBTF_DPRINTF_L2(ibtf_handlers,
1691 "ibtl_detach_all_clients: "
1692 "client '%s' failed to close the HCA.",
1693 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1694 retval = 1;
1695 }
1696 ibt_hca = ibt_hca->ha_clnt_link;
1697 }
1698 if (retval == 1)
1699 goto bailout;
1700
1701 /* Finally, inform IBMA */
1702 ibt_hca = hca_devp->hd_clnt_list;
1703 while (ibt_hca != NULL) {
1704 clntp = ibt_hca->ha_clnt_devp;
1705 if (clntp->clnt_modinfop->mi_clnt_class == IBT_IBMA) {
1706 ++ibt_hca->ha_clnt_devp->clnt_async_cnt;
1707 mutex_enter(&ibtl_async_mutex);
1708 ibt_hca->ha_async_cnt++;
1709 mutex_exit(&ibtl_async_mutex);
1710 hca_devp->hd_async_task_cnt++;
1711
1712 (void) taskq_dispatch(ibtl_async_taskq,
1713 ibtl_hca_client_async_task, ibt_hca, TQ_SLEEP);
1714 } else
1715 IBTF_DPRINTF_L2(ibtf_handlers,
1716 "ibtl_detach_all_clients: "
1717 "client '%s' is unexpectedly on the client list",
1718 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1719 ibt_hca = ibt_hca->ha_clnt_link;
1720 }
1721
1722 /* wait for IBMA to complete */
1723 while (hca_devp->hd_async_task_cnt != 0) {
1724 cv_wait(&hca_devp->hd_async_task_cv, &ibtl_clnt_list_mutex);
1725 }
1726
1727 /* Check if this HCA's client list is empty. */
1728 ibt_hca = hca_devp->hd_clnt_list;
1729 if (ibt_hca != NULL) {
1730 IBTF_DPRINTF_L2(ibtf_handlers,
1731 "ibtl_detach_all_clients: "
1732 "client '%s' failed to close the HCA.",
1733 ibt_hca->ha_clnt_devp->clnt_modinfop->mi_clnt_name);
1734 retval = 1;
1735 } else
1736 retval = 0;
1737
1738 bailout:
1739 if (retval) {
1740 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */
1741 mutex_exit(&ibtl_clnt_list_mutex);
1742 ibtl_announce_new_hca(hca_devp);
1743 mutex_enter(&ibtl_clnt_list_mutex);
1744 } else {
1745 hca_devp->hd_async_busy = 0;
1746 cv_broadcast(&hca_devp->hd_async_busy_cv);
1747 }
1748
1749 return (retval);
1750 }
1751
1752 void
1753 ibtl_free_clnt_async_check(ibtl_clnt_t *clntp)
1754 {
1755 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_free_clnt_async_check(%p)", clntp);
1756
1757 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex));
1758
1759 /* wait for all asyncs based on "ibtl_clnt_list" to complete */
1760 while (clntp->clnt_async_cnt != 0) {
1761 cv_wait(&ibtl_clnt_cv, &ibtl_clnt_list_mutex);
1762 }
1763 }
1764
1765 static void
1766 ibtl_dec_clnt_async_cnt(ibtl_clnt_t *clntp)
1767 {
1768 mutex_enter(&ibtl_clnt_list_mutex);
1769 if (--clntp->clnt_async_cnt == 0) {
1770 cv_broadcast(&ibtl_clnt_cv);
1771 }
1772 mutex_exit(&ibtl_clnt_list_mutex);
1773 }
1774
1775 static void
1776 ibtl_inc_clnt_async_cnt(ibtl_clnt_t *clntp)
1777 {
1778 mutex_enter(&ibtl_clnt_list_mutex);
1779 ++clntp->clnt_async_cnt;
1780 mutex_exit(&ibtl_clnt_list_mutex);
1781 }
1782
1783
1784 /*
1785 * Functions and data structures to inform clients that a notification
1786 * has occurred about Multicast Groups that might interest them.
1787 */
1788 struct ibtl_sm_notice {
1789 ibt_clnt_hdl_t np_ibt_hdl;
1790 ib_gid_t np_sgid;
1791 ibt_subnet_event_code_t np_code;
1792 ibt_subnet_event_t np_event;
1793 };
1794
1795 static void
1796 ibtl_sm_notice_task(void *arg)
1797 {
1798 struct ibtl_sm_notice *noticep = (struct ibtl_sm_notice *)arg;
1799 ibt_clnt_hdl_t ibt_hdl = noticep->np_ibt_hdl;
1800 ibt_sm_notice_handler_t sm_notice_handler;
1801
1802 sm_notice_handler = ibt_hdl->clnt_sm_trap_handler;
1803 if (sm_notice_handler != NULL)
1804 sm_notice_handler(ibt_hdl->clnt_sm_trap_handler_arg,
1805 noticep->np_sgid, noticep->np_code, ¬icep->np_event);
1806 kmem_free(noticep, sizeof (*noticep));
1807 ibtl_dec_clnt_async_cnt(ibt_hdl);
1808 }
1809
1810 /*
1811 * Inform the client that MCG notices are not working at this time.
1812 */
1813 void
1814 ibtl_cm_sm_notice_init_failure(ibtl_cm_sm_init_fail_t *ifail)
1815 {
1816 ibt_clnt_hdl_t ibt_hdl = ifail->smf_ibt_hdl;
1817 struct ibtl_sm_notice *noticep;
1818 ib_gid_t *sgidp = &ifail->smf_sgid[0];
1819 int i;
1820
1821 for (i = 0; i < ifail->smf_num_sgids; i++) {
1822 noticep = kmem_zalloc(sizeof (*noticep), KM_SLEEP);
1823 noticep->np_ibt_hdl = ibt_hdl;
1824 noticep->np_sgid = *sgidp++;
1825 noticep->np_code = IBT_SM_EVENT_UNAVAILABLE;
1826 ibtl_inc_clnt_async_cnt(ibt_hdl);
1827 (void) taskq_dispatch(ibtl_async_taskq,
1828 ibtl_sm_notice_task, noticep, TQ_SLEEP);
1829 }
1830 }
1831
1832 /*
1833 * Inform all clients of the event.
1834 */
1835 void
1836 ibtl_cm_sm_notice_handler(ib_gid_t sgid, ibt_subnet_event_code_t code,
1837 ibt_subnet_event_t *event)
1838 {
1839 struct ibtl_sm_notice *noticep;
1840 ibtl_clnt_t *clntp;
1841
1842 mutex_enter(&ibtl_clnt_list_mutex);
1843 clntp = ibtl_clnt_list;
1844 while (clntp != NULL) {
1845 if (clntp->clnt_sm_trap_handler) {
1846 noticep = kmem_zalloc(sizeof (*noticep), KM_SLEEP);
1847 noticep->np_ibt_hdl = clntp;
1848 noticep->np_sgid = sgid;
1849 noticep->np_code = code;
1850 noticep->np_event = *event;
1851 ++clntp->clnt_async_cnt;
1852 (void) taskq_dispatch(ibtl_async_taskq,
1853 ibtl_sm_notice_task, noticep, TQ_SLEEP);
1854 }
1855 clntp = clntp->clnt_list_link;
1856 }
1857 mutex_exit(&ibtl_clnt_list_mutex);
1858 }
1859
1860 /*
1861 * Record the handler for this client.
1862 */
1863 void
1864 ibtl_cm_set_sm_notice_handler(ibt_clnt_hdl_t ibt_hdl,
1865 ibt_sm_notice_handler_t sm_notice_handler, void *private)
1866 {
1867 ibt_hdl->clnt_sm_trap_handler = sm_notice_handler;
1868 ibt_hdl->clnt_sm_trap_handler_arg = private;
1869 }
1870
1871
1872 /*
1873 * ibtl_another_cq_handler_in_thread()
1874 *
1875 * Conditionally increase the number of cq_threads.
1876 * The number of threads grows, based on the number of cqs using threads.
1877 *
1878 * The table below controls the number of threads as follows:
1879 *
1880 * Number of CQs Number of cq_threads
1881 * 0 0
1882 * 1 1
1883 * 2-3 2
1884 * 4-5 3
1885 * 6-9 4
1886 * 10-15 5
1887 * 16-23 6
1888 * 24-31 7
1889 * 32+ 8
1890 */
1891
1892 #define IBTL_CQ_MAXTHREADS 8
1893 static uint8_t ibtl_cq_scaling[IBTL_CQ_MAXTHREADS] = {
1894 1, 2, 4, 6, 10, 16, 24, 32
1895 };
1896
1897 static kt_did_t ibtl_cq_did[IBTL_CQ_MAXTHREADS];
1898
1899 void
1900 ibtl_another_cq_handler_in_thread(void)
1901 {
1902 kthread_t *t;
1903 int my_idx;
1904
1905 mutex_enter(&ibtl_cq_mutex);
1906 if ((ibtl_cq_threads == IBTL_CQ_MAXTHREADS) ||
1907 (++ibtl_cqs_using_threads < ibtl_cq_scaling[ibtl_cq_threads])) {
1908 mutex_exit(&ibtl_cq_mutex);
1909 return;
1910 }
1911 my_idx = ibtl_cq_threads++;
1912 mutex_exit(&ibtl_cq_mutex);
1913 t = thread_create(NULL, 0, ibtl_cq_thread, NULL, 0, &p0, TS_RUN,
1914 ibtl_pri - 1);
1915 ibtl_cq_did[my_idx] = t->t_did; /* save for thread_join() */
1916 }
1917
1918 void
1919 ibtl_thread_init(void)
1920 {
1921 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_init()");
1922
1923 mutex_init(&ibtl_async_mutex, NULL, MUTEX_DEFAULT, NULL);
1924 cv_init(&ibtl_async_cv, NULL, CV_DEFAULT, NULL);
1925 cv_init(&ibtl_clnt_cv, NULL, CV_DEFAULT, NULL);
1926
1927 mutex_init(&ibtl_cq_mutex, NULL, MUTEX_DEFAULT, NULL);
1928 cv_init(&ibtl_cq_cv, NULL, CV_DEFAULT, NULL);
1929 }
1930
1931 void
1932 ibtl_thread_init2(void)
1933 {
1934 int i;
1935 static int initted = 0;
1936 kthread_t *t;
1937
1938 mutex_enter(&ibtl_async_mutex);
1939 if (initted == 1) {
1940 mutex_exit(&ibtl_async_mutex);
1941 return;
1942 }
1943 initted = 1;
1944 mutex_exit(&ibtl_async_mutex);
1945 ibtl_async_did = kmem_zalloc(ibtl_async_thread_init * sizeof (kt_did_t),
1946 KM_SLEEP);
1947
1948 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_init2()");
1949
1950 for (i = 0; i < ibtl_async_thread_init; i++) {
1951 t = thread_create(NULL, 0, ibtl_async_thread, NULL, 0, &p0,
1952 TS_RUN, ibtl_pri - 1);
1953 ibtl_async_did[i] = t->t_did; /* thread_join() */
1954 }
1955 for (i = 0; i < ibtl_cq_threads; i++) {
1956 t = thread_create(NULL, 0, ibtl_cq_thread, NULL, 0, &p0,
1957 TS_RUN, ibtl_pri - 1);
1958 ibtl_cq_did[i] = t->t_did; /* save for thread_join() */
1959 }
1960 }
1961
1962 void
1963 ibtl_thread_fini(void)
1964 {
1965 int i;
1966
1967 IBTF_DPRINTF_L3(ibtf_handlers, "ibtl_thread_fini()");
1968
1969 /* undo the work done by ibtl_thread_init() */
1970
1971 mutex_enter(&ibtl_cq_mutex);
1972 ibtl_cq_thread_exit = IBTL_THREAD_EXIT;
1973 cv_broadcast(&ibtl_cq_cv);
1974 mutex_exit(&ibtl_cq_mutex);
1975
1976 mutex_enter(&ibtl_async_mutex);
1977 ibtl_async_thread_exit = IBTL_THREAD_EXIT;
1978 cv_broadcast(&ibtl_async_cv);
1979 mutex_exit(&ibtl_async_mutex);
1980
1981 for (i = 0; i < ibtl_cq_threads; i++)
1982 thread_join(ibtl_cq_did[i]);
1983
1984 if (ibtl_async_did) {
1985 for (i = 0; i < ibtl_async_thread_init; i++)
1986 thread_join(ibtl_async_did[i]);
1987
1988 kmem_free(ibtl_async_did,
1989 ibtl_async_thread_init * sizeof (kt_did_t));
1990 }
1991 mutex_destroy(&ibtl_cq_mutex);
1992 cv_destroy(&ibtl_cq_cv);
1993
1994 mutex_destroy(&ibtl_async_mutex);
1995 cv_destroy(&ibtl_async_cv);
1996 cv_destroy(&ibtl_clnt_cv);
1997 }
1998
1999 /* ARGSUSED */
2000 ibt_status_t ibtl_dummy_node_info_cb(ib_guid_t hca_guid, uint8_t port,
2001 ib_lid_t lid, ibt_node_info_t *node_info)
2002 {
2003 return (IBT_SUCCESS);
2004 }