1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * ibcm_utils.c
28 *
29 * contains internal lookup functions of IB CM module
30 * along with some other miscellaneous stuff
31 *
32 * TBD:
33 * 1. Code needed to ensure that if any clients are using a service then
34 * don't de-register it.
35 */
36
37 #include <sys/ib/mgt/ibcm/ibcm_impl.h>
38 #include <sys/ddi.h>
39
40
41 /* statics */
42 static vmem_t *ibcm_local_sid_arena;
43 static vmem_t *ibcm_ip_sid_arena;
44 static ib_svc_id_t ibcm_local_sid_seed;
45 static ib_com_id_t ibcm_local_cid_seed;
46 static void ibcm_delete_state_from_avl(ibcm_state_data_t *statep);
47 static void ibcm_init_conn_trace(ibcm_state_data_t *statep);
48 static void ibcm_fini_conn_trace(ibcm_state_data_t *statep);
49 static void ibcm_dump_conn_trbuf(void *statep, char *line_prefix,
50 char *buf, int buf_size);
51 extern ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *,
52 uint64_t c_mask, void *, size_t *);
53
54 /*
55 * ibcm_lookup_msg:
56 *
57 * Retrieves an existing state structure or creates a new one if none found.
58 * This function is used during
59 * Passive connection side for INCOMING REQ/REJ/RTU/MRA/DREQ/DREP/LAP msgs
60 * Active connection side for INCOMING REP/REJ/MRA/DREQ/DREP/APR msgs
61 * Active side CM for outgoing REQ message.
62 *
63 * NOTE: Only return IBCM_LOOKUP_FAIL if lookup failed to find a match.
64 *
65 * Arguments are:-
66 * event_type - type of message
67 * incoming REQ, REP, REJ, MRA, RTU
68 * remote_qpn - Remote QP number
69 * comid - local/remote comid
70 * remote_hca_guid - Remote HCA GUID
71 * hcap - HCA entry ptr
72 * rstatep - return statep pointer
73 *
74 * Return Values:
75 * IBCM_LOOKUP_NEW - new statep allocated
76 * IBCM_LOOKUP_EXISTS - found an existing entry
77 * IBCM_LOOKUP_FAIL - No lookup entry found
78 * IBCM_MEMORY_FAILURE - Memory allocs failed
79 */
80 ibcm_status_t
81 ibcm_lookup_msg(ibcm_event_type_t event_type, ib_com_id_t comid,
82 ib_qpn_t remote_qpn, ib_guid_t remote_hca_guid, ibcm_hca_info_t *hcap,
83 ibcm_state_data_t **rstatep)
84 {
85 avl_index_t where;
86 ibcm_state_data_t *sp;
87
88 IBTF_DPRINTF_L4(cmlog, "ibcm_lookup_msg: event = 0x%x, comid = 0x%x",
89 event_type, comid);
90 IBTF_DPRINTF_L4(cmlog, "ibcm_lookup_msg: rem_qpn = 0x%lX, "
91 "rem_hca_guid = 0x%llX", remote_qpn, remote_hca_guid);
92
93 ASSERT(rw_lock_held(&hcap->hca_state_rwlock));
94
95 /*
96 * Lookup in "hca_passive_tree" for IBCM_INCOMING_REQ and
97 * IBCM_INCOMING_REP_STALE;
98 *
99 * Lookup in "hca_passive_comid_tree" for IBCM_INCOMING_REQ_STALE
100 *
101 * All other lookups in "hca_active_tree".
102 *
103 * NOTE: "hca_active_tree" lookups are based on the local comid.
104 * "hca_passive_state_tree" lookups are based on remote QPN
105 * and remote hca GUID.
106 *
107 * Call avl_find to lookup in the respective tree and save result in
108 * "sp". If "sp" is null it implies that no match was found. If so,
109 * allocate a new ibcm_state_data_t and insert it into the AVL tree(s).
110 */
111 if ((event_type == IBCM_INCOMING_REQ) ||
112 (event_type == IBCM_INCOMING_REP_STALE)) {
113 ibcm_passive_node_info_t info;
114
115 info.info_qpn = remote_qpn;
116 info.info_hca_guid = remote_hca_guid;
117
118 /* Lookup based on Remote QPN and Remote GUID in Passive Tree */
119 sp = avl_find(&hcap->hca_passive_tree, &info, &where);
120 } else if ((event_type == IBCM_INCOMING_REQ_STALE) ||
121 (event_type == IBCM_INCOMING_REJ_RCOMID)) {
122 ibcm_passive_comid_node_info_t info;
123
124 info.info_comid = comid;
125 info.info_hca_guid = remote_hca_guid;
126
127 /* Lookup based on Remote COMID in Passive Tree */
128 sp = avl_find(&hcap->hca_passive_comid_tree, &info, &where);
129 } else { /* any other event including IBCM_OUTGOING_REQ */
130 /* Lookup based on Local comid in Active Tree */
131 sp = avl_find(&hcap->hca_active_tree, &comid, &where);
132 }
133
134 /* matching entry found !! */
135 if (sp != NULL) {
136 IBTF_DPRINTF_L4(cmlog, "ibcm_lookup_msg: match found "
137 "statep = %p", sp);
138 if (event_type == IBCM_INCOMING_REQ)
139 kmem_free(*rstatep, sizeof (ibcm_state_data_t));
140 *rstatep = sp; /* return the matched statep */
141
142 mutex_enter(&(sp->state_mutex));
143 IBCM_REF_CNT_INCR(sp); /* increment the ref count */
144 mutex_exit(&(sp->state_mutex));
145
146 return (IBCM_LOOKUP_EXISTS);
147 }
148
149 /*
150 * If we came here then it implies that CM didn't
151 * find a matching entry. We will create a new entry in avl tree,
152 * if event_type is INCOMING/OUTGOING REQ, REQ_STALE/REP_STALE.
153 * statep is created for INCOMING/OUTGOING REQ.
154 * For all other event_types we return lookup failure
155 */
156 if (!((event_type == IBCM_INCOMING_REQ) ||
157 (event_type == IBCM_INCOMING_REQ_STALE) ||
158 (event_type == IBCM_INCOMING_REP_STALE) ||
159 (event_type == IBCM_OUTGOING_REQ))) {
160 IBTF_DPRINTF_L2(cmlog, "ibcm_lookup_msg: failed for "
161 "event type %x remote_comid = 0x%x",
162 event_type, comid);
163
164 return (IBCM_LOOKUP_FAIL);
165 }
166
167 if ((event_type == IBCM_INCOMING_REQ) ||
168 (event_type == IBCM_OUTGOING_REQ)) {
169
170 /* fill in the new ibcm_state_data */
171 sp = *rstatep;
172
173 /* initialize statep */
174 mutex_init(&sp->state_mutex, NULL, MUTEX_DEFAULT, NULL);
175 cv_init(&sp->block_client_cv, NULL, CV_DRIVER, NULL);
176 cv_init(&sp->block_mad_cv, NULL, CV_DRIVER, NULL);
177
178 sp->hcap = hcap;
179 IBCM_REF_CNT_INCR(sp);
180 sp->local_comid = comid;
181
182 if (ibcm_enable_trace != 0)
183 ibcm_init_conn_trace(sp);
184
185 if (event_type == IBCM_INCOMING_REQ) { /* Passive side */
186 sp->state = IBCM_STATE_REQ_RCVD;
187 sp->clnt_proceed = IBCM_BLOCK;
188 sp->close_nocb_state = IBCM_UNBLOCK;
189 sp->remote_hca_guid = remote_hca_guid;
190 sp->remote_qpn = remote_qpn;
191 } else if (event_type == IBCM_OUTGOING_REQ) { /* Active side */
192 sp->close_nocb_state = IBCM_UNBLOCK;
193 sp->state = IBCM_STATE_IDLE;
194 }
195 } else {
196 sp = *rstatep; /* for incoming REQ/REP STALE only */
197 }
198
199 if ((event_type == IBCM_INCOMING_REQ) ||
200 (event_type == IBCM_INCOMING_REP_STALE)) {
201
202 /* First, insert a new "sp" into "hca_passive_tree" @ "where" */
203 avl_insert(&(hcap->hca_passive_tree), (void *)sp, where);
204
205 if (event_type == IBCM_INCOMING_REQ) { /* Only INCOMING_REQ */
206 /*
207 * We have to do an avl_find() to figure out
208 * "where" to insert the statep into the active tree.
209 *
210 * CM doesn't care for avl_find's retval.
211 */
212 (void) avl_find(&hcap->hca_active_tree,
213 &sp->local_comid, &where);
214
215 /* Next, insert the "sp" into "hca_active_tree" */
216 avl_insert(&hcap->hca_active_tree, (void *)sp, where);
217 }
218 } else if (event_type == IBCM_INCOMING_REQ_STALE) {
219 avl_insert(&(hcap->hca_passive_comid_tree), (void *)sp, where);
220 } else { /* IBCM_OUTGOING_REQ */
221 /* Insert the new sp only into "hca_active_tree", @ "where" */
222 avl_insert(&(hcap->hca_active_tree), (void *)sp, where);
223 }
224
225 return (IBCM_LOOKUP_NEW); /* return new lookup */
226 }
227
228
229 /*
230 * ibcm_active_node_compare:
231 * - AVL active tree node compare
232 *
233 * Arguments:
234 * p1 : pointer to local comid
235 * p2 : pointer to passed ibcm_state_data_t
236 *
237 * Return values:
238 * 0 : match found
239 * -1 : no match but insert to left side of the tree
240 * +1 : no match but insert to right side of the tree
241 */
242 int
243 ibcm_active_node_compare(const void *p1, const void *p2)
244 {
245 ib_com_id_t *local_comid = (ib_com_id_t *)p1;
246 ibcm_state_data_t *statep = (ibcm_state_data_t *)p2;
247
248 IBTF_DPRINTF_L5(cmlog, "ibcm_active_node_compare: "
249 "comid: 0x%x, statep: 0x%p", *local_comid, statep);
250
251 if (*local_comid > statep->local_comid) {
252 return (+1);
253 } else if (*local_comid < statep->local_comid) {
254 return (-1);
255 } else {
256 return (0);
257 }
258 }
259
260
261 /*
262 * ibcm_passive_node_compare:
263 * - AVL passive tree node compare (passive side)
264 *
265 * Arguments:
266 * p1 : pointer to ibcm_passive_node_info (remote qpn and remote guid)
267 * p2 : pointer to passed ibcm_state_data_t
268 *
269 * Return values:
270 * 0 : match found
271 * -1 : no match but insert to left side of the tree
272 * +1 : no match but insert to right side of the tree
273 */
274 int
275 ibcm_passive_node_compare(const void *p1, const void *p2)
276 {
277 ibcm_state_data_t *statep = (ibcm_state_data_t *)p2;
278 ibcm_passive_node_info_t *infop = (ibcm_passive_node_info_t *)p1;
279
280 IBTF_DPRINTF_L5(cmlog, "ibcm_passive_node_compare: "
281 "statep: 0x%p, p1: 0x%p", statep, p1);
282
283 /*
284 * PASSIVE SIDE: (REQ, REP, MRA, REJ)
285 * always search by active COMID
286 */
287 if (infop->info_qpn > statep->remote_qpn) {
288 return (+1);
289 } else if (infop->info_qpn < statep->remote_qpn) {
290 return (-1);
291 } else {
292 if (infop->info_hca_guid < statep->remote_hca_guid) {
293 return (-1);
294 } else if (infop->info_hca_guid > statep->remote_hca_guid) {
295 return (+1);
296 } else {
297 return (0);
298 }
299 }
300 }
301
302 /*
303 * ibcm_passive_comid_node_compare:
304 * - AVL passive comid tree node compare (passive side)
305 *
306 * Arguments:
307 * p1 : pointer to ibcm_passive_comid_node_info
308 * (remote comid and remote guid)
309 * p2 : pointer to passed ibcm_state_data_t
310 *
311 * Return values:
312 * 0 : match found
313 * -1 : no match but insert to left side of the tree
314 * +1 : no match but insert to right side of the tree
315 */
316 int
317 ibcm_passive_comid_node_compare(const void *p1, const void *p2)
318 {
319 ibcm_state_data_t *statep = (ibcm_state_data_t *)p2;
320 ibcm_passive_comid_node_info_t *infop =
321 (ibcm_passive_comid_node_info_t *)p1;
322
323 IBTF_DPRINTF_L5(cmlog, "ibcm_passive_comid_node_compare: "
324 "statep: 0x%p, p1: 0x%p", statep, p1);
325
326 if (infop->info_comid > statep->remote_comid) {
327 return (+1);
328 } else if (infop->info_comid < statep->remote_comid) {
329 return (-1);
330 } else {
331 if (infop->info_hca_guid < statep->remote_hca_guid) {
332 return (-1);
333 } else if (infop->info_hca_guid > statep->remote_hca_guid) {
334 return (+1);
335 } else {
336 return (0);
337 }
338 }
339 }
340
341
342 void
343 ibcm_delete_state_from_avl(ibcm_state_data_t *statep)
344 {
345 avl_index_t a_where = 0;
346 avl_index_t p_where = 0;
347 avl_index_t pcomid_where = 0;
348 ibcm_hca_info_t *hcap;
349 ibcm_state_data_t *active_nodep, *passive_nodep;
350 ibcm_state_data_t *passive_comid_nodep;
351 ibcm_passive_node_info_t info;
352 ibcm_passive_comid_node_info_t info_comid;
353
354 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_state_from_avl: statep 0x%p",
355 statep);
356
357 if (statep == NULL) {
358 IBTF_DPRINTF_L2(cmlog, "ibcm_delete_state_from_avl: statep"
359 " NULL");
360 return;
361 }
362
363 hcap = statep->hcap;
364
365 /*
366 * Once the avl tree lock is acquired, no other thread can increment
367 * ref cnt, until tree lock is exit'ed. Since the statep is removed
368 * from the avl's after acquiring lock below, no other thread can
369 * increment the ref cnt after acquiring the lock below
370 */
371
372 rw_enter(&hcap->hca_state_rwlock, RW_WRITER);
373
374 /* Lookup based on Local comid in the active tree */
375 active_nodep = avl_find(&hcap->hca_active_tree, &(statep->local_comid),
376 &a_where);
377
378 /* Lookup based on Remote QPN and Remote GUID in the passive tree */
379 info.info_qpn = statep->remote_qpn;
380 info.info_hca_guid = statep->remote_hca_guid;
381 passive_nodep = avl_find(&hcap->hca_passive_tree, &info, &p_where);
382
383 /* Lookup based on Remote Comid and Remote GUID in the passive tree */
384 info_comid.info_comid = statep->remote_comid;
385 info_comid.info_hca_guid = statep->remote_hca_guid;
386 passive_comid_nodep = avl_find(&hcap->hca_passive_comid_tree,
387 &info_comid, &pcomid_where);
388
389 /* remove it from the tree, destroy record and the nodep */
390 if (active_nodep == statep) {
391 avl_remove(&hcap->hca_active_tree, active_nodep);
392 }
393
394 if (passive_nodep == statep) {
395 avl_remove(&hcap->hca_passive_tree, passive_nodep);
396 }
397
398 if (passive_comid_nodep == statep) {
399 avl_remove(&hcap->hca_passive_comid_tree, passive_comid_nodep);
400 }
401
402 rw_exit(&hcap->hca_state_rwlock);
403 }
404
405 /*
406 * ibcm_dealloc_state_data:
407 * Deallocates all buffers and the memory of state structure
408 * This routine can be called on statep that has ref_cnt of 0, and that is
409 * already deleted from the avl tree's
410 *
411 * Arguments are:-
412 * statep - statep to be deleted
413 *
414 * Return Values: NONE
415 */
416 void
417 ibcm_dealloc_state_data(ibcm_state_data_t *statep)
418 {
419 timeout_id_t timer_val;
420 int dump_trace;
421 IBTF_DPRINTF_L4(cmlog, "ibcm_dealloc_state_data: statep 0x%p", statep);
422
423 if (statep == NULL) {
424 IBTF_DPRINTF_L2(cmlog, "ibcm_dealloc_state_data: statep NULL");
425 return;
426 }
427
428 /* ref_cnt is 0 */
429 /* If timer is running - expire it */
430 mutex_enter(&statep->state_mutex);
431 timer_val = statep->timerid;
432 if (timer_val != 0) {
433 statep->timerid = 0;
434 mutex_exit(&statep->state_mutex);
435 (void) untimeout(timer_val);
436 } else
437 mutex_exit(&statep->state_mutex);
438
439 /* release the ref cnt on the associated ibmf qp */
440 if (statep->stored_reply_addr.cm_qp_entry != NULL)
441 ibcm_release_qp(statep->stored_reply_addr.cm_qp_entry);
442
443 if (statep->stored_msg != NULL)
444 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl,
445 &statep->stored_msg);
446
447 if (statep->dreq_msg != NULL)
448 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl,
449 &statep->dreq_msg);
450
451 if (statep->drep_msg != NULL)
452 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl,
453 &statep->drep_msg);
454
455 if (statep->mra_msg != NULL)
456 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl,
457 &statep->mra_msg);
458
459 if (statep->lapr_msg != NULL)
460 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl,
461 &statep->lapr_msg);
462
463 if (statep->defer_cm_msg != NULL)
464 kmem_free(statep->defer_cm_msg, IBCM_MSG_SIZE);
465
466 IBTF_DPRINTF_L4(cmlog, "ibcm_dealloc_state_data: done for sp = 0x%p",
467 statep);
468
469 /* Ensure the thread doing ref cnt decr releases the mutex */
470 mutex_enter(&statep->state_mutex);
471 dump_trace = statep->cm_retries > 0;
472 mutex_exit(&statep->state_mutex);
473
474 /*
475 * now call the mutex_destroy() and cv_destroy()
476 */
477 mutex_destroy(&statep->state_mutex);
478
479 cv_destroy(&statep->block_client_cv);
480 cv_destroy(&statep->block_mad_cv);
481
482 /* free the comid */
483 ibcm_free_comid(statep->hcap, statep->local_comid);
484
485 /* Decrement the resource on hcap */
486 ibcm_dec_hca_res_cnt(statep->hcap);
487
488 /* dump the trace data into ibtf_debug_buf */
489 if ((ibcm_enable_trace & 4) || dump_trace)
490 ibcm_dump_conn_trace(statep);
491
492 ibcm_fini_conn_trace(statep);
493
494 /* free the statep */
495 kmem_free(statep, sizeof (ibcm_state_data_t));
496 }
497
498 /*
499 * ibcm_delete_state_data:
500 * Deletes the state from avl trees, and tries to deallocate state
501 *
502 * Arguments are:-
503 * statep - statep to be deleted
504 *
505 * Return Values: NONE
506 */
507 void
508 ibcm_delete_state_data(ibcm_state_data_t *statep)
509 {
510 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_state_data:");
511
512 ibcm_delete_state_from_avl(statep);
513
514 /* Must acquire the state mutex to set delete_state_data */
515 mutex_enter(&statep->state_mutex);
516 if (statep->ref_cnt > 0) {
517 statep->delete_state_data = B_TRUE;
518 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_state_data: statep 0x%p "
519 "ref_cnt = %x", statep, statep->ref_cnt);
520 mutex_exit(&statep->state_mutex);
521 return;
522 }
523 mutex_exit(&statep->state_mutex);
524
525 ibcm_dealloc_state_data(statep);
526 }
527
528 /*
529 * ibcm_find_sidr_entry:
530 * Routines for CM SIDR state structure list manipulation.
531 * Finds an entry based on lid, gid and grh exists fields
532 *
533 * INPUTS:
534 * lid: LID of incoming SIDR REQ
535 * gid: GID of incoming SIDR REQ
536 * grh_exists: TRUE if GRH exists in the incoming SIDR REQ
537 * req_id: Request ID
538 * hcap: CM State table to search for SIDR state structure
539 * statep: Returns a valid state structure, if one exists based
540 * on lid, gid and grh_exists fields
541 * flag: IBCM_FLAG_LOOKUP - just lookup
542 * IBCM_FLAG_LOOKUP_AND_ADD - if lookup fails, add it.
543 * Return Values:
544 * IBCM_LOOKUP_EXISTS - found an existing entry
545 * IBCM_LOOKUP_FAIL - failed to find an entry
546 * IBCM_LOOKUP_NEW - created a new entry
547 */
548 ibcm_status_t
549 ibcm_find_sidr_entry(ibcm_sidr_srch_t *srch_param, ibcm_hca_info_t *hcap,
550 ibcm_ud_state_data_t **ud_statep, ibcm_lookup_flag_t flag)
551 {
552 ibcm_status_t status;
553 ibcm_ud_state_data_t *usp;
554
555 IBTF_DPRINTF_L5(cmlog, "ibcm_find_sidr_entry: srch_params are:"
556 "lid=%x, (%llX, %llX), grh: %x, id: %x",
557 srch_param->srch_lid, srch_param->srch_gid.gid_prefix,
558 srch_param->srch_gid.gid_guid, srch_param->srch_grh_exists,
559 srch_param->srch_req_id);
560
561 if (flag == IBCM_FLAG_ADD) {
562 *ud_statep = ibcm_add_sidr_entry(srch_param, hcap);
563 return (IBCM_LOOKUP_NEW);
564 }
565
566 usp = hcap->hca_sidr_list; /* Point to the list */
567
568 /* traverse the list for a matching entry */
569 while (usp != NULL) {
570 IBTF_DPRINTF_L5(cmlog, "ibcm_find_sidr_entry: "
571 "lid=%x, (%llX, %llX), grh: %x, id: %x",
572 usp->ud_sidr_req_lid, usp->ud_sidr_req_gid.gid_prefix,
573 usp->ud_sidr_req_gid.gid_guid, usp->ud_grh_exists,
574 usp->ud_req_id);
575
576 if ((usp->ud_sidr_req_lid == srch_param->srch_lid) &&
577 ((srch_param->srch_gid.gid_prefix == 0) ||
578 (srch_param->srch_gid.gid_prefix ==
579 usp->ud_sidr_req_gid.gid_prefix)) &&
580 ((srch_param->srch_gid.gid_guid == 0) ||
581 (srch_param->srch_gid.gid_guid ==
582 usp->ud_sidr_req_gid.gid_guid)) &&
583 (srch_param->srch_req_id == usp->ud_req_id) &&
584 (usp->ud_grh_exists == srch_param->srch_grh_exists) &&
585 (usp->ud_mode == srch_param->srch_mode)) { /* found match */
586 *ud_statep = usp;
587 IBTF_DPRINTF_L5(cmlog, "ibcm_find_sidr_entry: "
588 "found usp = %p", usp);
589 mutex_enter(&usp->ud_state_mutex);
590 IBCM_UD_REF_CNT_INCR(usp);
591 mutex_exit(&usp->ud_state_mutex);
592
593 return (IBCM_LOOKUP_EXISTS);
594 }
595 usp = usp->ud_nextp;
596 }
597
598 /*
599 * If code came here --> it couldn't find a match.
600 * OR
601 * the "hcap->hca_sidr_list" was NULL
602 */
603 if (flag == IBCM_FLAG_LOOKUP) {
604 IBTF_DPRINTF_L3(cmlog, "ibcm_find_sidr_entry: no match found "
605 "lid=%x, (%llX, %llX), grh: %x, id: %x",
606 srch_param->srch_lid, srch_param->srch_gid.gid_prefix,
607 srch_param->srch_gid.gid_guid, srch_param->srch_grh_exists,
608 srch_param->srch_req_id);
609 status = IBCM_LOOKUP_FAIL;
610 } else {
611 *ud_statep = ibcm_add_sidr_entry(srch_param, hcap);
612 status = IBCM_LOOKUP_NEW;
613 }
614
615 return (status);
616 }
617
618
619 /*
620 * ibcm_add_sidr_entry:
621 * Adds a SIDR entry. Called *ONLY* from ibcm_find_sidr_entry()
622 *
623 * INPUTS:
624 * lid: LID of incoming SIDR REQ
625 * gid: GID of incoming SIDR REQ
626 * grh_exists: TRUE if GRH exists in the incoming SIDR REQ
627 * req_id: Request ID
628 * hcap: CM State table to search for SIDR state structure
629 * Return Values: NONE
630 */
631 ibcm_ud_state_data_t *
632 ibcm_add_sidr_entry(ibcm_sidr_srch_t *srch_param, ibcm_hca_info_t *hcap)
633 {
634 ibcm_ud_state_data_t *ud_statep;
635
636 IBTF_DPRINTF_L5(cmlog, "ibcm_add_sidr_entry: lid=%x, guid=%llX, "
637 "grh = %x req_id = %x", srch_param->srch_lid,
638 srch_param->srch_gid.gid_guid, srch_param->srch_grh_exists,
639 srch_param->srch_req_id);
640
641 /* didn't find the entry - so create new */
642 ud_statep = kmem_zalloc(sizeof (ibcm_ud_state_data_t), KM_SLEEP);
643
644 mutex_init(&ud_statep->ud_state_mutex, NULL, MUTEX_DEFAULT, NULL);
645 cv_init(&ud_statep->ud_block_client_cv, NULL, CV_DRIVER, NULL);
646
647 /* Initialize some ud_statep fields */
648 mutex_enter(&ud_statep->ud_state_mutex);
649 ud_statep->ud_hcap = hcap;
650 ud_statep->ud_req_id = srch_param->srch_req_id;
651 ud_statep->ud_ref_cnt = 1;
652 ud_statep->ud_grh_exists = srch_param->srch_grh_exists;
653 ud_statep->ud_sidr_req_lid = srch_param->srch_lid;
654 ud_statep->ud_sidr_req_gid = srch_param->srch_gid;
655 ud_statep->ud_mode = srch_param->srch_mode;
656 ud_statep->ud_max_cm_retries = ibcm_max_retries;
657 mutex_exit(&ud_statep->ud_state_mutex);
658
659 /* Update the list */
660 ud_statep->ud_nextp = hcap->hca_sidr_list;
661 hcap->hca_sidr_list = ud_statep;
662
663 return (ud_statep);
664 }
665
666
667 /*
668 * ibcm_delete_ud_state_data:
669 * Deletes a given state structure
670 *
671 * Arguments are:-
672 * statep - statep to be deleted
673 *
674 * Return Values: NONE
675 */
676 void
677 ibcm_delete_ud_state_data(ibcm_ud_state_data_t *ud_statep)
678 {
679 ibcm_ud_state_data_t *prevp, *headp;
680 ibcm_hca_info_t *hcap;
681
682 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_ud_state_data: ud_statep 0x%p",
683 ud_statep);
684
685 if (ud_statep == NULL || ud_statep->ud_hcap == NULL) {
686 IBTF_DPRINTF_L2(cmlog, "ibcm_delete_ud_state_data: "
687 "ud_statep or hcap is NULL");
688 return;
689 }
690
691 hcap = ud_statep->ud_hcap;
692
693 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER);
694
695 /* Next, remove this from the HCA SIDR list */
696 if (hcap->hca_sidr_list != NULL) {
697 prevp = NULL;
698 headp = hcap->hca_sidr_list;
699
700 while (headp != NULL) {
701 /* delete the matching entry */
702 if (headp == ud_statep) {
703 if (prevp) {
704 prevp->ud_nextp = headp->ud_nextp;
705 } else {
706 prevp = headp->ud_nextp;
707 hcap->hca_sidr_list = prevp;
708 }
709 break;
710 }
711 prevp = headp;
712 headp = headp->ud_nextp;
713 }
714 }
715
716 rw_exit(&hcap->hca_sidr_list_lock);
717
718 /*
719 * While ref_cnt > 0
720 * - implies someone else is accessing the statep (possibly in
721 * a timeout function handler etc.)
722 * - don't delete statep unless they are done otherwise potentially
723 * one could access released memory and panic.
724 */
725 mutex_enter(&ud_statep->ud_state_mutex);
726 if (ud_statep->ud_ref_cnt > 0) {
727 ud_statep->ud_delete_state_data = B_TRUE;
728 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_ud_state_data: "
729 "ud_statep 0x%p ud_ref_cnt = %x", ud_statep,
730 ud_statep->ud_ref_cnt);
731 mutex_exit(&ud_statep->ud_state_mutex);
732 return;
733 }
734 mutex_exit(&ud_statep->ud_state_mutex);
735
736 ibcm_dealloc_ud_state_data(ud_statep);
737 }
738
739 /*
740 * ibcm_ud_dealloc_state_data:
741 * Deallocates a given ud state structure
742 *
743 * Arguments are:-
744 * ud statep - ud statep to be deleted
745 *
746 * Return Values: NONE
747 */
748 void
749 ibcm_dealloc_ud_state_data(ibcm_ud_state_data_t *ud_statep)
750 {
751 timeout_id_t timer_val;
752
753 IBTF_DPRINTF_L4(cmlog, "ibcm_dealloc_ud_state_data: ud_statep 0x%p",
754 ud_statep);
755
756 /* If timer is running - expire it */
757 mutex_enter(&ud_statep->ud_state_mutex);
758 if (ud_statep->ud_timerid) {
759 timer_val = ud_statep->ud_timerid;
760 ud_statep->ud_timerid = 0;
761 mutex_exit(&ud_statep->ud_state_mutex);
762 (void) untimeout(timer_val);
763 IBTF_DPRINTF_L2(cmlog, "ibcm_dealloc_ud_state_data: "
764 "Unexpected timer id 0x%p ud_statep 0x%p", timer_val,
765 ud_statep);
766 } else
767 mutex_exit(&ud_statep->ud_state_mutex);
768
769 if (ud_statep->ud_stored_msg != NULL) {
770 (void) ibcm_free_out_msg(
771 ud_statep->ud_stored_reply_addr.ibmf_hdl,
772 &ud_statep->ud_stored_msg);
773 }
774
775 /* release the ref cnt on the associated ibmf qp */
776 ASSERT(ud_statep->ud_stored_reply_addr.cm_qp_entry != NULL);
777 ibcm_release_qp(ud_statep->ud_stored_reply_addr.cm_qp_entry);
778
779 /* Ensure the thread doing ref cnt decr releases the mutex */
780 mutex_enter(&ud_statep->ud_state_mutex);
781 mutex_exit(&ud_statep->ud_state_mutex);
782
783 /* now do the mutex_destroy() and cv_destroy() */
784 mutex_destroy(&ud_statep->ud_state_mutex);
785
786 cv_destroy(&ud_statep->ud_block_client_cv);
787
788 /* free the req id on SIDR REQ sender side */
789 if (ud_statep->ud_mode == IBCM_ACTIVE_MODE)
790 ibcm_free_reqid(ud_statep->ud_hcap, ud_statep->ud_req_id);
791
792 /* Decrement the resource on hcap */
793 ibcm_dec_hca_res_cnt(ud_statep->ud_hcap);
794
795 /* free the statep */
796 kmem_free(ud_statep, sizeof (ibcm_ud_state_data_t));
797 }
798
799
800 /*
801 * ibcm_init_ids:
802 * Create the vmem arenas for the various global ids
803 *
804 * Arguments are:-
805 * NONE
806 *
807 * Return Values: ibcm_status_t
808 */
809
810 ibcm_status_t
811 ibcm_init_ids(void)
812 {
813 timespec_t tv;
814
815 ibcm_local_sid_arena = vmem_create("ibcm_local_sid",
816 (void *)IBCM_INITIAL_SID, IBCM_MAX_LOCAL_SIDS, 1, NULL, NULL, NULL,
817 0, VM_SLEEP | VMC_IDENTIFIER);
818
819 if (!ibcm_local_sid_arena)
820 return (IBCM_FAILURE);
821
822 ibcm_ip_sid_arena = vmem_create("ibcm_ip_sid", (void *)IBCM_INITIAL_SID,
823 IBCM_MAX_IP_SIDS, 1, NULL, NULL, NULL, 0,
824 VM_SLEEP | VMC_IDENTIFIER);
825
826 if (!ibcm_ip_sid_arena)
827 return (IBCM_FAILURE);
828
829 /* create a random starting value for local service ids */
830 gethrestime(&tv);
831 ibcm_local_sid_seed = ((uint64_t)tv.tv_sec << 20) & 0x007FFFFFFFF00000;
832 ASSERT((ibcm_local_sid_seed & IB_SID_AGN_MASK) == 0);
833 ibcm_local_sid_seed |= IB_SID_AGN_LOCAL;
834
835 ibcm_local_cid_seed = (ib_com_id_t)tv.tv_sec;
836
837 return (IBCM_SUCCESS);
838 }
839
840
841 /*
842 * ibcm_init_hca_ids:
843 * Create the vmem arenas for the various hca level ids
844 *
845 * Arguments are:-
846 * hcap pointer to ibcm_hca_info_t
847 *
848 * Return Values: ibcm_status_t
849 */
850 ibcm_status_t
851 ibcm_init_hca_ids(ibcm_hca_info_t *hcap)
852 {
853 hcap->hca_comid_arena = vmem_create("ibcm_com_ids",
854 (void *)IBCM_INITIAL_COMID, IBCM_MAX_COMIDS,
855 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
856
857 if (!hcap->hca_comid_arena)
858 return (IBCM_FAILURE);
859
860 hcap->hca_reqid_arena = vmem_create("ibcm_req_ids",
861 (void *)IBCM_INITIAL_REQID, IBCM_MAX_REQIDS,
862 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
863
864 if (!hcap->hca_reqid_arena) {
865 vmem_destroy(hcap->hca_comid_arena);
866 return (IBCM_FAILURE);
867 }
868
869 return (IBCM_SUCCESS);
870 }
871
872 /*
873 * ibcm_free_ids:
874 * Destroy the vmem arenas for the various ids
875 *
876 * Arguments are:-
877 * NONE
878 *
879 * Return Values: NONE
880 */
881 void
882 ibcm_fini_ids(void)
883 {
884 /* All arenas shall be valid */
885 vmem_destroy(ibcm_local_sid_arena);
886 vmem_destroy(ibcm_ip_sid_arena);
887 }
888
889 /*
890 * ibcm_free_hca_ids:
891 * Destroy the vmem arenas for the various ids
892 *
893 * Arguments are:-
894 * hcap pointer to ibcm_hca_info_t
895 *
896 * Return Values: NONE
897 */
898 void
899 ibcm_fini_hca_ids(ibcm_hca_info_t *hcap)
900 {
901 /* All arenas shall be valid */
902 vmem_destroy(hcap->hca_comid_arena);
903 vmem_destroy(hcap->hca_reqid_arena);
904 }
905
906 /* Communication id management routines ie., allocate, free up comids */
907
908 /*
909 * ibcm_alloc_comid:
910 * Allocate a new communication id
911 *
912 * Arguments are:-
913 * hcap : pointer to ibcm_hca_info_t
914 * comid: pointer to the newly allocated communication id
915 *
916 * Return Values: ibt_status_t
917 */
918 ibcm_status_t
919 ibcm_alloc_comid(ibcm_hca_info_t *hcap, ib_com_id_t *comidp)
920 {
921 ib_com_id_t comid;
922
923 /* Use next fit, so least recently used com id is allocated */
924 comid = (ib_com_id_t)(uintptr_t)vmem_alloc(hcap->hca_comid_arena, 1,
925 VM_SLEEP | VM_NEXTFIT);
926
927 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_comid: hcap 0x%p comid 0x%lX", hcap,
928 comid);
929
930 /*
931 * As comid is 32 bits, and maximum connections possible are 2^24
932 * per hca, comid allocation would never fail
933 */
934 *comidp = comid + ibcm_local_cid_seed;
935 if (comid == 0) {
936 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_comid: hcap 0x%p"
937 "no more comids available", hcap);
938 return (IBCM_FAILURE);
939 }
940
941 return (IBCM_SUCCESS);
942 }
943
944 /*
945 * ibcm_free_comid:
946 * Releases the given Communication Id
947 *
948 * Arguments are:
949 * hcap : pointer to ibcm_hca_info_t
950 * comid : Communication id to be free'd
951 *
952 * Return Values: NONE
953 */
954 void
955 ibcm_free_comid(ibcm_hca_info_t *hcap, ib_com_id_t comid)
956 {
957 IBTF_DPRINTF_L4(cmlog, "ibcm_free_comid: hcap 0x%p"
958 "comid %x", hcap, comid);
959 comid -= ibcm_local_cid_seed;
960 vmem_free(hcap->hca_comid_arena, (void *)(uintptr_t)comid, 1);
961 }
962
963 /* Allocate and Free local service ids */
964
965 /*
966 * ibcm_alloc_local_sids:
967 * Create and destroy the vmem arenas for the service ids
968 *
969 * Arguments are:-
970 * Number of contiguous SIDs needed
971 *
972 * Return Values: starting SID
973 */
974 ib_svc_id_t
975 ibcm_alloc_local_sids(int num_sids)
976 {
977 ib_svc_id_t sid;
978
979 sid = (ib_svc_id_t)(uintptr_t)vmem_alloc(ibcm_local_sid_arena,
980 num_sids, VM_SLEEP | VM_NEXTFIT);
981
982 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_local_sids: ServiceID 0x%llX "
983 "num_sids %d", sid, num_sids);
984 if (sid == 0) {
985 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_local_sids: "
986 "no more local sids available");
987 } else {
988 ASSERT((ibcm_local_sid_seed & IB_SID_AGN_MASK) ==
989 IB_SID_AGN_LOCAL);
990 sid += ibcm_local_sid_seed;
991 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_local_sids: Success: "
992 "allocated 0x%llX:%d", sid, num_sids);
993 }
994 return (sid);
995 }
996
997 /*
998 * ibcm_free_local_sids:
999 * Releases the given Local service id
1000 *
1001 * Arguments are:
1002 * num_sids: Number of local service id's to be free'd
1003 * service_id: Starting local service id that needs to be free'd
1004 *
1005 * Return Values: NONE
1006 */
1007 void
1008 ibcm_free_local_sids(ib_svc_id_t service_id, int num_sids)
1009 {
1010 service_id -= ibcm_local_sid_seed;
1011 IBTF_DPRINTF_L4(cmlog, "ibcm_free_local_sids: "
1012 "service_id 0x%llX num_sids %d", service_id, num_sids);
1013 vmem_free(ibcm_local_sid_arena,
1014 (void *)(uintptr_t)service_id, num_sids);
1015 }
1016
1017 /*
1018 * ibcm_alloc_ip_sid:
1019 * Allocate a local IP SID.
1020 */
1021 ib_svc_id_t
1022 ibcm_alloc_ip_sid()
1023 {
1024 ib_svc_id_t sid;
1025
1026 sid = (ib_svc_id_t)(uintptr_t)vmem_alloc(ibcm_ip_sid_arena, 1,
1027 VM_SLEEP | VM_NEXTFIT);
1028 if (sid == 0) {
1029 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_ip_sid: no more RDMA IP "
1030 "SIDs available");
1031 } else {
1032 sid += IB_SID_IPADDR_PREFIX;
1033 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_ip_sid: Success: RDMA IP SID"
1034 " allocated : 0x%016llX", sid);
1035 }
1036 return (sid);
1037 }
1038
1039 /*
1040 * ibcm_free_ip_sid:
1041 * Releases the given IP Service ID
1042 */
1043 void
1044 ibcm_free_ip_sid(ib_svc_id_t sid)
1045 {
1046 sid -= IB_SID_IPADDR_PREFIX;
1047 vmem_free(ibcm_ip_sid_arena, (void *)(uintptr_t)sid, 1);
1048 }
1049
1050
1051 /* Allocate and free request id routines for SIDR */
1052
1053 /*
1054 * ibcm_alloc_reqid:
1055 * Allocate a new SIDR REQ request id
1056 *
1057 * Arguments are:-
1058 * hcap : pointer to ibcm_hca_info_t
1059 * *reqid : pointer to the new request id returned
1060 *
1061 * Return Values: ibcm_status_t
1062 */
1063 ibcm_status_t
1064 ibcm_alloc_reqid(ibcm_hca_info_t *hcap, uint32_t *reqid)
1065 {
1066 /* Use next fit, so least recently used com id is allocated */
1067 *reqid = (uint32_t)(uintptr_t)vmem_alloc(hcap->hca_reqid_arena, 1,
1068 VM_SLEEP | VM_NEXTFIT);
1069
1070 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_reqid: hcap 0x%p reqid %x", hcap,
1071 *reqid);
1072 if (!(*reqid)) {
1073 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_reqid: "
1074 "no more req ids available");
1075 return (IBCM_FAILURE);
1076 }
1077 return (IBCM_SUCCESS);
1078 }
1079
1080 /*
1081 * ibcm_free_reqid:
1082 * Releases the given SIDR REQ request id
1083 *
1084 * Arguments are:
1085 * hcap : pointer to ibcm_hca_info_t
1086 * reqid : Request id to be free'd
1087 *
1088 * Return Values: NONE
1089 */
1090 void
1091 ibcm_free_reqid(ibcm_hca_info_t *hcap, uint32_t reqid)
1092 {
1093 IBTF_DPRINTF_L4(cmlog, "ibcm_free_reqid: hcap 0x%p reqid %x", hcap,
1094 reqid);
1095 vmem_free(hcap->hca_reqid_arena, (void *)(uintptr_t)reqid, 1);
1096 }
1097
1098 /*
1099 * ibcm_generate_tranid:
1100 * Generate a new transaction id based on args
1101 *
1102 * Arguments are:-
1103 * event_type CM Message REQ/DREQ/LAP
1104 * id 32 bit identifier
1105 * cm_tran_priv CM private data to be filled in top 28 MSB bits of
1106 * tran id
1107 *
1108 *
1109 * Return Value: uint64_t
1110 */
1111 uint64_t
1112 ibcm_generate_tranid(uint8_t event, uint32_t id, uint32_t cm_tran_priv)
1113 {
1114 /*
1115 * copy comid to bits 31-0 of tran id,
1116 * attr id to bits 35-32 of tran id,
1117 * cm_priv to bits 63-36 of tran id
1118 */
1119 if (cm_tran_priv == 0)
1120 /*
1121 * The below ensures that no duplicate transaction id is
1122 * generated atleast for next 6 months. Calculations:
1123 * (2^28)/(1000 * 60 * 24 * 30) = 6 approx
1124 */
1125 cm_tran_priv = gethrtime() >> 20; /* ~time in ms */
1126
1127 return ((((uint64_t)cm_tran_priv << 36) | (uint64_t)event << 32) | id);
1128 }
1129
1130 #ifdef DEBUG
1131
1132 /*
1133 * ibcm_decode_tranid:
1134 * Decodes a given transaction id, assuming certain format.
1135 *
1136 * Arguments are:-
1137 * tran_id Transaction id to be decoded
1138 * cm_tran_priv CM private data retrieved from transaction id
1139 *
1140 * Return Value: None
1141 */
1142 void
1143 ibcm_decode_tranid(uint64_t tran_id, uint32_t *cm_tran_priv)
1144 {
1145 ib_com_id_t id;
1146 ibcm_event_type_t event;
1147
1148 id = tran_id & 0xFFFFFFFF;
1149 event = (tran_id >> 32) & 0xF;
1150
1151 IBTF_DPRINTF_L5(cmlog, "ibcm_decode_tranid: id = 0x%x, event = %x",
1152 id, event);
1153
1154 if (cm_tran_priv) {
1155 *cm_tran_priv = tran_id >> 36;
1156 IBTF_DPRINTF_L5(cmlog, "ibcm_decode_tranid: "
1157 "cm_tran_priv = %x", *cm_tran_priv);
1158 }
1159 }
1160
1161 #endif
1162
1163 /*
1164 * Service ID entry create and lookup functions
1165 */
1166
1167 /*
1168 * ibcm_svc_compare:
1169 * - AVL svc tree node compare
1170 *
1171 * Arguments:
1172 * p1 : pointer to local comid
1173 * p2 : pointer to passed ibcm_state_data_t
1174 *
1175 * Return values:
1176 * 0 : match found
1177 * -1 : no match but insert to left side of the tree
1178 * +1 : no match but insert to right side of the tree
1179 */
1180 int
1181 ibcm_svc_compare(const void *p1, const void *p2)
1182 {
1183 ibcm_svc_lookup_t *sidp = (ibcm_svc_lookup_t *)p1;
1184 ibcm_svc_info_t *svcp = (ibcm_svc_info_t *)p2;
1185 ib_svc_id_t start_sid = sidp->sid;
1186 ib_svc_id_t end_sid = start_sid + sidp->num_sids - 1;
1187
1188 IBTF_DPRINTF_L5(cmlog, "ibcm_svc_compare: "
1189 "sid: 0x%llx, numsids: %d, node_sid: 0x%llx node_num_sids: %d",
1190 sidp->sid, sidp->num_sids, svcp->svc_id, svcp->svc_num_sids);
1191
1192 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock));
1193
1194 if (svcp->svc_id > end_sid)
1195 return (-1);
1196 if (svcp->svc_id + svcp->svc_num_sids - 1 < start_sid)
1197 return (+1);
1198 return (0); /* means there is some overlap of SIDs */
1199 }
1200
1201
1202 /*
1203 * ibcm_create_svc_entry:
1204 * Make sure no conflicting entry exists, then allocate it.
1205 * Fill in the critical "look up" details that are provided
1206 * in the arguments before dropping the lock.
1207 *
1208 * Return values:
1209 * Pointer to ibcm_svc_info_t, if created, otherwise NULL.
1210 */
1211 ibcm_svc_info_t *
1212 ibcm_create_svc_entry(ib_svc_id_t sid, int num_sids)
1213 {
1214 ibcm_svc_info_t *svcp;
1215 ibcm_svc_info_t *svcinfop;
1216 ibcm_svc_lookup_t svc;
1217 avl_index_t where = 0;
1218
1219 /* assume success, and avoid kmem while holding the writer lock */
1220 svcinfop = kmem_zalloc(sizeof (*svcinfop), KM_SLEEP);
1221 svcinfop->svc_id = sid;
1222 svcinfop->svc_num_sids = num_sids;
1223
1224 svc.sid = sid;
1225 svc.num_sids = num_sids;
1226
1227 mutex_enter(&ibcm_svc_info_lock);
1228 svcp = avl_find(&ibcm_svc_avl_tree, &svc, &where);
1229 if (svcp != NULL) { /* overlab exists */
1230 mutex_exit(&ibcm_svc_info_lock);
1231 kmem_free(svcinfop, sizeof (*svcinfop));
1232 return (NULL);
1233 }
1234 avl_insert(&ibcm_svc_avl_tree, (void *)svcinfop, where);
1235 mutex_exit(&ibcm_svc_info_lock);
1236
1237 return (svcinfop);
1238 }
1239
1240 /*
1241 * ibcm_find_svc_entry:
1242 * Finds a ibcm_svc_info_t entry into the CM's global table.
1243 * The search done here assumes the list is sorted by SID.
1244 *
1245 * Arguments are:
1246 * sid - Service ID to look up
1247 *
1248 * Return values:
1249 * Pointer to ibcm_svc_info_t, if found, otherwise NULL.
1250 */
1251 ibcm_svc_info_t *
1252 ibcm_find_svc_entry(ib_svc_id_t sid)
1253 {
1254 ibcm_svc_info_t *svcp;
1255 ibcm_svc_lookup_t svc;
1256
1257 IBTF_DPRINTF_L3(cmlog, "ibcm_find_svc_entry: finding SID 0x%llX", sid);
1258
1259 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock));
1260
1261 svc.sid = sid;
1262 svc.num_sids = 1;
1263 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL);
1264 if (svcp != NULL) {
1265 IBTF_DPRINTF_L3(cmlog, "ibcm_find_svc_entry: "
1266 "found SID = 0x%llX", sid);
1267 return (svcp); /* found it */
1268 }
1269 IBTF_DPRINTF_L3(cmlog, "ibcm_find_svc_entry: SID %llX not found", sid);
1270 return (NULL);
1271 }
1272
1273 /*
1274 * ibcm_alloc_ibmf_msg:
1275 * Allocate an ibmf message structure and the additional memory required for
1276 * sending an outgoing CM mad. The ibmf message structure contains two
1277 * ibmf_msg_bufs_t fields, one for the incoming MAD and one for the outgoing
1278 * MAD. The CM must allocate the memory for the outgoing MAD. The msg_buf
1279 * field has three buffers: the mad header, the class header, and the class
1280 * data. To simplify the code and reduce the number of kmem_zalloc() calls,
1281 * ibcm_alloc_ibmf_msg will allocate one buffer and set the pointers to the
1282 * right offsets. No class header is needed so only the mad header and class
1283 * data fields are used.
1284 */
1285 ibt_status_t
1286 ibcm_alloc_out_msg(ibmf_handle_t ibmf_handle, ibmf_msg_t **ibmf_msgpp,
1287 uint8_t method)
1288 {
1289 ib_mad_hdr_t *output_mad_hdr;
1290 int sa_retval;
1291
1292 if ((sa_retval =
1293 ibmf_alloc_msg(ibmf_handle, IBMF_ALLOC_SLEEP, ibmf_msgpp)) !=
1294 IBMF_SUCCESS) {
1295 IBTF_DPRINTF_L1(cmlog, "ibcm_alloc_out_msg: "
1296 "ibmf_alloc_msg failed with IBMF_ALLOC_SLEEP");
1297 return (ibcm_ibmf_analyze_error(sa_retval));
1298 }
1299
1300 (*ibmf_msgpp)->im_msgbufs_send.im_bufs_mad_hdr = kmem_zalloc(
1301 IBCM_MAD_SIZE, KM_SLEEP);
1302
1303 (*ibmf_msgpp)->im_msgbufs_send.im_bufs_cl_data_len = IBCM_MSG_SIZE;
1304 (*ibmf_msgpp)->im_msgbufs_send.im_bufs_cl_data =
1305 (uchar_t *)((*ibmf_msgpp)->im_msgbufs_send.im_bufs_mad_hdr) +
1306 IBCM_MAD_HDR_SIZE;
1307
1308 /* initialize generic CM MAD header fields */
1309 output_mad_hdr = IBCM_OUT_HDRP((*ibmf_msgpp));
1310 output_mad_hdr->BaseVersion = IBCM_MAD_BASE_VERSION;
1311 output_mad_hdr->MgmtClass = MAD_MGMT_CLASS_COMM_MGT;
1312 output_mad_hdr->ClassVersion = IBCM_MAD_CLASS_VERSION;
1313 output_mad_hdr->R_Method = method;
1314
1315 return (IBT_SUCCESS);
1316 }
1317
1318 /*
1319 * ibcm_free_ibmf_msg:
1320 * Frees the buffer and ibmf message associated with an outgoing CM message.
1321 * This function should only be used to free messages created by
1322 * ibcm_alloc_out_msg. Will return IBCM_FAILURE if the ibmf_free_msg() call
1323 * fails and IBCM_SUCCESS otherwise.
1324 */
1325 ibcm_status_t
1326 ibcm_free_out_msg(ibmf_handle_t ibmf_handle, ibmf_msg_t **ibmf_msgpp)
1327 {
1328 int ibmf_status;
1329
1330 kmem_free((*ibmf_msgpp)->im_msgbufs_send.im_bufs_mad_hdr,
1331 IBCM_MAD_SIZE);
1332
1333 if ((ibmf_status = ibmf_free_msg(ibmf_handle, ibmf_msgpp)) !=
1334 IBMF_SUCCESS) {
1335 IBTF_DPRINTF_L2(cmlog, "ibcm_free_out_msg: "
1336 "ibmf_free_msg failed %d", ibmf_status);
1337 return (IBCM_FAILURE);
1338 } else
1339 return (IBCM_SUCCESS);
1340 }
1341
1342 ibcm_qp_list_t *
1343 ibcm_find_qp(ibcm_hca_info_t *hcap, int port_no, ib_pkey_t pkey)
1344 {
1345 ibcm_qp_list_t *entry;
1346 ibmf_qp_handle_t ibmf_qp;
1347 int ibmf_status;
1348
1349 mutex_enter(&ibcm_qp_list_lock);
1350
1351 /*
1352 * CM currently does not track port up and down status. If tracking of
1353 * " port status" is added in the future, then CM could be optimized to
1354 * re-use other ports on hcap, if the port associated with the above
1355 * port_no is down. But, the issue of "reachability" needs to be
1356 * handled, before selecting an alternative port different from above.
1357 */
1358 entry = hcap->hca_port_info[port_no-1].port_qplist;
1359 while (entry != NULL) {
1360 if (entry->qp_pkey == pkey) {
1361 ++entry->qp_ref_cnt;
1362 mutex_exit(&ibcm_qp_list_lock);
1363 return (entry);
1364 }
1365 entry = entry->qp_next;
1366 }
1367
1368 /*
1369 * entry not found, attempt to alloc a qp
1370 * This may be optimized in the future, to allocate ibmf qp's
1371 * once the "CM mgmt pkeys" are precisely known.
1372 */
1373 ibmf_status = ibmf_alloc_qp(
1374 hcap->hca_port_info[port_no-1].port_ibmf_hdl, pkey, IB_GSI_QKEY,
1375 IBMF_ALT_QP_MAD_NO_RMPP, &ibmf_qp);
1376
1377 if (ibmf_status != IBMF_SUCCESS) {
1378 mutex_exit(&ibcm_qp_list_lock);
1379 IBTF_DPRINTF_L2(cmlog, "ibcm_find_qp: failed to alloc IBMF QP"
1380 "for Pkey = %x port_no = %x status = %d hcaguid = %llXp",
1381 pkey, port_no, ibmf_status, hcap->hca_guid);
1382 /*
1383 * This may be optimized in the future, so as CM would attempt
1384 * to re-use other QP's whose ref cnt is 0 in the respective
1385 * port_qplist, by doing an ibmf_modify_qp with pkey above.
1386 */
1387 return (NULL);
1388 }
1389
1390 entry = kmem_alloc(sizeof (ibcm_qp_list_t), KM_SLEEP);
1391 entry->qp_next = hcap->hca_port_info[port_no-1].port_qplist;
1392 hcap->hca_port_info[port_no-1].port_qplist = entry;
1393 entry->qp_cm = ibmf_qp;
1394 entry->qp_ref_cnt = 1;
1395 entry->qp_pkey = pkey;
1396 entry->qp_port = &(hcap->hca_port_info[port_no-1]);
1397
1398 mutex_exit(&ibcm_qp_list_lock);
1399
1400 /* set-up the handler */
1401 ibmf_status = ibmf_setup_async_cb(
1402 hcap->hca_port_info[port_no-1].port_ibmf_hdl, ibmf_qp,
1403 ibcm_recv_cb, entry, 0);
1404
1405 ASSERT(ibmf_status == IBMF_SUCCESS);
1406
1407 #ifdef DEBUG
1408 ibcm_query_qp(hcap->hca_port_info[port_no-1].port_ibmf_hdl, ibmf_qp);
1409 #endif
1410
1411 return (entry);
1412 }
1413
1414 void
1415 ibcm_release_qp(ibcm_qp_list_t *cm_qp_entry)
1416 {
1417 mutex_enter(&ibcm_qp_list_lock);
1418 --cm_qp_entry->qp_ref_cnt;
1419 ASSERT(cm_qp_entry->qp_ref_cnt >= 0);
1420 mutex_exit(&ibcm_qp_list_lock);
1421 }
1422
1423
1424 /* called holding the ibcm_qp_list_lock mutex */
1425 ibcm_status_t
1426 ibcm_free_qp(ibcm_qp_list_t *cm_qp_entry)
1427 {
1428 int ibmf_status;
1429
1430 IBTF_DPRINTF_L5(cmlog, "ibcm_free_qp: qp_hdl %p ref_cnt %d pkey %x",
1431 cm_qp_entry->qp_cm, cm_qp_entry->qp_ref_cnt, cm_qp_entry->qp_pkey);
1432
1433 /* check, there are no users of this ibmf qp */
1434 if (cm_qp_entry->qp_ref_cnt != 0)
1435 return (IBCM_FAILURE);
1436
1437 /* Tear down the receive callback */
1438 ibmf_status = ibmf_tear_down_async_cb(
1439 cm_qp_entry->qp_port->port_ibmf_hdl, cm_qp_entry->qp_cm, 0);
1440 if (ibmf_status != IBMF_SUCCESS) {
1441 IBTF_DPRINTF_L2(cmlog, "ibcm_free_qp: "
1442 "ibmf_tear_down_async_cb failed %d port_num %d",
1443 ibmf_status, cm_qp_entry->qp_port->port_num);
1444 return (IBCM_FAILURE);
1445 }
1446
1447 ibmf_status = ibmf_free_qp(cm_qp_entry->qp_port->port_ibmf_hdl,
1448 &cm_qp_entry->qp_cm, 0);
1449 if (ibmf_status != IBMF_SUCCESS) {
1450 IBTF_DPRINTF_L2(cmlog, "ibcm_free_qp: ibmf_free_qp failed for"
1451 " ibmf_status %d qp hdl %p port_no %x", ibmf_status,
1452 cm_qp_entry->qp_cm, cm_qp_entry->qp_port->port_num);
1453 return (IBCM_FAILURE);
1454 }
1455
1456 return (IBCM_SUCCESS);
1457 }
1458
1459 ibcm_status_t
1460 ibcm_free_allqps(ibcm_hca_info_t *hcap, int port_no)
1461 {
1462 ibcm_qp_list_t *entry, *freed;
1463 ibcm_status_t ibcm_status = IBCM_SUCCESS;
1464
1465 IBTF_DPRINTF_L5(cmlog, "ibcm_free_allqps: hcap %p port_no %d", hcap,
1466 port_no);
1467
1468 mutex_enter(&ibcm_qp_list_lock);
1469 entry = hcap->hca_port_info[port_no-1].port_qplist;
1470 while ((entry != NULL) &&
1471 ((ibcm_status = ibcm_free_qp(entry)) == IBCM_SUCCESS)) {
1472 freed = entry;
1473 entry = entry->qp_next;
1474 kmem_free(freed, sizeof (ibcm_qp_list_t));
1475 }
1476
1477 if (ibcm_status != IBCM_SUCCESS) /* sanity the linked list */
1478 hcap->hca_port_info[port_no-1].port_qplist = entry;
1479 else /* all ibmf qp's of port must have been free'd successfully */
1480 hcap->hca_port_info[port_no-1].port_qplist = NULL;
1481
1482 mutex_exit(&ibcm_qp_list_lock);
1483 return (ibcm_status);
1484 }
1485
1486 /*
1487 * ibt_bind_service() and ibt_get_paths() needs the following helper function
1488 * to handle endianess in case of Service Data.
1489 */
1490 void
1491 ibcm_swizzle_from_srv(ibt_srv_data_t *sb_data, uint8_t *service_bytes)
1492 {
1493 uint8_t *p8 = service_bytes;
1494 uint16_t *p16;
1495 uint32_t *p32;
1496 uint64_t *p64;
1497 int i;
1498
1499 for (i = 0; i < 16; i++)
1500 *p8++ = sb_data->s_data8[i];
1501
1502 p16 = (uint16_t *)p8;
1503 for (i = 0; i < 8; i++)
1504 *p16++ = h2b16(sb_data->s_data16[i]);
1505
1506 p32 = (uint32_t *)p16;
1507 for (i = 0; i < 4; i++)
1508 *p32++ = h2b32(sb_data->s_data32[i]);
1509
1510 p64 = (uint64_t *)p32;
1511 for (i = 0; i < 2; i++)
1512 *p64++ = h2b64(sb_data->s_data64[i]);
1513 }
1514
1515 void
1516 ibcm_swizzle_to_srv(uint8_t *service_bytes, ibt_srv_data_t *sb_data)
1517 {
1518 uint8_t *p8 = service_bytes;
1519 uint16_t *p16;
1520 uint32_t *p32;
1521 uint64_t *p64;
1522 int i;
1523
1524 for (i = 0; i < 16; i++)
1525 sb_data->s_data8[i] = *p8++;
1526
1527 p16 = (uint16_t *)p8;
1528 for (i = 0; i < 8; i++)
1529 sb_data->s_data16[i] = h2b16(*p16++);
1530
1531 p32 = (uint32_t *)p16;
1532 for (i = 0; i < 4; i++)
1533 sb_data->s_data32[i] = h2b32(*p32++);
1534 p64 = (uint64_t *)p32;
1535
1536 for (i = 0; i < 2; i++)
1537 sb_data->s_data64[i] = h2b64(*p64++);
1538 }
1539
1540 /* Trace related functions */
1541
1542 void
1543 ibcm_init_conn_trace(ibcm_state_data_t *sp)
1544 {
1545 IBTF_DPRINTF_L5(cmlog, "ibcm_init_conn_trace: statep %p", sp);
1546
1547 /* Initialize trace related fields */
1548
1549 sp->conn_trace = kmem_zalloc(sizeof (ibcm_conn_trace_t), KM_SLEEP);
1550 if ((ibcm_enable_trace & 1) == 0)
1551 sp->conn_trace->conn_base_tm = gethrtime();
1552 sp->conn_trace->conn_allocated_trcnt = ibcm_conn_max_trcnt;
1553 sp->conn_trace->conn_trace_events =
1554 kmem_zalloc(sp->conn_trace->conn_allocated_trcnt, KM_SLEEP);
1555 sp->conn_trace->conn_trace_event_times =
1556 kmem_zalloc(sp->conn_trace->conn_allocated_trcnt *
1557 sizeof (tm_diff_type), KM_SLEEP);
1558 }
1559
1560 void
1561 ibcm_fini_conn_trace(ibcm_state_data_t *statep)
1562 {
1563 IBTF_DPRINTF_L5(cmlog, "ibcm_fini_conn_trace: statep %p tracep %p",
1564 statep, statep->conn_trace);
1565
1566 /* free the trace data */
1567 if (statep->conn_trace) {
1568 if (statep->conn_trace->conn_trace_events)
1569 kmem_free(statep->conn_trace->conn_trace_events,
1570 statep->conn_trace->conn_allocated_trcnt);
1571 if (statep->conn_trace->conn_trace_event_times)
1572 kmem_free(statep->conn_trace->conn_trace_event_times,
1573 statep->conn_trace->conn_allocated_trcnt *
1574 sizeof (tm_diff_type));
1575
1576 kmem_free(statep->conn_trace, sizeof (ibcm_conn_trace_t));
1577 }
1578 }
1579
1580 /* mostly used to profile connection establishment times with dtrace */
1581 void
1582 ibcm_established(hrtime_t time_diff)
1583 {
1584 if (time_diff > 1000000000LL) /* 1 second */
1585 IBTF_DPRINTF_L2(cmlog, "slow connection time (%d seconds)",
1586 (uint_t)(time_diff >> 30));
1587 }
1588
1589 void
1590 ibcm_insert_trace(void *statep, ibcm_state_rc_trace_qualifier_t event_qualifier)
1591 {
1592 ibcm_conn_trace_t *conn_trace;
1593 uint8_t conn_trace_ind;
1594 hrtime_t time_diff;
1595 hrtime_t hrt;
1596
1597 if (!(((ibcm_state_data_t *)statep)->conn_trace))
1598 return;
1599
1600 conn_trace = ((ibcm_state_data_t *)statep)->conn_trace;
1601
1602 if (!conn_trace->conn_trace_events)
1603 return;
1604
1605 IBTF_DPRINTF_L5(cmlog, "ibcm_insert_trace: statep %p event %d",
1606 statep, event_qualifier);
1607
1608 mutex_enter(&ibcm_trace_mutex);
1609
1610 /* No more trace memory available, hence return */
1611 if (conn_trace->conn_trace_ind == conn_trace->conn_allocated_trcnt) {
1612 mutex_exit(&ibcm_trace_mutex);
1613 return;
1614 } else
1615 ++conn_trace->conn_trace_ind;
1616
1617 conn_trace_ind = conn_trace->conn_trace_ind - 1;
1618
1619 conn_trace->conn_trace_events[conn_trace_ind] = event_qualifier;
1620
1621 if ((ibcm_enable_trace & 1) == 0) {
1622 hrt = gethrtime();
1623 time_diff = hrt - conn_trace->conn_base_tm;
1624 if (event_qualifier == IBCM_TRACE_CALLED_CONN_EST_EVENT)
1625 ibcm_established(time_diff);
1626 time_diff >>= 10;
1627 if (time_diff >= TM_DIFF_MAX) {
1628 /* RESET, future times are relative to new base time. */
1629 conn_trace->conn_base_tm = hrt;
1630 time_diff = 0;
1631 }
1632 conn_trace->conn_trace_event_times[conn_trace_ind] = time_diff;
1633 }
1634
1635 mutex_exit(&ibcm_trace_mutex);
1636
1637 IBTF_DPRINTF_L5(cmlog, "ibcm_insert_trace: statep %p inserted event %d",
1638 statep, event_qualifier);
1639 }
1640
1641 void
1642 ibcm_dump_conn_trace(void *statep)
1643 {
1644 IBTF_DPRINTF_L5(cmlog, "ibcm_dump_conn_trace: statep %p",
1645 statep);
1646
1647 mutex_enter(&ibcm_trace_print_mutex);
1648 ibcm_debug_buf[0] = '\0';
1649 ibcm_dump_conn_trbuf(statep, "ibcm: ", ibcm_debug_buf,
1650 IBCM_DEBUG_BUF_SIZE);
1651 if (ibcm_debug_buf[0] != '\0')
1652 IBTF_DPRINTF_L2(cmlog, "\n%s", ibcm_debug_buf);
1653
1654 #ifdef DEBUG
1655
1656 if (ibcm_test_mode > 1)
1657 cmn_err(CE_CONT, "IBCM DEBUG TRACE:\n%s", ibcm_debug_buf);
1658 #endif
1659
1660 mutex_exit(&ibcm_trace_print_mutex);
1661 }
1662
1663 void
1664 ibcm_dump_conn_trbuf(void *statep, char *line_prefix, char *buf, int buf_size)
1665 {
1666 ibcm_conn_trace_t *conn_trace;
1667 int tr_ind;
1668 ibcm_state_data_t *sp;
1669 int cur_size = 0; /* size of item copied */
1670 int rem_size; /* remaining size in trace buffer */
1671 int next_data = 0; /* location where next item copied */
1672
1673 if ((buf == NULL) || (buf_size <= 0))
1674 return;
1675
1676 sp = (ibcm_state_data_t *)statep;
1677
1678 if (!sp->conn_trace)
1679 return;
1680
1681 conn_trace = sp->conn_trace;
1682
1683 if (!conn_trace->conn_trace_events)
1684 return;
1685
1686 rem_size = buf_size;
1687
1688 /* Print connection level global data */
1689
1690 /* Print statep, local comid, local qpn */
1691 cur_size = snprintf(&buf[next_data], rem_size, "%s%s0x%p\n%s%s0x%p\n"
1692 "%s%s0x%x/%llx/%d\n%s%s0x%x\n%s%s0x%x/%llx\n%s%s0x%x\n%s%s%llu\n",
1693 line_prefix, event_str[IBCM_DISPLAY_SID], (void *)sp,
1694 line_prefix, event_str[IBCM_DISPLAY_CHAN], (void *)sp->channel,
1695 line_prefix, event_str[IBCM_DISPLAY_LCID], sp->local_comid,
1696 (longlong_t)sp->local_hca_guid, sp->prim_port,
1697 line_prefix, event_str[IBCM_DISPLAY_LQPN], sp->local_qpn,
1698 line_prefix, event_str[IBCM_DISPLAY_RCID], sp->remote_comid,
1699 (longlong_t)sp->remote_hca_guid,
1700 line_prefix, event_str[IBCM_DISPLAY_RQPN], sp->remote_qpn,
1701 line_prefix, event_str[IBCM_DISPLAY_TM], conn_trace->conn_base_tm);
1702
1703 rem_size = rem_size - cur_size;
1704 if (rem_size <= 0) {
1705 buf[buf_size-1] = '\n';
1706 return;
1707 }
1708
1709 next_data = next_data + cur_size;
1710
1711 for (tr_ind = 0; tr_ind < conn_trace->conn_trace_ind; tr_ind++) {
1712 cur_size = snprintf(&buf[next_data], rem_size,
1713 "%s%sTM_DIFF %u\n", line_prefix,
1714 event_str[conn_trace->conn_trace_events[tr_ind]],
1715 conn_trace->conn_trace_event_times[tr_ind]);
1716 rem_size = rem_size - cur_size;
1717 if (rem_size <= 0) {
1718 buf[buf_size-1] = '\n';
1719 return;
1720 }
1721 next_data = next_data + cur_size;
1722 }
1723
1724 buf[next_data] = '\0';
1725 IBTF_DPRINTF_L5(cmlog, "ibcm_dump_conn_trbuf: statep %p "
1726 "debug buf size %d bytes", statep, next_data);
1727 }
1728
1729
1730 #ifdef DEBUG
1731
1732 void
1733 ibcm_query_qp(ibmf_handle_t ibmf_hdl, ibmf_qp_handle_t ibmf_qp)
1734 {
1735 uint8_t qp_port_num;
1736 ib_qpn_t qp_num;
1737 ib_pkey_t qp_pkey;
1738 ib_qkey_t qp_qkey;
1739 int ibmf_status;
1740
1741 if (ibmf_qp == IBMF_QP_HANDLE_DEFAULT) {
1742 IBTF_DPRINTF_L4(cmlog, "ibcm_query_qp: QP1");
1743 return;
1744 }
1745
1746 ibmf_status =
1747 ibmf_query_qp(ibmf_hdl, ibmf_qp, &qp_num, &qp_pkey, &qp_qkey,
1748 &qp_port_num, 0);
1749
1750 ASSERT(ibmf_status == IBMF_SUCCESS);
1751
1752 IBTF_DPRINTF_L5(cmlog, "ibcm_query_qp: qpn %x qkey %x pkey %x port %d",
1753 qp_num, qp_qkey, qp_pkey, qp_port_num);
1754 }
1755
1756 /*
1757 * ibcm_dump_raw_message:
1758 * dumps 256 bytes of data of a raw message (REP/REQ/DREQ ...)
1759 * (can be called from the kernel debugger w/ the message pointer)
1760 *
1761 * Arguments:
1762 * msgp - the messages that needs to be dumped
1763 *
1764 * Return values: NONE
1765 */
1766 void
1767 ibcm_dump_raw_message(uchar_t *c)
1768 {
1769 int i;
1770
1771 for (i = 0; i < IBCM_MAD_SIZE; i += 16) {
1772 /* print in batches of 16 chars at a time */
1773 IBTF_DPRINTF_L4(cmlog,
1774 "%x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
1775 c[i], c[i + 1], c[i + 2], c[i + 3], c[i + 4], c[i + 5],
1776 c[i + 6], c[i + 7], c[i + 8], c[i + 9], c[i + 10],
1777 c[i + 11], c[i + 12], c[i + 13], c[i + 14], c[i + 15]);
1778 }
1779 }
1780
1781
1782 /*
1783 * ibcm_dump_srv_rec:
1784 * Dumps Service Records.
1785 *
1786 * Arguments:
1787 * srv_rec - the pointer to sa_service_record_t struct.
1788 *
1789 * Return values: NONE
1790 */
1791 void
1792 ibcm_dump_srvrec(sa_service_record_t *srv_rec)
1793 {
1794 uint8_t i;
1795
1796 IBTF_DPRINTF_L4(cmlog, "ibcm_dump_srvrec: Service Records");
1797 IBTF_DPRINTF_L4(cmlog, "SID : 0x%016llX", srv_rec->ServiceID);
1798 IBTF_DPRINTF_L4(cmlog, "Svc GID : 0x%016llX:0x%016llX",
1799 srv_rec->ServiceGID.gid_prefix, srv_rec->ServiceGID.gid_guid);
1800 IBTF_DPRINTF_L4(cmlog, "Svc PKey : 0x%X", srv_rec->ServiceP_Key);
1801
1802 IBTF_DPRINTF_L4(cmlog, "Svc Lease : 0x%lX", srv_rec->ServiceLease);
1803 IBTF_DPRINTF_L4(cmlog, "Svc Key-hi: 0x%016llX", srv_rec->ServiceKey_hi);
1804 IBTF_DPRINTF_L4(cmlog, "Svc Key-lo: 0x%016llX", srv_rec->ServiceKey_lo);
1805 IBTF_DPRINTF_L4(cmlog, "Svc Name : %s", srv_rec->ServiceName);
1806 IBTF_DPRINTF_L4(cmlog, "Svc Data : ");
1807 for (i = 0; i < IB_SVC_DATA_LEN; i += 8) {
1808 IBTF_DPRINTF_L4(cmlog,
1809 "\t 0x%X, 0x%X, 0x%X, 0x%X, 0x%X, 0x%X, 0x%X, 0x%X",
1810 srv_rec->ServiceData[i], srv_rec->ServiceData[i+1],
1811 srv_rec->ServiceData[i+2], srv_rec->ServiceData[i+3],
1812 srv_rec->ServiceData[i+4], srv_rec->ServiceData[i+5],
1813 srv_rec->ServiceData[i+6], srv_rec->ServiceData[i+7]);
1814 }
1815 }
1816
1817
1818 /*
1819 * ibcm_dump_pathrec:
1820 * Dumps Path Records.
1821 *
1822 * Arguments:
1823 * path_rec - the pointer to sa_path_record_t struct.
1824 *
1825 * Return values: NONE
1826 */
1827 void
1828 ibcm_dump_pathrec(sa_path_record_t *path_rec)
1829 {
1830 IBTF_DPRINTF_L5(cmlog, "Path Record:");
1831 IBTF_DPRINTF_L5(cmlog, "SGID: (sn_prefix) %016llX",
1832 path_rec->SGID.gid_prefix);
1833 IBTF_DPRINTF_L5(cmlog, "SGID: (GUID) %016llX",
1834 path_rec->SGID.gid_guid);
1835 IBTF_DPRINTF_L5(cmlog, "DGID: (sn_prefix) %016llX",
1836 path_rec->DGID.gid_prefix);
1837 IBTF_DPRINTF_L5(cmlog, "DGID: (GUID) %016llX",
1838 path_rec->DGID.gid_guid);
1839 IBTF_DPRINTF_L5(cmlog, "SLID: %04X", path_rec->SLID);
1840 IBTF_DPRINTF_L5(cmlog, "DLID: %04X", path_rec->DLID);
1841 IBTF_DPRINTF_L5(cmlog, "Raw Traffic: %01X", path_rec->RawTraffic);
1842 IBTF_DPRINTF_L5(cmlog, "Flow Label: %05X", path_rec->FlowLabel);
1843 IBTF_DPRINTF_L5(cmlog, "Hop Limit: %02X", path_rec->HopLimit);
1844 IBTF_DPRINTF_L5(cmlog, "TClass: %02X", path_rec->TClass);
1845 IBTF_DPRINTF_L5(cmlog, "Reversible: %01X", path_rec->Reversible);
1846 IBTF_DPRINTF_L5(cmlog, "Numb Paths: %02d", path_rec->NumbPath);
1847 IBTF_DPRINTF_L5(cmlog, "P_Key: %04X", path_rec->P_Key);
1848 IBTF_DPRINTF_L5(cmlog, "SL: %02X", path_rec->SL);
1849 IBTF_DPRINTF_L5(cmlog, "Path MTU Selector: %01X",
1850 path_rec->MtuSelector);
1851 IBTF_DPRINTF_L5(cmlog, "Path MTU: %02X", path_rec->Mtu);
1852 IBTF_DPRINTF_L5(cmlog, "Path Rate Selector:%01X",
1853 path_rec->RateSelector);
1854 IBTF_DPRINTF_L5(cmlog, "Path Rate: %02X", path_rec->Rate);
1855 IBTF_DPRINTF_L5(cmlog, "Packet LT Selector:%01X",
1856 path_rec->PacketLifeTimeSelector);
1857 IBTF_DPRINTF_L5(cmlog, "Packet Life Time: %d (dec)",
1858 path_rec->PacketLifeTime);
1859 IBTF_DPRINTF_L5(cmlog, "Preference Bit: %02X", path_rec->Preference);
1860 }
1861
1862 /*
1863 * ibcm_dump_node_rec:
1864 * Dumps Node Records.
1865 *
1866 * Arguments:
1867 * nrec - the pointer to sa_node_record_t struct.
1868 *
1869 * Return values: NONE
1870 */
1871 void
1872 ibcm_dump_noderec(sa_node_record_t *nrec)
1873 {
1874 IBTF_DPRINTF_L5(cmlog, "ibcm_dump_noderec: Node Info Record");
1875 IBTF_DPRINTF_L5(cmlog, "LID : %04X", nrec->LID);
1876 IBTF_DPRINTF_L5(cmlog, "Base Ver : %02X", nrec->NodeInfo.BaseVersion);
1877 IBTF_DPRINTF_L5(cmlog, "Class Ver : %02X", nrec->NodeInfo.ClassVersion);
1878 IBTF_DPRINTF_L5(cmlog, "Node Type : %02d", nrec->NodeInfo.NodeType);
1879 IBTF_DPRINTF_L5(cmlog, "Num Ports : %02X", nrec->NodeInfo.NumPorts);
1880 IBTF_DPRINTF_L5(cmlog, "SysImgGUID: %016llX",
1881 nrec->NodeInfo.SystemImageGUID);
1882 IBTF_DPRINTF_L5(cmlog, "NODE GUID : %016llX", nrec->NodeInfo.NodeGUID);
1883 IBTF_DPRINTF_L5(cmlog, "Port GUID : %016llX", nrec->NodeInfo.PortGUID);
1884 IBTF_DPRINTF_L5(cmlog, "PartionCap: %04X", nrec->NodeInfo.PartitionCap);
1885 IBTF_DPRINTF_L5(cmlog, "Device ID : %04X", nrec->NodeInfo.DeviceID);
1886 IBTF_DPRINTF_L5(cmlog, "Revision : %06X", nrec->NodeInfo.Revision);
1887 IBTF_DPRINTF_L5(cmlog, "LocalPort#: %02X", nrec->NodeInfo.LocalPortNum);
1888 IBTF_DPRINTF_L5(cmlog, "Vendor ID : %06X", nrec->NodeInfo.VendorID);
1889 IBTF_DPRINTF_L5(cmlog, "Description: %s",
1890 (char *)&nrec->NodeDescription);
1891 }
1892 #endif
1893
1894 /*
1895 * ibcm_ibtl_node_info:
1896 * Get the node record of the destination specified by lid via the HCA
1897 * and port specified.
1898 *
1899 * Arguments:
1900 * hca_guid - GUID of the local HCA.
1901 * port - port in the HCA to be used.
1902 * lid - destination LID
1903 * node_info_p - pointer to the Node Info to be returned.
1904 *
1905 * Return values:
1906 * IBT_SUCCESS : Got the node record sucessfully
1907 * IBT_FILURE : Failed to get the node record.
1908 */
1909 ibt_status_t
1910 ibcm_ibtl_node_info(ib_guid_t hca_guid, uint8_t port, ib_lid_t lid,
1911 ibt_node_info_t *node_info_p)
1912 {
1913 sa_node_record_t nr_req, *nr_resp;
1914 void *res_p;
1915 ibmf_saa_handle_t saa_handle;
1916 ibt_status_t ibt_status;
1917 ibcm_hca_info_t *hcap;
1918 uint_t num_rec;
1919 size_t len;
1920
1921 IBTF_DPRINTF_L3(cmlog, "ibcm_ibtl_node_info: ENTER: port %x "
1922 "guid %llx\n", port, hca_guid);
1923
1924 hcap = ibcm_find_hca_entry(hca_guid);
1925 if (hcap == NULL) {
1926 IBTF_DPRINTF_L2(cmlog, "ibcm_ibtl_node_info: "
1927 "HCA(%llX) info not found", hca_guid);
1928 return (IBT_FAILURE);
1929 }
1930
1931 /* Get SA Access Handle. */
1932 saa_handle = ibcm_get_saa_handle(hcap, port);
1933 if (saa_handle == NULL) {
1934 IBTF_DPRINTF_L2(cmlog, "ibcm_ibtl_node_info: "
1935 "Port %d of HCA (%llX) is NOT ACTIVE", port, hca_guid);
1936 ibcm_dec_hca_acc_cnt(hcap);
1937 return (IBT_FAILURE);
1938 }
1939
1940 /* Retrieve Node Records from SA Access. */
1941 bzero(&nr_req, sizeof (sa_node_record_t));
1942 nr_req.LID = lid;
1943
1944 ibt_status = ibcm_get_node_rec(saa_handle, &nr_req,
1945 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len);
1946 if (ibt_status != IBT_SUCCESS) {
1947 IBTF_DPRINTF_L2(cmlog, "ibcm_ibtl_node_info: "
1948 "failed (%d) to get Node records", ibt_status);
1949 ibcm_dec_hca_acc_cnt(hcap);
1950 return (IBT_FAILURE);
1951 }
1952
1953 num_rec = len/sizeof (sa_node_record_t);
1954 nr_resp = (sa_node_record_t *)(uchar_t *)res_p;
1955
1956 if ((nr_resp != NULL) && (num_rec > 0)) {
1957 IBCM_DUMP_NODE_REC(nr_resp);
1958
1959 node_info_p->n_sys_img_guid =
1960 nr_resp->NodeInfo.SystemImageGUID;
1961 node_info_p->n_node_guid =
1962 nr_resp->NodeInfo.NodeGUID;
1963 node_info_p->n_port_guid =
1964 nr_resp->NodeInfo.PortGUID;
1965 node_info_p->n_dev_id =
1966 nr_resp->NodeInfo.DeviceID;
1967 node_info_p->n_revision =
1968 nr_resp->NodeInfo.Revision;
1969 node_info_p->n_vendor_id =
1970 nr_resp->NodeInfo.VendorID;
1971 node_info_p->n_num_ports =
1972 nr_resp->NodeInfo.NumPorts;
1973 node_info_p->n_port_num =
1974 nr_resp->NodeInfo.LocalPortNum;
1975 node_info_p->n_node_type =
1976 nr_resp->NodeInfo.NodeType;
1977 (void) strncpy(node_info_p->n_description,
1978 (char *)&nr_resp->NodeDescription, 64);
1979
1980 kmem_free(nr_resp, len);
1981 }
1982 ibcm_dec_hca_acc_cnt(hcap);
1983 return (IBT_SUCCESS);
1984 }
1985
1986 /*
1987 * ibcm_ibmf_analyze_error:
1988 * Checks IBMF status and determines appropriate ibt status.
1989 *
1990 * Arguments:
1991 * ibmf_status - IBMF Status
1992 *
1993 * Return values:
1994 * ibt_status_t
1995 */
1996 ibt_status_t
1997 ibcm_ibmf_analyze_error(int ibmf_status)
1998 {
1999 if (ibt_check_failure(ibmf_status, NULL) != IBT_FAILURE_STANDARD) {
2000 /*
2001 * IBMF specific failure, return special error code
2002 * to the client so that it can retrieve any associated ENA.
2003 */
2004 return (ibmf_status);
2005 } else if (ibmf_status == IBMF_TRANS_TIMEOUT) {
2006 return (IBT_IBMF_TIMEOUT);
2007 } else {
2008 /*
2009 * IBMF failed for some other reason, invalid arguments etc.
2010 * Analyze, log ENA with IBTF and obtain a special ibt_status_t
2011 * that indicates IBMF failure.
2012 */
2013 if ((ibmf_status == IBMF_BAD_CLASS) ||
2014 (ibmf_status == IBMF_BAD_HANDLE) ||
2015 (ibmf_status == IBMF_BAD_QP_HANDLE) ||
2016 (ibmf_status == IBMF_BAD_NODE) ||
2017 (ibmf_status == IBMF_BAD_PORT) ||
2018 (ibmf_status == IBMF_BAD_VERSION) ||
2019 (ibmf_status == IBMF_BAD_FLAGS) ||
2020 (ibmf_status == IBMF_BAD_SIZE) ||
2021 (ibmf_status == IBMF_INVALID_GID) ||
2022 (ibmf_status == IBMF_INVALID_ARG) ||
2023 (ibmf_status == IBMF_INVALID_FIELD) ||
2024 (ibmf_status == IBMF_UNSUPP_METHOD) ||
2025 (ibmf_status == IBMF_UNSUPP_METHOD_ATTR)) {
2026
2027 /*
2028 * These errors, we should not see...
2029 * something really bad happened!.
2030 */
2031 IBTF_DPRINTF_L2(cmlog, "ibcm_ibmf_analyze_error: "
2032 "Unexpected ERROR from IBMF - %d", ibmf_status);
2033 }
2034 return (ibt_get_module_failure(IBT_FAILURE_IBMF, 0));
2035 }
2036 }