1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/ib/mgt/ibcm/ibcm_impl.h>
26 #include <sys/ib/ibtl/ibti.h>
27 #include <sys/ib/mgt/ibcm/ibcm_arp.h>
28
29 /*
30 * ibcm_ti.c
31 * These routines implement the Communication Manager's interfaces to IBTL.
32 */
33
34 /* CM rc recycle task args structure definition */
35 typedef struct ibcm_taskq_recycle_arg_s {
36 ibt_channel_hdl_t rc_chan;
37 ibt_cep_flags_t control;
38 uint8_t hca_port_num;
39 ibt_recycle_handler_t func;
40 void *arg;
41 } ibcm_taskq_recycle_arg_t;
42
43 static ibt_status_t ibcm_init_reply_addr(ibcm_hca_info_t *hcap,
44 ibcm_mad_addr_t *reply_addr, ibt_chan_open_args_t *chan_args,
45 ibt_chan_open_flags_t flags, ib_time_t *cm_pkt_lt, ib_lid_t prim_slid);
46 static void ibcm_process_abort_via_taskq(void *args);
47 static ibt_status_t ibcm_process_rc_recycle_ret(void *recycle_arg);
48 static ibt_status_t ibcm_process_join_mcg(void *taskq_arg);
49 static void ibcm_process_async_join_mcg(void *tq_arg);
50
51 ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *,
52 uint64_t c_mask, void *, size_t *);
53
54 static ibt_status_t ibcm_close_rc_channel(ibt_channel_hdl_t channel,
55 ibcm_state_data_t *statep, ibt_execution_mode_t mode);
56
57 /* Address Record management definitions */
58 #define IBCM_DAPL_ATS_NAME "DAPL Address Translation Service"
59 #define IBCM_DAPL_ATS_SID 0x10000CE100415453ULL
60 #define IBCM_DAPL_ATS_NBYTES 16
61 ibcm_svc_info_t *ibcm_ar_svcinfop;
62 ibcm_ar_t *ibcm_ar_list;
63
64 /*
65 * Tunable parameter to turnoff the overriding of pi_path_mtu value.
66 * 1 By default override the path record's pi_path_mtu value to
67 * IB_MTU_1K for all RC channels. This is done only for the
68 * channels established on Tavor HCA and the path's pi_path_mtu
69 * is greater than IB_MTU_1K.
70 * 0 Do not override, use pi_path_mtu by default.
71 */
72 int ibcm_override_path_mtu = 1;
73
74 #ifdef DEBUG
75 static void ibcm_print_reply_addr(ibt_channel_hdl_t channel,
76 ibcm_mad_addr_t *cm_reply_addr);
77 #endif
78
79 /*
80 * ibt_open_rc_channel()
81 * ibt_open_rc_channel opens a communication channel on the specified
82 * channel to the specified service. For connection service type qp's
83 * the CM initiates the CEP to establish the connection and transitions
84 * the QP/EEC to the "Ready to send" State modifying the QP/EEC's
85 * attributes as necessary.
86 * The implementation of this function assumes that alt path is different
87 * from primary path. It is assumed that the Path functions ensure that.
88 *
89 * RETURN VALUES:
90 * IBT_SUCCESS on success (or respective failure on error)
91 */
92 ibt_status_t
93 ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags,
94 ibt_execution_mode_t mode, ibt_chan_open_args_t *chan_args,
95 ibt_rc_returns_t *ret_args)
96 {
97 /* all fields that are related to REQ MAD formation */
98
99 ib_pkey_t prim_pkey;
100 ib_lid_t primary_slid, alternate_slid;
101 ib_qpn_t local_qpn = 0;
102 ib_guid_t hca_guid;
103 ib_qkey_t local_qkey = 0;
104 ib_eecn_t local_eecn = 0;
105 ib_eecn_t remote_eecn = 0;
106 boolean_t primary_grh;
107 boolean_t alternate_grh = B_FALSE;
108 ib_lid_t base_lid;
109 ib_com_id_t local_comid;
110 ibmf_msg_t *ibmf_msg, *ibmf_msg_dreq;
111 ibcm_req_msg_t *req_msgp;
112
113 uint8_t rdma_in, rdma_out;
114 uint8_t cm_retries;
115 uint64_t local_cm_proc_time; /* In usec */
116 uint8_t local_cm_resp_time; /* IB time */
117 uint64_t remote_cm_resp_time; /* In usec */
118 uint32_t starting_psn = 0;
119
120 /* CM path related fields */
121 ibmf_handle_t ibmf_hdl;
122 ibcm_qp_list_t *cm_qp_entry;
123 ibcm_mad_addr_t cm_reply_addr;
124
125 uint8_t cm_pkt_lt;
126
127 /* Local args for ibtl/internal CM functions called within */
128 ibt_status_t status;
129 ibcm_status_t lkup_status;
130 ibt_qp_query_attr_t qp_query_attr;
131
132 /* Other misc local args */
133 ibt_priv_data_len_t len;
134 ibcm_hca_info_t *hcap;
135 ibcm_state_data_t *statep;
136 uint8_t port_no;
137
138 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel(chan %p, %X, %x, %p, %p)",
139 channel, flags, mode, chan_args, ret_args);
140
141 if (IBCM_INVALID_CHANNEL(channel)) {
142 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: invalid channel");
143 return (IBT_CHAN_HDL_INVALID);
144 }
145
146 /* cm handler should always be specified */
147 if (chan_args->oc_cm_handler == NULL) {
148 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
149 "CM handler is not be specified", channel);
150 return (IBT_INVALID_PARAM);
151 }
152
153 if (mode == IBT_NONBLOCKING) {
154 if (ret_args != NULL) {
155 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p"
156 " ret_args should be NULL when called in "
157 "non-blocking mode", channel);
158 return (IBT_INVALID_PARAM);
159 }
160 } else if (mode == IBT_BLOCKING) {
161 if (ret_args == NULL) {
162 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p"
163 " ret_args should be Non-NULL when called in "
164 "blocking mode", channel);
165 return (IBT_INVALID_PARAM);
166 }
167 if (ret_args->rc_priv_data_len > IBT_REP_PRIV_DATA_SZ) {
168 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p"
169 " private data length is too large", channel);
170 return (IBT_INVALID_PARAM);
171 }
172 if ((ret_args->rc_priv_data_len > 0) &&
173 (ret_args->rc_priv_data == NULL)) {
174 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p"
175 " rc_priv_data_len > 0, but rc_priv_data NULL",
176 channel);
177 return (IBT_INVALID_PARAM);
178 }
179 } else { /* any other mode is not valid for ibt_open_rc_channel */
180 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
181 "invalid mode %x specified", channel, mode);
182 return (IBT_INVALID_PARAM);
183 }
184
185 /*
186 * XXX: no support yet for ibt_chan_open_flags_t - IBT_OCHAN_DUP
187 */
188 if (flags & IBT_OCHAN_DUP) {
189 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
190 "Unsupported Flags specified: 0x%X", channel, flags);
191 return (IBT_INVALID_PARAM);
192 }
193
194 if ((flags & IBT_OCHAN_REDIRECTED) &&
195 (flags & IBT_OCHAN_PORT_REDIRECTED)) {
196 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
197 "Illegal to specify IBT_OCHAN_REDIRECTED and "
198 "IBT_OCHAN_PORT_REDIRECTED flags together", channel);
199 return (IBT_INVALID_PARAM);
200 }
201
202 if (((flags & IBT_OCHAN_REDIRECTED) &&
203 (chan_args->oc_cm_redirect_info == NULL)) ||
204 ((flags & IBT_OCHAN_PORT_REDIRECTED) &&
205 (chan_args->oc_cm_cep_path == NULL))) {
206 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
207 "Redirect flag specified, but respective arg is NULL",
208 channel);
209 return (IBT_INVALID_PARAM);
210 }
211
212 if ((flags & IBT_OCHAN_REDIRECTED) &&
213 (chan_args->oc_cm_redirect_info->rdi_dlid == 0) &&
214 (chan_args->oc_cm_redirect_info->rdi_gid.gid_guid == 0)) {
215 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
216 "Either rdi_dlid or rdi_gid must be specified for"
217 " IBT_OCHAN_REDIRECTED", channel);
218 return (IBT_INVALID_PARAM);
219 }
220
221 /* primary dlid and hca_port_num should never be zero */
222 port_no = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num;
223
224 if ((IBCM_PRIM_ADDS_VECT(chan_args).av_dlid == 0) && (port_no == 0)) {
225 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
226 "Primary Path's information is not valid", channel);
227 return (IBT_INVALID_PARAM);
228 }
229
230 /* validate SID */
231 if (chan_args->oc_path->pi_sid == 0) {
232 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
233 "ERROR: Service ID in path information is 0", channel);
234 return (IBT_INVALID_PARAM);
235 }
236 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p SID %llX",
237 channel, chan_args->oc_path->pi_sid);
238
239 /* validate rnr_retry_cnt (enum has more than 3 bits) */
240 if ((uint_t)chan_args->oc_path_rnr_retry_cnt > IBT_RNR_INFINITE_RETRY) {
241 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
242 "ERROR: oc_path_rnr_retry_cnt(%d) is out of range",
243 channel, chan_args->oc_path_rnr_retry_cnt);
244 return (IBT_INVALID_PARAM);
245 }
246
247 /*
248 * Ensure that client is not re-using a QP that is still associated
249 * with a statep
250 */
251 IBCM_GET_CHAN_PRIVATE(channel, statep);
252 if (statep != NULL) {
253 IBCM_RELEASE_CHAN_PRIVATE(channel);
254 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
255 "Channel being re-used on active side", channel);
256 return (IBT_CHAN_IN_USE);
257 }
258
259 /* Get GUID from Channel */
260 hca_guid = ibt_channel_to_hca_guid(channel);
261
262 /* validate QP's hca guid with that from primary path */
263 if (hca_guid != chan_args->oc_path->pi_hca_guid) {
264 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
265 "GUID from Channel and primary path don't match", channel);
266 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
267 "Channel GUID %llX primary path GUID %llX", channel,
268 hca_guid, chan_args->oc_path->pi_hca_guid);
269 return (IBT_CHAN_HDL_INVALID);
270 }
271
272 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
273 "Local HCA GUID %llX", channel, hca_guid);
274
275 status = ibt_query_qp(channel, &qp_query_attr);
276 if (status != IBT_SUCCESS) {
277 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
278 "ibt_query_qp failed %d", channel, status);
279 return (status);
280 }
281
282 /* If client specified "no port change on QP" */
283 if ((qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num !=
284 port_no) && (flags & IBT_OCHAN_PORT_FIXED)) {
285 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
286 "chan port %d and path port %d does not match", channel,
287 qp_query_attr.qp_info.qp_transport.rc.rc_path. \
288 cep_hca_port_num, port_no);
289 return (IBT_INVALID_PARAM);
290 }
291
292 if (qp_query_attr.qp_info.qp_trans != IBT_RC_SRV) {
293 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
294 "Invalid Channel type: Applicable only to RC Channel",
295 channel);
296 return (IBT_CHAN_SRV_TYPE_INVALID);
297 }
298
299 /* Check if QP is in INIT state or not */
300 if (qp_query_attr.qp_info.qp_state != IBT_STATE_INIT) {
301 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
302 "QP is not in INIT state %x", channel,
303 qp_query_attr.qp_info.qp_state);
304 return (IBT_CHAN_STATE_INVALID);
305 }
306
307 local_qpn = qp_query_attr.qp_qpn;
308
309 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p Active QPN 0x%x",
310 channel, local_qpn);
311
312 #ifdef NO_EEC_SUPPORT_YET
313
314 if (flags & IBT_OCHAN_RDC_EXISTS) {
315 ibt_eec_query_attr_t eec_query_attr;
316
317 local_qkey = qp_query_attr.qp_info.qp_transport.rd_qkey;
318
319 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: RD");
320
321 status = ibt_query_eec(channel, &eec_query_attr);
322 if (status != IBT_SUCCESS) {
323 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p"
324 " ibt_query_eec failed %d", channel, status);
325 return (status);
326 }
327 local_eecn = eec_query_attr.eec_eecn;
328 }
329
330 #endif
331 if (chan_args->oc_path->pi_prim_pkt_lt > ibcm_max_ib_pkt_lt) {
332 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
333 "Huge PktLifeTime %d, Max is %d", channel,
334 chan_args->oc_path->pi_prim_pkt_lt, ibcm_max_ib_pkt_lt);
335 return (IBT_PATH_PKT_LT_TOO_HIGH);
336 }
337
338 /* If no HCA found return failure */
339 if ((hcap = ibcm_find_hca_entry(hca_guid)) == NULL) {
340 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
341 "hcap is NULL. Probably hca is not in active state",
342 channel);
343 return (IBT_CHAN_HDL_INVALID);
344 }
345
346 rdma_out = chan_args->oc_rdma_ra_out;
347 rdma_in = chan_args->oc_rdma_ra_in;
348
349 if ((rdma_in > hcap->hca_max_rdma_in_qp) ||
350 (rdma_out > hcap->hca_max_rdma_out_qp)) {
351 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
352 "rdma in %d/out %d values exceed hca limits(%d/%d)",
353 channel, rdma_in, rdma_out, hcap->hca_max_rdma_in_qp,
354 hcap->hca_max_rdma_out_qp);
355 ibcm_dec_hca_acc_cnt(hcap);
356 return (IBT_INVALID_PARAM);
357 }
358
359 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
360 "rdma_in %d rdma_out %d", channel, rdma_in, rdma_out);
361
362 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no,
363 NULL, &base_lid);
364 if (status != IBT_SUCCESS) {
365 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
366 "primary port_num %d not active", channel, port_no);
367 ibcm_dec_hca_acc_cnt(hcap);
368 return (status);
369 }
370
371 /* Validate P_KEY Index */
372 status = ibt_index2pkey_byguid(hcap->hca_guid, port_no,
373 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix, &prim_pkey);
374 if (status != IBT_SUCCESS) {
375 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
376 "Invalid Primary PKeyIx %x", channel,
377 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix);
378 ibcm_dec_hca_acc_cnt(hcap);
379 return (status);
380 }
381
382 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
383 "primary_port_num %d primary_pkey 0x%x", channel, port_no,
384 prim_pkey);
385
386 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) &&
387 ((status = ibcm_hca_reinit_port(hcap, port_no - 1))
388 != IBT_SUCCESS)) {
389 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
390 "ibmf reg or callback setup failed during re-initialize",
391 channel);
392 ibcm_dec_hca_acc_cnt(hcap);
393 return (status);
394 }
395
396 ibmf_hdl = hcap->hca_port_info[port_no - 1].port_ibmf_hdl;
397 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
398 "primary ibmf_hdl = 0x%p", channel, ibmf_hdl);
399
400 primary_slid = base_lid + IBCM_PRIM_ADDS_VECT(chan_args).av_src_path;
401
402 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: channel 0x%p "
403 "primary SLID = %x", channel, primary_slid);
404
405 /* check first if alternate path exists or not as it is OPTIONAL */
406 if (IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num != 0) {
407 uint8_t alt_port_no;
408
409 alt_port_no = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num;
410
411 if (chan_args->oc_path->pi_alt_pkt_lt > ibcm_max_ib_pkt_lt) {
412 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
413 "Huge Alt Pkt lt %d", channel,
414 chan_args->oc_path->pi_alt_pkt_lt);
415 ibcm_dec_hca_acc_cnt(hcap);
416 return (IBT_PATH_PKT_LT_TOO_HIGH);
417 }
418
419 if (port_no != alt_port_no) {
420
421 status = ibt_get_port_state_byguid(hcap->hca_guid,
422 alt_port_no, NULL, &base_lid);
423 if (status != IBT_SUCCESS) {
424
425 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: "
426 "chan 0x%p alt_port_num %d inactive %d",
427 channel, alt_port_no, status);
428 ibcm_dec_hca_acc_cnt(hcap);
429 return (status);
430 }
431
432 }
433 alternate_slid =
434 base_lid + IBCM_ALT_ADDS_VECT(chan_args).av_src_path;
435
436 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
437 "alternate SLID = %x", channel, alternate_slid);
438 }
439
440 /*
441 * only pkey needs to be zero'ed, because all other fields are set in
442 * in ibcm_init_reply_addr. But, let's bzero the complete struct for
443 * any future modifications.
444 */
445 bzero(&cm_reply_addr, sizeof (cm_reply_addr));
446
447 /* Initialize the MAD destination address in stored_reply_addr */
448 if ((status = ibcm_init_reply_addr(hcap, &cm_reply_addr, chan_args,
449 flags, &cm_pkt_lt, primary_slid)) != IBT_SUCCESS) {
450
451 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
452 "ibcm_init_reply_addr failed status %d ", channel, status);
453 ibcm_dec_hca_acc_cnt(hcap);
454 return (status);
455 }
456
457
458 /* Initialize the pkey for CM MAD communication */
459 if (cm_reply_addr.rcvd_addr.ia_p_key == 0)
460 cm_reply_addr.rcvd_addr.ia_p_key = prim_pkey;
461
462 #ifdef DEBUG
463 ibcm_print_reply_addr(channel, &cm_reply_addr);
464 #endif
465
466 /* Retrieve an ibmf qp for sending CM MADs */
467 if ((cm_qp_entry = ibcm_find_qp(hcap, port_no,
468 cm_reply_addr.rcvd_addr.ia_p_key)) == NULL) {
469 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p "
470 "unable to allocate ibmf qp for CM MADs", channel);
471 ibcm_dec_hca_acc_cnt(hcap);
472 return (IBT_INSUFF_RESOURCE);
473 }
474
475
476 if (ibcm_alloc_comid(hcap, &local_comid) != IBCM_SUCCESS) {
477 ibcm_release_qp(cm_qp_entry);
478 ibcm_dec_hca_acc_cnt(hcap);
479 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p"
480 " Unable to allocate comid", channel);
481 return (IBT_INSUFF_KERNEL_RESOURCE);
482 }
483
484 /* allocate an IBMF mad buffer (REQ) */
485 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg,
486 MAD_METHOD_SEND)) != IBT_SUCCESS) {
487 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: "
488 "chan 0x%p ibcm_alloc_out_msg failed", channel);
489 ibcm_release_qp(cm_qp_entry);
490 ibcm_free_comid(hcap, local_comid);
491 ibcm_dec_hca_acc_cnt(hcap);
492 return (status);
493 }
494
495 /* allocate an IBMF mad buffer (DREQ) */
496 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg_dreq,
497 MAD_METHOD_SEND)) != IBT_SUCCESS) {
498 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: "
499 "chan 0x%p ibcm_alloc_out_msg failed", channel);
500 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg);
501 ibcm_release_qp(cm_qp_entry);
502 ibcm_free_comid(hcap, local_comid);
503 ibcm_dec_hca_acc_cnt(hcap);
504 return (status);
505 }
506
507 /* Init to Init, if QP's port does not match with path information */
508 if (qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num !=
509 IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num) {
510
511 ibt_qp_info_t qp_info;
512 ibt_cep_modify_flags_t cep_flags;
513
514 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: "
515 "chan 0x%p chan port %d", channel,
516 qp_query_attr.qp_info.qp_transport.rc.rc_path.\
517 cep_hca_port_num);
518
519 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: "
520 "chan 0x%p path port %d", channel, port_no);
521
522 bzero(&qp_info, sizeof (qp_info));
523 /* For now, set it to RC type */
524
525 qp_info.qp_trans = IBT_RC_SRV;
526 qp_info.qp_state = IBT_STATE_INIT;
527 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = port_no;
528
529 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT;
530
531 status = ibt_modify_qp(channel, cep_flags, &qp_info, NULL);
532
533 if (status != IBT_SUCCESS) {
534 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: "
535 "chan 0x%p ibt_modify_qp() = %d", channel, status);
536 ibcm_release_qp(cm_qp_entry);
537 ibcm_free_comid(hcap, local_comid);
538 ibcm_dec_hca_acc_cnt(hcap);
539 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg);
540 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg_dreq);
541 return (status);
542 } else
543 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: "
544 "chan 0x%p ibt_modify_qp() = %d", channel, status);
545 }
546
547 /* allocate ibcm_state_data_t before grabbing the WRITER lock */
548 statep = kmem_zalloc(sizeof (ibcm_state_data_t), KM_SLEEP);
549 rw_enter(&hcap->hca_state_rwlock, RW_WRITER);
550 lkup_status = ibcm_lookup_msg(IBCM_OUTGOING_REQ, local_comid, 0, 0,
551 hcap, &statep);
552 rw_exit(&hcap->hca_state_rwlock);
553
554 /* CM should be seeing this for the first time */
555 ASSERT(lkup_status == IBCM_LOOKUP_NEW);
556
557 /* Increment the hca's resource count */
558 ibcm_inc_hca_res_cnt(hcap);
559
560 /* Once a resource created on hca, no need to hold the acc cnt */
561 ibcm_dec_hca_acc_cnt(hcap);
562
563 statep->timerid = 0;
564 statep->local_hca_guid = hca_guid;
565 statep->local_qpn = local_qpn;
566 statep->stored_reply_addr.cm_qp_entry = cm_qp_entry;
567 statep->prim_port = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num;
568 statep->alt_port = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num;
569
570
571 /* Save "statep" as channel's CM private data. */
572 statep->channel = channel;
573 IBCM_SET_CHAN_PRIVATE(statep->channel, statep);
574
575 statep->stored_msg = ibmf_msg;
576 statep->dreq_msg = ibmf_msg_dreq;
577
578 /* Start filling in the REQ MAD */
579 req_msgp = (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg);
580 req_msgp->req_local_comm_id = h2b32(local_comid);
581 req_msgp->req_svc_id = h2b64(chan_args->oc_path->pi_sid);
582 req_msgp->req_local_ca_guid = h2b64(hca_guid);
583 req_msgp->req_local_qkey = h2b32(local_qkey); /* for EEC/RD */
584
585 /* Bytes 32-35 are req_local_qpn and req_off_resp_resources */
586 req_msgp->req_local_qpn_plus = h2b32(local_qpn << 8 | rdma_in);
587
588 /* Bytes 36-39 are req_local_eec_no and req_off_initiator_depth */
589 req_msgp->req_local_eec_no_plus = h2b32(local_eecn << 8 | rdma_out);
590
591 if (flags & IBT_OCHAN_REMOTE_CM_TM)
592 remote_cm_resp_time = chan_args->oc_remote_cm_time;
593 else
594 remote_cm_resp_time = ibcm_remote_response_time;
595
596 /*
597 * Bytes 40-43 - remote_eecn, remote_cm_resp_time, tran_type,
598 * IBT_CM_FLOW_CONTROL is always set by default.
599 */
600 req_msgp->req_remote_eecn_plus = h2b32(
601 remote_eecn << 8 | (ibt_usec2ib(remote_cm_resp_time) & 0x1f) << 3 |
602 IBT_RC_SRV << 1 | IBT_CM_FLOW_CONTROL);
603
604 if (flags & IBT_OCHAN_LOCAL_CM_TM)
605 local_cm_proc_time = chan_args->oc_local_cm_time;
606 else
607 local_cm_proc_time = ibcm_local_processing_time;
608
609 local_cm_resp_time = ibt_usec2ib(local_cm_proc_time +
610 2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt) +
611 ibcm_sw_delay);
612
613 /* save retry count */
614 statep->cep_retry_cnt = chan_args->oc_path_retry_cnt;
615
616 if (flags & IBT_OCHAN_STARTING_PSN)
617 starting_psn = chan_args->oc_starting_psn;
618
619 if (local_cm_resp_time > 0x1f)
620 local_cm_resp_time = 0x1f;
621
622 /* Bytes 44-47 are req_starting_psn, local_cm_resp_time and retry_cnt */
623 req_msgp->req_starting_psn_plus = h2b32(starting_psn << 8 |
624 local_cm_resp_time << 3 | statep->cep_retry_cnt);
625
626 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
627 "Prim Pkt lt (IB time) 0x%x", channel,
628 chan_args->oc_path->pi_prim_pkt_lt);
629
630 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
631 "local_cm_proc_time(usec) %d ", channel, local_cm_proc_time);
632
633 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
634 "local_cm_resp_time(ib_time) %d", channel, local_cm_resp_time);
635
636 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
637 "remote_cm_resp_time (usec) %d", channel, remote_cm_resp_time);
638
639 statep->starting_psn = starting_psn;
640
641 /* Pkey - bytes 48-49 */
642 req_msgp->req_part_key = h2b16(prim_pkey);
643
644 if (flags & IBT_OCHAN_CM_RETRY)
645 cm_retries = chan_args->oc_cm_retry_cnt;
646 else
647 cm_retries = ibcm_max_retries;
648
649 statep->max_cm_retries = statep->remaining_retry_cnt = cm_retries;
650 req_msgp->req_max_cm_retries_plus = statep->max_cm_retries << 4;
651
652 /*
653 * Check whether SRQ is associated with this Channel, if yes, then
654 * set the SRQ Exists bit in the REQ.
655 */
656 if (qp_query_attr.qp_srq != NULL) {
657 req_msgp->req_max_cm_retries_plus |= (1 << 3);
658 }
659
660 /*
661 * By default on Tavor, we override the PathMTU to 1K.
662 * To turn this off, set ibcm_override_path_mtu = 0.
663 */
664 if (ibcm_override_path_mtu && IBCM_IS_HCA_TAVOR(hcap) &&
665 (chan_args->oc_path->pi_path_mtu > IB_MTU_1K)) {
666 req_msgp->req_mtu_plus = IB_MTU_1K << 4 |
667 chan_args->oc_path_rnr_retry_cnt;
668 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p PathMTU"
669 " overridden to IB_MTU_1K(%d) from %d", channel, IB_MTU_1K,
670 chan_args->oc_path->pi_path_mtu);
671 } else
672 req_msgp->req_mtu_plus = chan_args->oc_path->pi_path_mtu << 4 |
673 chan_args->oc_path_rnr_retry_cnt;
674
675 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p CM retry cnt %d"
676 " staring PSN %x", channel, cm_retries, starting_psn);
677
678
679 #ifdef NO_EEC_SUPPORT_YET
680 if (flags & IBT_OCHAN_RDC_EXISTS)
681 req_msgp->req_mtu_plus |= 8;
682 #endif
683
684 /* Initialize the "primary" port stuff next - bytes 52-95 */
685 req_msgp->req_primary_l_port_lid = h2b16(primary_slid);
686 req_msgp->req_primary_r_port_lid =
687 h2b16(IBCM_PRIM_ADDS_VECT(chan_args).av_dlid);
688 req_msgp->req_primary_l_port_gid.gid_prefix =
689 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_prefix);
690 req_msgp->req_primary_l_port_gid.gid_guid =
691 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_guid);
692 req_msgp->req_primary_r_port_gid.gid_prefix =
693 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix);
694 req_msgp->req_primary_r_port_gid.gid_guid =
695 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid);
696 primary_grh = IBCM_PRIM_ADDS_VECT(chan_args).av_send_grh;
697
698 statep->remote_hca_guid = /* not correct, but helpful for debugging */
699 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid;
700
701 /* Bytes 88-91 - primary_flowlbl, and primary_srate */
702 req_msgp->req_primary_flow_label_plus =
703 h2b32(((primary_grh == B_TRUE) ?
704 (IBCM_PRIM_ADDS_VECT(chan_args).av_flow << 12) : 0) |
705 IBCM_PRIM_ADDS_VECT(chan_args).av_srate);
706 req_msgp->req_primary_traffic_class = (primary_grh == B_TRUE) ?
707 IBCM_PRIM_ADDS_VECT(chan_args).av_tclass : 0;
708 req_msgp->req_primary_hop_limit = (primary_grh == B_TRUE) ?
709 IBCM_PRIM_ADDS_VECT(chan_args).av_hop : 1;
710 req_msgp->req_primary_sl_plus =
711 IBCM_PRIM_ADDS_VECT(chan_args).av_srvl << 4 |
712 ((primary_grh == B_TRUE) ? 0 : 8);
713
714 req_msgp->req_primary_localtime_plus =
715 ibt_usec2ib((2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt)) +
716 ibt_ib2usec(hcap->hca_ack_delay)) << 3;
717
718 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan %p statep %p",
719 channel, statep);
720 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
721 "active hca_ack_delay (usec) %d", channel,
722 req_msgp->req_primary_localtime_plus);
723
724 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
725 "Sent primary cep timeout (IB Time) %d", channel,
726 hcap->hca_ack_delay);
727
728 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p prim_dlid %x ",
729 channel, IBCM_PRIM_ADDS_VECT(chan_args).av_dlid);
730
731 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
732 "prim GID %llX:%llX", channel,
733 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix,
734 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid);
735
736 /* Initialize the "alternate" port stuff - optional */
737 if (chan_args->oc_path->pi_alt_cep_path.cep_hca_port_num != 0) {
738 ib_gid_t tmp_gid;
739
740 req_msgp->req_alt_l_port_lid = h2b16(alternate_slid);
741 req_msgp->req_alt_r_port_lid =
742 h2b16(IBCM_ALT_ADDS_VECT(chan_args).av_dlid);
743 /*
744 * doing all this as req_alt_r/l_port_gid is at offset
745 * 100, 116 which is not divisible by 8
746 */
747
748 tmp_gid.gid_prefix =
749 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix);
750 tmp_gid.gid_guid =
751 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid);
752 bcopy(&tmp_gid, &req_msgp->req_alt_r_port_gid[0],
753 sizeof (ib_gid_t));
754 tmp_gid.gid_prefix =
755 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_prefix);
756 tmp_gid.gid_guid =
757 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_guid);
758
759 bcopy(&tmp_gid, &req_msgp->req_alt_l_port_gid[0],
760 sizeof (ib_gid_t));
761 alternate_grh = IBCM_ALT_ADDS_VECT(chan_args).av_send_grh;
762
763 /* Bytes 132-135 - alternate_flow_label, and alternate srate */
764 req_msgp->req_alt_flow_label_plus = h2b32(
765 (((alternate_grh == B_TRUE) ?
766 (IBCM_ALT_ADDS_VECT(chan_args).av_flow << 12) : 0) |
767 IBCM_ALT_ADDS_VECT(chan_args).av_srate));
768 req_msgp->req_alt_traffic_class = (alternate_grh == B_TRUE) ?
769 IBCM_ALT_ADDS_VECT(chan_args).av_tclass : 0;
770 req_msgp->req_alt_hop_limit = (alternate_grh == B_TRUE) ?
771 IBCM_ALT_ADDS_VECT(chan_args).av_hop : 1;
772 req_msgp->req_alt_sl_plus =
773 IBCM_ALT_ADDS_VECT(chan_args).av_srvl << 4 |
774 ((alternate_grh == B_TRUE) ? 0 : 8);
775 req_msgp->req_alt_localtime_plus = ibt_usec2ib((2 *
776 ibt_ib2usec(chan_args->oc_path->pi_alt_pkt_lt)) +
777 ibt_ib2usec(hcap->hca_ack_delay)) << 3;
778
779 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
780 "alt_dlid %x ", channel,
781 IBCM_ALT_ADDS_VECT(chan_args).av_dlid);
782
783 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p "
784 "alt GID %llX:%llX", channel,
785 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix,
786 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid);
787 }
788
789 len = min(chan_args->oc_priv_data_len, IBT_REQ_PRIV_DATA_SZ);
790 if ((len > 0) && chan_args->oc_priv_data)
791 bcopy(chan_args->oc_priv_data, req_msgp->req_private_data, len);
792
793 /* return_data is filled up in the state machine code */
794 if (ret_args != NULL) {
795 statep->open_return_data = ret_args;
796 }
797
798 /* initialize some statep fields here */
799 statep->mode = IBCM_ACTIVE_MODE;
800 statep->hcap = hcap;
801
802 statep->cm_handler = chan_args->oc_cm_handler;
803 statep->state_cm_private = chan_args->oc_cm_clnt_private;
804
805 statep->pkt_life_time =
806 ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt);
807
808 statep->timer_value = ibt_ib2usec(ibt_usec2ib(
809 2 * ibt_ib2usec(cm_pkt_lt) + remote_cm_resp_time));
810
811 /* Initialize statep->stored_reply_addr */
812 statep->stored_reply_addr.ibmf_hdl = ibmf_hdl;
813
814 /* Initialize stored reply addr fields */
815 statep->stored_reply_addr.grh_hdr = cm_reply_addr.grh_hdr;
816 statep->stored_reply_addr.rcvd_addr = cm_reply_addr.rcvd_addr;
817 statep->stored_reply_addr.grh_exists = cm_reply_addr.grh_exists;
818 statep->stored_reply_addr.port_num = cm_reply_addr.port_num;
819
820 /*
821 * The IPD on local/active side is calculated by path functions,
822 * hence available in the args of ibt_open_rc_channel
823 */
824 statep->local_srate = IBCM_PRIM_ADDS_VECT(chan_args).av_srate;
825 statep->local_alt_srate = IBCM_ALT_ADDS_VECT(chan_args).av_srate;
826
827 /* Store the source path bits for primary and alt paths */
828 statep->prim_src_path_bits = IBCM_PRIM_ADDS_VECT(chan_args).av_src_path;
829 statep->alt_src_path_bits = IBCM_ALT_ADDS_VECT(chan_args).av_src_path;
830
831 statep->open_flow = 1;
832 statep->open_done = B_FALSE;
833 statep->state = statep->timer_stored_state = IBCM_STATE_REQ_SENT;
834 IBCM_REF_CNT_INCR(statep); /* Decremented before return */
835 IBCM_REF_CNT_INCR(statep); /* Decremented after REQ is posted */
836 statep->send_mad_flags |= IBCM_REQ_POST_BUSY;
837
838 /*
839 * Skip moving channel to error state during close, for OFUV clients.
840 * OFUV clients transition the channel to error state by itself.
841 */
842 if (flags & IBT_OCHAN_OFUV)
843 statep->is_this_ofuv_chan = B_TRUE;
844
845 IBCM_OUT_HDRP(statep->stored_msg)->AttributeID =
846 h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID);
847
848 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID =
849 h2b64(ibcm_generate_tranid(IBCM_INCOMING_REQ, statep->local_comid,
850 0));
851
852 ibtl_cm_chan_is_opening(channel);
853
854 ibcm_open_enqueue(statep);
855
856 mutex_enter(&statep->state_mutex);
857
858 if (mode == IBT_BLOCKING) {
859
860 /* wait for REQ/REP/RTU */
861 while (statep->open_done != B_TRUE) {
862 cv_wait(&statep->block_client_cv, &statep->state_mutex);
863 }
864
865 /*
866 * In the case that open_channel() fails because of a
867 * REJ or timeout, change retval to IBT_CM_FAILURE
868 */
869 if (statep->open_return_data->rc_status != IBT_CM_SUCCESS) {
870 status = IBT_CM_FAILURE;
871 ibtl_cm_chan_open_is_aborted(channel);
872 }
873
874 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p "
875 "ret status %d cm status %d", channel, status,
876 statep->open_return_data->rc_status);
877 }
878
879 /* decrement the ref-count before leaving here */
880 IBCM_REF_CNT_DECR(statep);
881
882 mutex_exit(&statep->state_mutex);
883
884 IBTF_DPRINTF_L4(cmlog, "ibt_open_rc_channel: chan 0x%p done", channel);
885 return (status);
886 }
887
888 /*
889 * ibcm_init_reply_addr:
890 *
891 * The brief description of functionality below.
892 *
893 * For IBT_OCHAN_PORT_REDIRECTED (ie., port redirected case):
894 * Build CM path from chan_args->oc_cm_cep_path
895 * Set CM pkt lt (ie.,life time) to chan_args->oc_cm_pkt_lt
896 *
897 * For IBT_OCHAN_REDIRECTED (ie., port and CM redirected case):
898 * If Redirect LID is specified,
899 * If Redirect GID is not specified or specified to be on the same
900 * subnet, then
901 * Build CM path from chan_args->oc_cm_redirect_info
902 * Set CM pkt lt to subnet timeout
903 * Else (ie., GID specified, but on a different subnet)
904 * Do a path lookup to build CM Path and set CM pkt lt
905 *
906 */
907 static ibt_status_t
908 ibcm_init_reply_addr(ibcm_hca_info_t *hcap, ibcm_mad_addr_t *reply_addr,
909 ibt_chan_open_args_t *chan_args, ibt_chan_open_flags_t flags,
910 ib_time_t *cm_pkt_lt, ib_lid_t prim_slid)
911 {
912 ibt_adds_vect_t *cm_adds;
913 ibt_path_info_t path;
914 boolean_t cm_grh;
915 ibt_status_t status;
916
917 IBTF_DPRINTF_L5(cmlog, "ibcm_init_reply_addr:");
918
919 /*
920 * sending side CM lid/gid/port num are not based on any redirect
921 * params. These values are set to primary RC path lid/gid/port num.
922 * In the future, these values can be set based on framework policy
923 * decisions ensuring reachability.
924 */
925 reply_addr->grh_hdr.ig_sender_gid =
926 IBCM_PRIM_ADDS_VECT(chan_args).av_sgid;
927 reply_addr->rcvd_addr.ia_local_lid = prim_slid;
928 reply_addr->port_num = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num;
929
930 if (flags & IBT_OCHAN_PORT_REDIRECTED) {
931 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: "
932 "IBT_OCHAN_PORT_REDIRECTED specified");
933
934 status = ibt_index2pkey_byguid(hcap->hca_guid,
935 chan_args->oc_cm_cep_path->cep_hca_port_num,
936 chan_args->oc_cm_cep_path->cep_pkey_ix,
937 &reply_addr->rcvd_addr.ia_p_key);
938
939 if (status != IBT_SUCCESS) {
940 IBTF_DPRINTF_L2(cmlog, "ibcm_init_rely_addr: Invalid "
941 "CM PKeyIx %x port_num %x",
942 chan_args->oc_cm_cep_path->cep_pkey_ix,
943 chan_args->oc_cm_cep_path->cep_hca_port_num);
944 return (status);
945 }
946
947 cm_adds = &(chan_args->oc_cm_cep_path->cep_adds_vect);
948 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: dlid = %x",
949 cm_adds->av_dlid);
950
951 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY;
952 reply_addr->rcvd_addr.ia_remote_qno = 1;
953 *cm_pkt_lt = chan_args->oc_cm_pkt_lt;
954
955 } else if (flags & IBT_OCHAN_REDIRECTED) {
956 ibt_redirect_info_t *redirect_info;
957 ibt_hca_portinfo_t *port_infop;
958 uint_t psize, nports;
959
960 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: "
961 "IBT_OCHAN_REDIRECTED specified");
962
963 redirect_info = chan_args->oc_cm_redirect_info;
964
965 if ((redirect_info->rdi_gid.gid_prefix == 0) ||
966 (redirect_info->rdi_gid.gid_guid == 0)) {
967 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: "
968 "ERROR: Re-direct GID value NOT Provided.");
969 return (IBT_INVALID_PARAM);
970 }
971
972 /* As per spec definition 1.1, it's always IB_GSI_QKEY */
973 reply_addr->rcvd_addr.ia_q_key = redirect_info->rdi_qkey;
974 reply_addr->rcvd_addr.ia_remote_qno = redirect_info->rdi_qpn;
975 reply_addr->rcvd_addr.ia_p_key = redirect_info->rdi_pkey;
976
977 /*
978 * if LID is non-zero in classportinfo then use classportinfo
979 * fields to form CM MAD destination address.
980 */
981 if (redirect_info->rdi_dlid != 0) {
982 status = ibtl_cm_query_hca_ports_byguid(hcap->hca_guid,
983 reply_addr->port_num, &port_infop, &nports, &psize);
984 if ((status != IBT_SUCCESS) || (nports == 0)) {
985 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: "
986 "Query Ports Failed: %d", status);
987 return (status);
988 } else if (port_infop->p_subnet_timeout >
989 ibcm_max_ib_pkt_lt) {
990 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: "
991 "large subnet timeout %x port_no %x",
992 port_infop->p_subnet_timeout,
993 reply_addr->port_num);
994 ibt_free_portinfo(port_infop, psize);
995 return (IBT_PATH_PKT_LT_TOO_HIGH);
996 } else {
997 IBTF_DPRINTF_L3(cmlog, "ibcm_init_reply_addr: "
998 "subnet timeout %x port_no %x",
999 port_infop->p_subnet_timeout,
1000 reply_addr->port_num);
1001
1002 *cm_pkt_lt =
1003 ibt_ib2usec(min(ibcm_max_ib_mad_pkt_lt,
1004 port_infop->p_subnet_timeout));
1005
1006 ibt_free_portinfo(port_infop, psize);
1007 }
1008
1009 reply_addr->rcvd_addr.ia_remote_lid =
1010 redirect_info->rdi_dlid;
1011 reply_addr->rcvd_addr.ia_service_level =
1012 redirect_info->rdi_sl;
1013 reply_addr->grh_exists = B_TRUE;
1014 reply_addr->grh_hdr.ig_recver_gid =
1015 redirect_info->rdi_gid;
1016 reply_addr->grh_hdr.ig_tclass =
1017 redirect_info->rdi_tclass;
1018 reply_addr->grh_hdr.ig_flow_label =
1019 redirect_info->rdi_flow;
1020
1021 /* Classportinfo doesn't have hoplimit field */
1022 reply_addr->grh_hdr.ig_hop_limit = 1;
1023 return (IBT_SUCCESS);
1024
1025 } else {
1026 ibt_path_attr_t path_attr;
1027 ib_gid_t path_dgid[1];
1028
1029 /*
1030 * If GID is specified, and LID is zero in classportinfo
1031 * do a path lookup using specified GID, Pkey,
1032 * in classportinfo
1033 */
1034
1035 bzero(&path_attr, sizeof (path_attr));
1036
1037 path_attr.pa_dgids = &path_dgid[0];
1038 path_attr.pa_dgids[0] = redirect_info->rdi_gid;
1039
1040 /*
1041 * use reply_addr below, as sender_gid in reply_addr
1042 * may have been set above based on some policy decision
1043 * for originating end point for CM MADs above
1044 */
1045 path_attr.pa_sgid = reply_addr->grh_hdr.ig_sender_gid;
1046 path_attr.pa_num_dgids = 1;
1047 path_attr.pa_pkey = redirect_info->rdi_pkey;
1048
1049 if ((status = ibt_get_paths(ibcm_ibt_handle,
1050 IBT_PATH_PKEY, &path_attr, 1, &path, NULL)) !=
1051 IBT_SUCCESS)
1052 return (status);
1053
1054 /* Initialize cm_adds */
1055 cm_adds = &path.pi_prim_cep_path.cep_adds_vect;
1056 *cm_pkt_lt = path.pi_prim_pkt_lt;
1057 }
1058
1059 } else { /* cm_pkey initialized in ibt_open_rc_channel */
1060 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY;
1061 reply_addr->rcvd_addr.ia_remote_qno = 1;
1062 *cm_pkt_lt = chan_args->oc_path->pi_prim_pkt_lt;
1063 cm_adds = &(IBCM_PRIM_ADDS_VECT(chan_args));
1064 }
1065
1066
1067 cm_grh = cm_adds->av_send_grh;
1068 reply_addr->grh_exists = cm_grh;
1069
1070 reply_addr->rcvd_addr.ia_remote_lid =
1071 cm_adds->av_dlid;
1072 reply_addr->grh_hdr.ig_recver_gid =
1073 cm_adds->av_dgid;
1074 reply_addr->grh_hdr.ig_flow_label =
1075 cm_adds->av_flow & IB_GRH_FLOW_LABEL_MASK;
1076 reply_addr->grh_hdr.ig_tclass =
1077 (cm_grh == B_TRUE) ? cm_adds->av_tclass : 0;
1078 reply_addr->grh_hdr.ig_hop_limit =
1079 (cm_grh == B_TRUE) ? cm_adds->av_hop : 1;
1080 reply_addr->rcvd_addr.ia_service_level =
1081 cm_adds->av_srvl;
1082
1083 return (IBT_SUCCESS);
1084 }
1085
1086
1087 /*
1088 * ibt_prime_close_rc_channel()
1089 * It allocates resources required for close channel operation, so
1090 * ibt_close_rc_channel can be called from interrupt routine.
1091 *
1092 * INPUTS:
1093 * channel The address of an ibt_channel_t struct that
1094 * specifies the channel to open.
1095 *
1096 * RETURN VALUES:
1097 * IBT_SUCCESS on success(or respective failure on error)
1098 *
1099 * Clients are typically expected to call this function in established state
1100 */
1101 ibt_status_t
1102 ibt_prime_close_rc_channel(ibt_channel_hdl_t channel)
1103 {
1104 ibcm_state_data_t *statep;
1105 ibt_status_t status = IBT_SUCCESS;
1106
1107 IBTF_DPRINTF_L3(cmlog, "ibt_prime_close_rc_channel(%p)", channel);
1108
1109 /* validate channel, first */
1110 if (IBCM_INVALID_CHANNEL(channel)) {
1111 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p "
1112 "invalid channel", channel);
1113 return (IBT_CHAN_HDL_INVALID);
1114 }
1115
1116 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) {
1117 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p "
1118 "Invalid Channel type: Applicable only to RC Channel",
1119 channel);
1120 return (IBT_CHAN_SRV_TYPE_INVALID);
1121 }
1122
1123 /* get the statep */
1124 IBCM_GET_CHAN_PRIVATE(channel, statep);
1125
1126 /*
1127 * This can happen, if the statep is already gone by a DREQ from
1128 * the remote side
1129 */
1130
1131 if (statep == NULL) {
1132 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p "
1133 "statep NULL", channel);
1134 return (IBT_SUCCESS);
1135 }
1136
1137 mutex_enter(&statep->state_mutex);
1138 IBCM_RELEASE_CHAN_PRIVATE(channel);
1139 if (statep->state != IBCM_STATE_ESTABLISHED) {
1140 mutex_exit(&statep->state_mutex);
1141 return (IBT_CHAN_STATE_INVALID);
1142 }
1143 IBCM_REF_CNT_INCR(statep);
1144 IBTF_DPRINTF_L4(cmlog, "ibt_prime_close_rc_channel: chan 0x%p statep %p"
1145 " state %x", channel, statep, statep->state);
1146 mutex_exit(&statep->state_mutex);
1147
1148 /* clients could pre-allocate dreq mad, even before connection est */
1149 if (statep->dreq_msg == NULL)
1150 status = ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl,
1151 &statep->dreq_msg, MAD_METHOD_SEND);
1152
1153 mutex_enter(&statep->state_mutex);
1154 IBCM_REF_CNT_DECR(statep);
1155 mutex_exit(&statep->state_mutex);
1156
1157 if (status != IBT_SUCCESS) {
1158 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p "
1159 "ibcm_alloc_out_msg failed ", channel);
1160 return (status);
1161 }
1162
1163 /* If this message isn't seen then ibt_prime_close_rc_channel failed */
1164 IBTF_DPRINTF_L5(cmlog, "ibt_prime_close_rc_channel: chan 0x%p done",
1165 channel);
1166
1167 return (IBT_SUCCESS);
1168 }
1169
1170 /*
1171 * ibt_close_rc_channel()
1172 * It closes an established channel.
1173 *
1174 * RETURN VALUES:
1175 * IBT_SUCCESS on success(or respective failure on error)
1176 */
1177 ibt_status_t
1178 ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode,
1179 void *priv_data, ibt_priv_data_len_t priv_data_len, uint8_t *ret_status,
1180 void *ret_priv_data, ibt_priv_data_len_t *ret_priv_data_len_p)
1181 {
1182 ibcm_state_data_t *statep;
1183
1184 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel(%p, %x, %p, %d, %p)",
1185 channel, mode, priv_data, priv_data_len,
1186 (ret_priv_data_len_p == NULL) ? 0 : *ret_priv_data_len_p);
1187
1188 /* validate channel, first */
1189 if (IBCM_INVALID_CHANNEL(channel)) {
1190 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p "
1191 "invalid channel", channel);
1192 return (IBT_CHAN_HDL_INVALID);
1193 }
1194
1195 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) {
1196 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p "
1197 "Invalid Channel type: Applicable only to RC Channel",
1198 channel);
1199 return (IBT_CHAN_SRV_TYPE_INVALID);
1200 }
1201
1202 if (mode == IBT_BLOCKING) {
1203 /* valid only for BLOCKING MODE */
1204 if ((ret_priv_data_len_p != NULL) &&
1205 (*ret_priv_data_len_p > IBT_DREP_PRIV_DATA_SZ)) {
1206 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p"
1207 " private data len %d is too large", channel,
1208 *ret_priv_data_len_p);
1209 return (IBT_INVALID_PARAM);
1210 }
1211 } else if ((mode != IBT_NONBLOCKING) && (mode != IBT_NOCALLBACKS)) {
1212 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p "
1213 "invalid mode %x specified", channel, mode);
1214 return (IBT_INVALID_PARAM);
1215 }
1216
1217 if (ibtl_cm_is_chan_closing(channel) ||
1218 ibtl_cm_is_chan_closed(channel)) {
1219 if (ret_status)
1220 *ret_status = IBT_CM_CLOSED_ALREADY;
1221
1222 /* No private data to return to the client */
1223 if (ret_priv_data_len_p != NULL)
1224 *ret_priv_data_len_p = 0;
1225
1226 if ((mode == IBT_BLOCKING) ||
1227 (mode == IBT_NOCALLBACKS)) {
1228 IBCM_GET_CHAN_PRIVATE(channel, statep);
1229 if (statep == NULL)
1230 return (IBT_SUCCESS);
1231 mutex_enter(&statep->state_mutex);
1232 IBCM_RELEASE_CHAN_PRIVATE(channel);
1233 IBCM_REF_CNT_INCR(statep);
1234 while (statep->close_done != B_TRUE)
1235 cv_wait(&statep->block_client_cv,
1236 &statep->state_mutex);
1237 IBCM_REF_CNT_DECR(statep);
1238 mutex_exit(&statep->state_mutex);
1239 }
1240
1241 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: chan 0x%p "
1242 "already marked for closing", channel);
1243
1244 return (IBT_SUCCESS);
1245 }
1246
1247 /* get the statep */
1248 IBCM_GET_CHAN_PRIVATE(channel, statep);
1249 if (statep == NULL) {
1250 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p "
1251 "statep NULL", channel);
1252 return (IBT_CHAN_STATE_INVALID);
1253 }
1254
1255 mutex_enter(&statep->state_mutex);
1256
1257 if (statep->dreq_msg == NULL) {
1258 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p "
1259 "Fatal Error: dreq_msg is NULL", channel);
1260 IBCM_RELEASE_CHAN_PRIVATE(channel);
1261 mutex_exit(&statep->state_mutex);
1262 return (IBT_CHAN_STATE_INVALID);
1263 }
1264
1265 if ((ret_priv_data == NULL) || (ret_priv_data_len_p == NULL)) {
1266 statep->close_ret_priv_data = NULL;
1267 statep->close_ret_priv_data_len = NULL;
1268 } else {
1269 statep->close_ret_priv_data = ret_priv_data;
1270 statep->close_ret_priv_data_len = ret_priv_data_len_p;
1271 }
1272
1273 priv_data_len = min(priv_data_len, IBT_DREQ_PRIV_DATA_SZ);
1274 if ((priv_data != NULL) && (priv_data_len > 0)) {
1275 bcopy(priv_data, ((ibcm_dreq_msg_t *)
1276 IBCM_OUT_MSGP(statep->dreq_msg))->dreq_private_data,
1277 priv_data_len);
1278 }
1279 statep->close_ret_status = ret_status;
1280
1281 IBCM_RELEASE_CHAN_PRIVATE(channel);
1282 IBCM_REF_CNT_INCR(statep);
1283
1284 if (mode != IBT_NONBLOCKING) {
1285 return (ibcm_close_rc_channel(channel, statep, mode));
1286 }
1287
1288 /* IBT_NONBLOCKING */
1289 ibcm_close_enqueue(statep);
1290 mutex_exit(&statep->state_mutex);
1291
1292 return (IBT_SUCCESS);
1293 }
1294
1295 void
1296 ibcm_close_start(ibcm_state_data_t *statep)
1297 {
1298 mutex_enter(&statep->state_mutex);
1299 (void) ibcm_close_rc_channel(statep->channel, statep, IBT_NONBLOCKING);
1300 }
1301
1302 static
1303 ibt_status_t
1304 ibcm_close_rc_channel(ibt_channel_hdl_t channel, ibcm_state_data_t *statep,
1305 ibt_execution_mode_t mode)
1306 {
1307 ibcm_hca_info_t *hcap;
1308
1309 ASSERT(MUTEX_HELD(&statep->state_mutex));
1310
1311 IBTF_DPRINTF_L3(cmlog, "ibcm_close_rc_channel: chan 0x%p statep %p",
1312 channel, statep);
1313
1314 hcap = statep->hcap;
1315
1316 /* HCA must have been in active state. If not, it's a client bug */
1317 if (!IBCM_ACCESS_HCA_OK(hcap)) {
1318 IBTF_DPRINTF_L2(cmlog, "ibcm_close_rc_channel: chan 0x%p "
1319 "hcap 0x%p not active", channel, hcap);
1320 IBCM_REF_CNT_DECR(statep);
1321 mutex_exit(&statep->state_mutex);
1322 return (IBT_CHAN_HDL_INVALID);
1323 }
1324
1325 if (statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) {
1326 while (statep->cep_in_rts == IBCM_BLOCK)
1327 cv_wait(&statep->block_mad_cv, &statep->state_mutex);
1328 }
1329
1330 /* Do TRANSIENT_DREQ check after TRANSIENT_ESTABLISHED check */
1331 while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT)
1332 cv_wait(&statep->block_mad_cv, &statep->state_mutex);
1333
1334 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: chan 0x%p "
1335 "connection state is %x", channel, statep->state);
1336
1337 /* If state is in pre-established states, abort the connection est */
1338 if (statep->state != IBCM_STATE_ESTABLISHED) {
1339 statep->cm_retries++; /* ensure connection trace is dumped */
1340
1341 /* No DREP private data possible */
1342 if (statep->close_ret_priv_data_len != NULL)
1343 *statep->close_ret_priv_data_len = 0;
1344
1345 /*
1346 * If waiting for a response mad, then cancel the timer,
1347 * and delete the connection
1348 */
1349 if (statep->state == IBCM_STATE_REQ_SENT ||
1350 statep->state == IBCM_STATE_REP_SENT ||
1351 statep->state == IBCM_STATE_REP_WAIT ||
1352 statep->state == IBCM_STATE_MRA_REP_RCVD) {
1353 timeout_id_t timer_val = statep->timerid;
1354 ibcm_conn_state_t old_state;
1355
1356 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: "
1357 "chan 0x%p connection aborted in state %x", channel,
1358 statep->state);
1359
1360 old_state = statep->state;
1361 statep->state = IBCM_STATE_DELETE;
1362
1363 if (mode == IBT_NONBLOCKING) {
1364 if (taskq_dispatch(ibcm_taskq,
1365 ibcm_process_abort_via_taskq, statep,
1366 TQ_NOSLEEP) == 0) {
1367
1368 IBCM_REF_CNT_DECR(statep);
1369 statep->state = old_state;
1370 mutex_exit(&statep->state_mutex);
1371 return (IBT_INSUFF_KERNEL_RESOURCE);
1372 } /* if taskq_dispatch succeeds */
1373 /* Cancel the timer */
1374 statep->timerid = 0;
1375 mutex_exit(&statep->state_mutex);
1376 } else {
1377 /* Cancel the timer */
1378 statep->timerid = 0;
1379 mutex_exit(&statep->state_mutex);
1380 (void) taskq_dispatch(ibcm_taskq,
1381 ibcm_process_abort_via_taskq, statep,
1382 TQ_SLEEP);
1383 }
1384
1385 /* cancel the currently running timer */
1386 if (timer_val != 0)
1387 (void) untimeout(timer_val);
1388
1389 /* wait until cm handler returns for BLOCKING cases */
1390 mutex_enter(&statep->state_mutex);
1391 if ((mode == IBT_BLOCKING) ||
1392 (mode == IBT_NOCALLBACKS)) {
1393 while (statep->close_done != B_TRUE)
1394 cv_wait(&statep->block_client_cv,
1395 &statep->state_mutex);
1396 }
1397
1398 if (statep->close_ret_status)
1399 *statep->close_ret_status = IBT_CM_CLOSED_ABORT;
1400 mutex_exit(&statep->state_mutex);
1401
1402 /*
1403 * It would ideal to post a REJ MAD, but that would
1404 * be non-conformance to spec. Hence, delete the state
1405 * data. Assuming that happens quickly, any retransmits
1406 * from the remote are replied by CM with reject
1407 * reason " no valid com id". That would stop remote
1408 * sending any more MADs.
1409 */
1410 ibcm_delete_state_data(statep);
1411 return (IBT_SUCCESS);
1412
1413 /* if CM busy in cm handler, wait until cm handler returns */
1414 } else if (statep->state == IBCM_STATE_REQ_RCVD ||
1415 statep->state == IBCM_STATE_REP_RCVD ||
1416 statep->state == IBCM_STATE_MRA_SENT ||
1417 statep->state == IBCM_STATE_MRA_REP_SENT) {
1418
1419 /* take control of statep */
1420 statep->abort_flag |= IBCM_ABORT_CLIENT;
1421
1422 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: "
1423 "chan 0x%p connection aborted in state = %x",
1424 channel, statep->state);
1425
1426 /*
1427 * wait until state machine modifies qp state to error,
1428 * including disassociating statep and QP
1429 */
1430 if ((mode == IBT_BLOCKING) || (mode == IBT_NOCALLBACKS))
1431 while (statep->close_done != B_TRUE)
1432 cv_wait(&statep->block_client_cv,
1433 &statep->state_mutex);
1434
1435 /* a sanity setting */
1436 if (mode == IBT_NOCALLBACKS)
1437 statep->cm_handler = NULL;
1438 IBCM_REF_CNT_DECR(statep);
1439
1440 /*
1441 * In rare situations, connection attempt could be
1442 * terminated for some other reason, before abort is
1443 * processed, but CM still returns ret_status as abort
1444 */
1445 if (statep->close_ret_status)
1446 *statep->close_ret_status = IBT_CM_CLOSED_ABORT;
1447 mutex_exit(&statep->state_mutex);
1448
1449 /*
1450 * REJ MAD is posted by the CM state machine for this
1451 * case, hence state structure is deleted in the
1452 * state machine processing.
1453 */
1454 return (IBT_SUCCESS);
1455
1456 } else if ((statep->state == IBCM_STATE_TIMEWAIT) ||
1457 (statep->state == IBCM_STATE_DELETE)) {
1458
1459 /* State already in timewait, so no return priv data */
1460 IBCM_REF_CNT_DECR(statep);
1461
1462 /* The teardown has already been done */
1463 if (statep->close_ret_status)
1464 *statep->close_ret_status =
1465 IBT_CM_CLOSED_ALREADY;
1466 mutex_exit(&statep->state_mutex);
1467
1468 return (IBT_SUCCESS);
1469
1470 } else if ((statep->state == IBCM_STATE_DREQ_RCVD) ||
1471 (statep->state == IBCM_STATE_DREQ_SENT) ||
1472 (statep->state == IBCM_STATE_DREP_RCVD) ||
1473 ((statep->state == IBCM_STATE_TIMED_OUT) &&
1474 (statep->timedout_state == IBCM_STATE_DREQ_SENT))) {
1475
1476 /*
1477 * Either the remote or local client has already
1478 * initiated the teardown. IBCM_STATE_DREP_RCVD is
1479 * possible, if CM initiated teardown without client's
1480 * knowledge, for stale handling, etc.,
1481 */
1482 if (mode == IBT_NOCALLBACKS) {
1483 if (statep->close_nocb_state == IBCM_UNBLOCK) {
1484 statep->close_nocb_state = IBCM_FAIL;
1485 /* enable free qp after return */
1486 ibtl_cm_chan_is_closing(
1487 statep->channel);
1488 } else while (statep->close_nocb_state ==
1489 IBCM_BLOCK)
1490 cv_wait(&statep->block_client_cv,
1491 &statep->state_mutex);
1492 statep->cm_handler = NULL; /* sanity setting */
1493 if (statep->close_ret_status)
1494 *statep->close_ret_status =
1495 IBT_CM_CLOSED_ALREADY;
1496 } else if (mode == IBT_BLOCKING) {
1497 /* wait until state is moved to timewait */
1498 while (statep->close_done != B_TRUE)
1499 cv_wait(&statep->block_client_cv,
1500 &statep->state_mutex);
1501 }
1502
1503 IBCM_REF_CNT_DECR(statep);
1504 mutex_exit(&statep->state_mutex);
1505
1506 /* ret_status is set in state machine code */
1507 return (IBT_SUCCESS);
1508
1509 } else if (statep->state == IBCM_STATE_TIMED_OUT) {
1510
1511 if ((mode == IBT_BLOCKING) ||
1512 (mode == IBT_NOCALLBACKS)) {
1513
1514 /*
1515 * wait until cm handler invocation and
1516 * disassociation between statep and channel
1517 * is complete
1518 */
1519 while (statep->close_done != B_TRUE)
1520 cv_wait(&statep->block_client_cv,
1521 &statep->state_mutex);
1522 }
1523
1524 if (statep->close_ret_status)
1525 *statep->close_ret_status = IBT_CM_CLOSED_ABORT;
1526 IBCM_REF_CNT_DECR(statep);
1527 mutex_exit(&statep->state_mutex);
1528
1529 return (IBT_SUCCESS);
1530 } else {
1531 IBCM_REF_CNT_DECR(statep);
1532 mutex_exit(&statep->state_mutex);
1533
1534 return (IBT_CM_FAILURE);
1535 }
1536 }
1537
1538 ASSERT(statep->close_nocb_state != IBCM_BLOCK);
1539
1540 if (mode == IBT_NOCALLBACKS) {
1541 statep->close_nocb_state = IBCM_FAIL;
1542 statep->cm_handler = NULL;
1543 ibtl_cm_chan_is_closing(statep->channel);
1544 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: "
1545 "NOCALLBACKS on in statep = %p", statep);
1546 }
1547
1548 if (statep->state != IBCM_STATE_ESTABLISHED) {
1549 goto lost_race;
1550 }
1551
1552 /*
1553 * Cancel/wait for any pending ibt_set_alt_path, and
1554 * release state mutex
1555 */
1556 ibcm_sync_lapr_idle(statep);
1557
1558 ibcm_close_enter();
1559
1560 mutex_enter(&statep->state_mutex);
1561 if (statep->state != IBCM_STATE_ESTABLISHED) {
1562 ibcm_close_exit();
1563 goto lost_race;
1564 }
1565
1566 statep->state = IBCM_STATE_TRANSIENT_DREQ_SENT;
1567 statep->timerid = 0;
1568 statep->close_done = B_FALSE;
1569 statep->close_flow = 1;
1570 mutex_exit(&statep->state_mutex);
1571
1572 ibcm_post_dreq_mad(statep);
1573
1574 mutex_enter(&statep->state_mutex);
1575
1576 lost_race:
1577 if (mode == IBT_BLOCKING) {
1578
1579 /* wait for DREP */
1580 while (statep->close_done != B_TRUE)
1581 cv_wait(&statep->block_client_cv,
1582 &statep->state_mutex);
1583
1584 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: chan 0x%p "
1585 "done blocking", channel);
1586 }
1587
1588 IBCM_REF_CNT_DECR(statep);
1589 mutex_exit(&statep->state_mutex);
1590
1591 /* If this message isn't seen then ibt_close_rc_channel failed */
1592 IBTF_DPRINTF_L5(cmlog, "ibcm_close_rc_channel: chan 0x%p done",
1593 channel);
1594
1595 return (IBT_SUCCESS);
1596 }
1597
1598 ibt_status_t
1599 ibt_recycle_rc(ibt_channel_hdl_t rc_chan, ibt_cep_flags_t control,
1600 uint8_t hca_port_num, ibt_recycle_handler_t func, void *arg)
1601 {
1602 ibcm_state_data_t *statep;
1603 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg;
1604 ibt_qp_query_attr_t qp_attr;
1605 ibt_status_t retval;
1606
1607 IBTF_DPRINTF_L3(cmlog, "ibt_recycle_rc (%p, 0x%X, %d, %p, %p)", rc_chan,
1608 control, hca_port_num, func, arg);
1609
1610 if (IBCM_INVALID_CHANNEL(rc_chan)) {
1611 IBTF_DPRINTF_L2(cmlog, "ibt_recycle_rc: invalid channel");
1612 return (IBT_CHAN_HDL_INVALID);
1613 }
1614
1615 /* check qp state */
1616 retval = ibt_query_qp(rc_chan, &qp_attr);
1617
1618 if (retval != IBT_SUCCESS)
1619 return (retval);
1620
1621 if (qp_attr.qp_info.qp_trans != IBT_RC_SRV)
1622 return (IBT_CHAN_SRV_TYPE_INVALID);
1623
1624 if (qp_attr.qp_info.qp_state != IBT_STATE_ERROR)
1625 return (IBT_CHAN_STATE_INVALID);
1626
1627 ibcm_tq_recycle_arg = kmem_alloc(sizeof (ibcm_taskq_recycle_arg_t),
1628 KM_SLEEP);
1629
1630 ibcm_tq_recycle_arg->rc_chan = rc_chan;
1631 ibcm_tq_recycle_arg->control = control;
1632 ibcm_tq_recycle_arg->hca_port_num = hca_port_num;
1633 ibcm_tq_recycle_arg->func = func;
1634 ibcm_tq_recycle_arg->arg = arg;
1635
1636 IBCM_GET_CHAN_PRIVATE(rc_chan, statep);
1637
1638 /*
1639 * If non-blocking ie., func specified and channel has not yet completed
1640 * the timewait, then schedule the work for later
1641 */
1642 if ((func != NULL) && (statep != NULL)) {
1643 IBCM_RELEASE_CHAN_PRIVATE(rc_chan);
1644 statep->recycle_arg = ibcm_tq_recycle_arg;
1645 return (IBT_SUCCESS);
1646 }
1647
1648 /*
1649 * if blocking ie., func specified, and channel has not yet completed
1650 * the timewait, then block until the channel completes the timewait
1651 */
1652 if (statep != NULL)
1653 IBCM_RELEASE_CHAN_PRIVATE(rc_chan);
1654 IBCM_WAIT_CHAN_PRIVATE(rc_chan);
1655
1656 if (func) { /* NON BLOCKING case. Taskq for QP state change */
1657 (void) taskq_dispatch(ibcm_taskq, ibcm_process_rc_recycle,
1658 ibcm_tq_recycle_arg, TQ_SLEEP);
1659 return (IBT_SUCCESS);
1660 } else /* BLOCKING case */
1661 return (ibcm_process_rc_recycle_ret(ibcm_tq_recycle_arg));
1662 }
1663
1664 void
1665 ibcm_process_rc_recycle(void *recycle_arg)
1666 {
1667 (void) ibcm_process_rc_recycle_ret(recycle_arg);
1668 }
1669
1670 static ibt_status_t
1671 ibcm_process_rc_recycle_ret(void *recycle_arg)
1672 {
1673 ibt_qp_info_t qp_info;
1674 ibt_status_t ibt_status = IBT_SUCCESS;
1675 ibt_cep_modify_flags_t cep_flags;
1676 ibt_qp_query_attr_t qp_attr;
1677 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg =
1678 (ibcm_taskq_recycle_arg_t *)recycle_arg;
1679
1680 /* QP must have been in error state */
1681 ibt_status = ibt_query_qp(ibcm_tq_recycle_arg->rc_chan, &qp_attr);
1682 if (ibt_status != IBT_SUCCESS)
1683 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: "
1684 "chanp %p ibt_query_qp() = %d",
1685 ibcm_tq_recycle_arg->rc_chan, ibt_status);
1686 else {
1687 /* perform the QP state change from ERROR to RESET */
1688 bzero(&qp_info, sizeof (qp_info));
1689
1690 qp_info.qp_trans = IBT_RC_SRV;
1691 qp_info.qp_state = IBT_STATE_RESET;
1692
1693 /* Call modify_qp to move to RESET state */
1694 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan,
1695 IBT_CEP_SET_STATE, &qp_info, NULL);
1696
1697 if (ibt_status != IBT_SUCCESS)
1698 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: "
1699 "chanp %p ibt_modify_qp() = %d for ERROR to RESET",
1700 ibcm_tq_recycle_arg->rc_chan, ibt_status);
1701 }
1702
1703 if (ibt_status == IBT_SUCCESS) {
1704
1705 qp_info.qp_state = IBT_STATE_INIT;
1706
1707 /* set flags for all mandatory args from RESET to INIT */
1708 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT;
1709 cep_flags |= IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W;
1710 cep_flags |= IBT_CEP_SET_ATOMIC;
1711
1712 qp_info.qp_transport.rc.rc_path.cep_hca_port_num =
1713 ibcm_tq_recycle_arg->hca_port_num;
1714 qp_info.qp_flags |=
1715 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_RD;
1716 qp_info.qp_flags |=
1717 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_WR;
1718 qp_info.qp_flags |=
1719 ibcm_tq_recycle_arg->control & IBT_CEP_ATOMIC;
1720
1721 /* Always use the existing pkey */
1722 qp_info.qp_transport.rc.rc_path.cep_pkey_ix =
1723 qp_attr. qp_info.qp_transport.rc.rc_path.cep_pkey_ix;
1724
1725 /* Call modify_qp to move to INIT state */
1726 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan,
1727 cep_flags, &qp_info, NULL);
1728
1729 if (ibt_status != IBT_SUCCESS)
1730 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: "
1731 "chanp %p ibt_modify_qp() = %d for RESET to INIT",
1732 ibcm_tq_recycle_arg->rc_chan, ibt_status);
1733 }
1734
1735 /* Change the QP CM state to indicate QP being re-used */
1736 if (ibt_status == IBT_SUCCESS)
1737 ibtl_cm_chan_is_reused(ibcm_tq_recycle_arg->rc_chan);
1738
1739 /* Call func, if defined */
1740 if (ibcm_tq_recycle_arg->func)
1741 (*(ibcm_tq_recycle_arg->func))(ibt_status,
1742 ibcm_tq_recycle_arg->arg);
1743
1744 kmem_free(ibcm_tq_recycle_arg, sizeof (ibcm_taskq_recycle_arg_t));
1745
1746 return (ibt_status);
1747 }
1748
1749 static void
1750 ibcm_process_abort_via_taskq(void *args)
1751 {
1752 ibcm_state_data_t *statep = (ibcm_state_data_t *)args;
1753
1754 ibcm_process_abort(statep);
1755 mutex_enter(&statep->state_mutex);
1756 IBCM_REF_CNT_DECR(statep);
1757 mutex_exit(&statep->state_mutex);
1758 }
1759
1760 /*
1761 * Local UD CM Handler's private data, used during ibt_request_ud_dest() in
1762 * Non-Blocking mode operations.
1763 */
1764 typedef struct ibcm_local_handler_s {
1765 ibt_cm_ud_handler_t actual_cm_handler;
1766 void *actual_cm_private;
1767 ibt_ud_dest_t *dest_hdl;
1768 } ibcm_local_handler_t;
1769
1770 /*
1771 * Local UD CM Handler, used when ibt_alloc_ud_dest() is issued in
1772 * NON-Blocking mode.
1773 *
1774 * Out here, we update the UD Destination handle with
1775 * the obtained DQPN and QKey (from SIDR REP) and invokes actual client
1776 * handler that was specified by the client.
1777 */
1778 static ibt_cm_status_t
1779 ibcm_local_cm_handler(void *priv, ibt_cm_ud_event_t *event,
1780 ibt_cm_ud_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
1781 {
1782 ibcm_local_handler_t *handler_priv = (ibcm_local_handler_t *)priv;
1783
1784 IBTF_DPRINTF_L4(cmlog, "ibcm_local_cm_handler: event %d",
1785 event->cm_type);
1786
1787 ASSERT(handler_priv != NULL);
1788
1789 switch (event->cm_type) {
1790 case IBT_CM_UD_EVENT_SIDR_REP:
1791 /* Update QPN & QKey from event into destination handle. */
1792 if (handler_priv->dest_hdl != NULL) {
1793 handler_priv->dest_hdl->ud_dst_qpn =
1794 event->cm_event.sidr_rep.srep_remote_qpn;
1795 handler_priv->dest_hdl->ud_qkey =
1796 event->cm_event.sidr_rep.srep_remote_qkey;
1797 }
1798
1799 /* Invoke the client handler - inform only, so ignore retval */
1800 (void) handler_priv->actual_cm_handler(
1801 handler_priv->actual_cm_private, event, ret_args, priv_data,
1802 len);
1803
1804 /* Free memory allocated for local handler's private data. */
1805 if (handler_priv != NULL)
1806 kmem_free(handler_priv, sizeof (*handler_priv));
1807
1808 break;
1809 default:
1810 IBTF_DPRINTF_L2(cmlog, "ibcm_local_cm_handler: ERROR");
1811 break;
1812 }
1813
1814 return (IBT_CM_ACCEPT);
1815 }
1816
1817
1818 /* Validate the input UD destination attributes. */
1819 static ibt_status_t
1820 ibcm_validate_dqpn_data(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode,
1821 ibt_ud_returns_t *ret_args)
1822 {
1823 /* cm handler must always be specified */
1824 if (mode == IBT_NONBLOCKING && attr->ud_cm_handler == NULL) {
1825 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: "
1826 "CM handler is not specified ");
1827 return (IBT_INVALID_PARAM);
1828 }
1829
1830 if (mode == IBT_NONBLOCKING) {
1831 if (ret_args != NULL) {
1832 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: "
1833 "ret_args should be NULL when called in "
1834 "non-blocking mode");
1835 return (IBT_INVALID_PARAM);
1836 }
1837 } else if (mode == IBT_BLOCKING) {
1838 if (ret_args == NULL) {
1839 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: "
1840 "ret_args should be Non-NULL when called in "
1841 "blocking mode");
1842 return (IBT_INVALID_PARAM);
1843 }
1844 } else {
1845 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: "
1846 "invalid mode %x specified ", mode);
1847 return (IBT_INVALID_PARAM);
1848 }
1849
1850 if (attr->ud_sid == 0) {
1851 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: "
1852 "ServiceID must be specified. ");
1853 return (IBT_INVALID_PARAM);
1854 }
1855
1856 if (attr->ud_addr == NULL) {
1857 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: "
1858 "Address Info NULL");
1859 return (IBT_INVALID_PARAM);
1860 }
1861
1862 /* Validate SGID */
1863 if ((attr->ud_addr->av_sgid.gid_prefix == 0) ||
1864 (attr->ud_addr->av_sgid.gid_guid == 0)) {
1865 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid SGID");
1866 return (IBT_INVALID_PARAM);
1867 }
1868 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: SGID<%llX:%llX>",
1869 attr->ud_addr->av_sgid.gid_prefix,
1870 attr->ud_addr->av_sgid.gid_guid);
1871
1872 /* Validate DGID */
1873 if ((attr->ud_addr->av_dgid.gid_prefix == 0) ||
1874 (attr->ud_addr->av_dgid.gid_guid == 0)) {
1875 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid DGID");
1876 return (IBT_INVALID_PARAM);
1877 }
1878 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: DGID<%llX:%llX>",
1879 attr->ud_addr->av_dgid.gid_prefix,
1880 attr->ud_addr->av_dgid.gid_guid);
1881
1882 return (IBT_SUCCESS);
1883 }
1884
1885
1886 /* Perform SIDR to retrieve DQPN and QKey. */
1887 static ibt_status_t
1888 ibcm_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode,
1889 ibt_ud_returns_t *ret_args)
1890 {
1891 ibt_status_t retval;
1892 ib_pkey_t ud_pkey;
1893 ibmf_handle_t ibmf_hdl;
1894 ibmf_msg_t *ibmf_msg;
1895 ibcm_hca_info_t *hcap;
1896 ibcm_sidr_req_msg_t *sidr_req_msgp;
1897 ibcm_ud_state_data_t *ud_statep;
1898 ibtl_cm_hca_port_t port;
1899 ibcm_sidr_srch_t sidr_entry;
1900 ibcm_qp_list_t *cm_qp_entry;
1901
1902 /* Retrieve HCA GUID value from the available SGID info. */
1903 retval = ibtl_cm_get_hca_port(attr->ud_addr->av_sgid, 0, &port);
1904 if ((retval != IBT_SUCCESS) || (port.hp_port == 0)) {
1905 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: "
1906 "ibtl_cm_get_hca_port failed: %d", retval);
1907 return (retval);
1908 }
1909
1910 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: "
1911 "HCA GUID:%llX, port_num:%d", port.hp_hca_guid, port.hp_port);
1912
1913 /* Lookup the HCA info for this GUID */
1914 if ((hcap = ibcm_find_hca_entry(port.hp_hca_guid)) == NULL) {
1915 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: hcap is NULL");
1916 return (IBT_HCA_INVALID);
1917 }
1918
1919 /* Return failure if the HCA device or Port is not operational */
1920
1921 if ((retval = ibt_get_port_state_byguid(port.hp_hca_guid, port.hp_port,
1922 NULL, NULL)) != IBT_SUCCESS) {
1923 /* Device Port is not in good state, don't use it. */
1924 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: Invalid "
1925 "port specified or port not active");
1926 ibcm_dec_hca_acc_cnt(hcap);
1927 return (retval);
1928 }
1929
1930 retval = ibt_index2pkey_byguid(port.hp_hca_guid, port.hp_port,
1931 attr->ud_pkey_ix, &ud_pkey);
1932 if (retval != IBT_SUCCESS) {
1933 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: "
1934 "Failed to convert index2pkey: %d", retval);
1935 ibcm_dec_hca_acc_cnt(hcap);
1936 return (retval);
1937 }
1938
1939 /* Allocate a new request id */
1940 if (ibcm_alloc_reqid(hcap, &sidr_entry.srch_req_id) == IBCM_FAILURE) {
1941 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: "
1942 "no req id available");
1943 ibcm_dec_hca_acc_cnt(hcap);
1944 return (IBT_INSUFF_KERNEL_RESOURCE);
1945 }
1946
1947 if ((hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl == NULL) &&
1948 ((retval = ibcm_hca_reinit_port(hcap, port.hp_port - 1))
1949 != IBT_SUCCESS)) {
1950 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: "
1951 "ibmf reg or callback setup failed during re-initialize");
1952 return (retval);
1953 }
1954
1955 ibmf_hdl = hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl;
1956
1957 /* find the ibmf QP to post the SIDR REQ */
1958 if ((cm_qp_entry = ibcm_find_qp(hcap, port.hp_port, ud_pkey)) ==
1959 NULL) {
1960 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF QP allocation"
1961 " failed");
1962 ibcm_dec_hca_acc_cnt(hcap);
1963 return (IBT_INSUFF_RESOURCE);
1964 }
1965
1966 if ((retval = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, MAD_METHOD_SEND))
1967 != IBT_SUCCESS) {
1968 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF MSG allocation"
1969 " failed");
1970 ibcm_release_qp(cm_qp_entry);
1971 ibcm_dec_hca_acc_cnt(hcap);
1972 return (retval);
1973 }
1974
1975 sidr_entry.srch_lid = port.hp_base_lid;
1976 sidr_entry.srch_gid = attr->ud_addr->av_sgid;
1977 sidr_entry.srch_grh_exists = attr->ud_addr->av_send_grh;
1978 sidr_entry.srch_mode = IBCM_ACTIVE_MODE;
1979
1980 /* do various allocations needed here */
1981 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER);
1982
1983 (void) ibcm_find_sidr_entry(&sidr_entry, hcap, &ud_statep,
1984 IBCM_FLAG_ADD);
1985 rw_exit(&hcap->hca_sidr_list_lock);
1986
1987 /* Increment hca's resource count */
1988 ibcm_inc_hca_res_cnt(hcap);
1989
1990 /* After a resource created on hca, no need to hold the acc cnt */
1991 ibcm_dec_hca_acc_cnt(hcap);
1992
1993 /* Initialize some ud_statep fields */
1994 ud_statep->ud_stored_msg = ibmf_msg;
1995 ud_statep->ud_svc_id = attr->ud_sid;
1996 ud_statep->ud_pkt_life_time =
1997 ibt_ib2usec(attr->ud_pkt_lt);
1998 ud_statep->ud_stored_reply_addr.cm_qp_entry = cm_qp_entry;
1999
2000 /* set remaining retry cnt */
2001 ud_statep->ud_remaining_retry_cnt = ud_statep->ud_max_cm_retries;
2002
2003 /*
2004 * Get UD handler and corresponding args which is pass it back
2005 * as first argument for the handler.
2006 */
2007 ud_statep->ud_state_cm_private = attr->ud_cm_private;
2008
2009 if (mode == IBT_BLOCKING)
2010 ud_statep->ud_return_data = ret_args;
2011 else
2012 ud_statep->ud_cm_handler = attr->ud_cm_handler;
2013
2014 /* Initialize the fields of ud_statep->ud_stored_reply_addr */
2015 ud_statep->ud_stored_reply_addr.grh_exists = attr->ud_addr->av_send_grh;
2016 ud_statep->ud_stored_reply_addr.ibmf_hdl = ibmf_hdl;
2017 ud_statep->ud_stored_reply_addr.grh_hdr.ig_hop_limit =
2018 attr->ud_addr->av_hop;
2019 ud_statep->ud_stored_reply_addr.grh_hdr.ig_sender_gid =
2020 attr->ud_addr->av_sgid;
2021 ud_statep->ud_stored_reply_addr.grh_hdr.ig_recver_gid =
2022 attr->ud_addr->av_dgid;
2023 ud_statep->ud_stored_reply_addr.grh_hdr.ig_tclass =
2024 attr->ud_addr->av_tclass;
2025 ud_statep->ud_stored_reply_addr.grh_hdr.ig_flow_label =
2026 attr->ud_addr->av_flow & IB_GRH_FLOW_LABEL_MASK;
2027
2028 /* needs to be derived based on the base LID and path bits */
2029 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_local_lid =
2030 port.hp_base_lid;
2031 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_lid =
2032 attr->ud_addr->av_dlid;
2033 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_p_key = ud_pkey;
2034 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_q_key = IB_GSI_QKEY;
2035 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_service_level =
2036 attr->ud_addr->av_srvl;
2037
2038 /*
2039 * This may be enchanced later, to use a remote qno based on past
2040 * redirect rej mad responses. This would be the place to specify
2041 * appropriate remote qno
2042 */
2043 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_qno = 1;
2044
2045 /* Initialize the SIDR REQ message fields */
2046 sidr_req_msgp =
2047 (ibcm_sidr_req_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg);
2048
2049 sidr_req_msgp->sidr_req_request_id = h2b32(ud_statep->ud_req_id);
2050 sidr_req_msgp->sidr_req_service_id = h2b64(attr->ud_sid);
2051 sidr_req_msgp->sidr_req_pkey = h2b16(ud_pkey);
2052 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->AttributeID =
2053 h2b16(IBCM_INCOMING_SIDR_REQ + IBCM_ATTR_BASE_ID);
2054
2055 if ((attr->ud_priv_data != NULL) && (attr->ud_priv_data_len > 0)) {
2056 bcopy(attr->ud_priv_data, sidr_req_msgp->sidr_req_private_data,
2057 min(attr->ud_priv_data_len, IBT_SIDR_REQ_PRIV_DATA_SZ));
2058 }
2059
2060 /* Send out the SIDR REQ message */
2061 ud_statep->ud_state = IBCM_STATE_SIDR_REQ_SENT;
2062 ud_statep->ud_timer_stored_state = IBCM_STATE_SIDR_REQ_SENT;
2063 IBCM_UD_REF_CNT_INCR(ud_statep); /* for non-blocking SIDR REQ post */
2064 ud_statep->ud_timer_value = ibt_ib2usec(ibcm_max_sidr_rep_proctime) +
2065 (ud_statep->ud_pkt_life_time * 2);
2066
2067 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->TransactionID =
2068 h2b64(ibcm_generate_tranid(IBCM_INCOMING_SIDR_REQ,
2069 ud_statep->ud_req_id, 0));
2070
2071 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: timer_value in HZ = %x",
2072 ud_statep->ud_timer_value);
2073
2074 ibcm_post_ud_mad(ud_statep, ud_statep->ud_stored_msg,
2075 ibcm_post_sidr_req_complete, ud_statep);
2076
2077 mutex_enter(&ud_statep->ud_state_mutex);
2078
2079 /* Wait for SIDR_REP */
2080 if (mode == IBT_BLOCKING) {
2081 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: blocking");
2082
2083 while (ud_statep->ud_blocking_done != B_TRUE) {
2084 cv_wait(&ud_statep->ud_block_client_cv,
2085 &ud_statep->ud_state_mutex);
2086 }
2087
2088 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: finished blocking");
2089
2090 if (ret_args->ud_status == IBT_CM_SREP_QPN_VALID) {
2091 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: DQPN = %x, "
2092 "status = %x, QKey = %x", ret_args->ud_dqpn,
2093 ret_args->ud_status, ret_args->ud_qkey);
2094
2095 } else {
2096 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: Status<%x>",
2097 ret_args->ud_status);
2098 retval = IBT_CM_FAILURE;
2099 }
2100 }
2101
2102 IBCM_UD_REF_CNT_DECR(ud_statep);
2103 mutex_exit(&ud_statep->ud_state_mutex);
2104
2105 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: done");
2106
2107 return (retval);
2108 }
2109
2110
2111 /*
2112 * Function:
2113 * ibt_request_ud_dest
2114 * Input:
2115 * ud_dest A previously allocated UD destination handle.
2116 * mode This function can execute in blocking or non blocking
2117 * modes.
2118 * attr UD destination attributes to be modified.
2119 * Output:
2120 * ud_ret_args If the function is called in blocking mode, ud_ret_args
2121 * should be a pointer to an ibt_ud_returns_t struct.
2122 * Returns:
2123 * IBT_SUCCESS
2124 * Description:
2125 * Modify a previously allocated UD destination handle based on the
2126 * results of doing the SIDR protocol.
2127 */
2128 ibt_status_t
2129 ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, ibt_execution_mode_t mode,
2130 ibt_ud_dest_attr_t *attr, ibt_ud_returns_t *ud_ret_args)
2131 {
2132 ibt_status_t retval;
2133 ibt_ud_dest_t *ud_destp;
2134 ibcm_local_handler_t *local_handler_priv = NULL;
2135
2136 IBTF_DPRINTF_L3(cmlog, "ibt_request_ud_dest(%p, %x, %p, %p)",
2137 ud_dest, mode, attr, ud_ret_args);
2138
2139 retval = ibcm_validate_dqpn_data(attr, mode, ud_ret_args);
2140 if (retval != IBT_SUCCESS) {
2141 return (retval);
2142 }
2143
2144 ud_destp = ud_dest;
2145
2146 /* Allocate an Address handle. */
2147 retval = ibt_modify_ah(ud_destp->ud_dest_hca, ud_destp->ud_ah,
2148 attr->ud_addr);
2149 if (retval != IBT_SUCCESS) {
2150 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: "
2151 "Address Handle Modification failed: %d", retval);
2152 return (retval);
2153 }
2154
2155 if (mode == IBT_NONBLOCKING) {
2156 /*
2157 * In NON-BLOCKING mode, and we need to update the destination
2158 * handle with the DQPN and QKey that are obtained from
2159 * SIDR REP, hook-up our own handler, so that we can catch
2160 * the event, and we ourselves call the actual client's
2161 * ud_cm_handler, in our handler.
2162 */
2163
2164 /* Allocate memory for local handler's private data. */
2165 local_handler_priv =
2166 kmem_alloc(sizeof (*local_handler_priv), KM_SLEEP);
2167
2168 local_handler_priv->actual_cm_handler = attr->ud_cm_handler;
2169 local_handler_priv->actual_cm_private = attr->ud_cm_private;
2170 local_handler_priv->dest_hdl = ud_destp;
2171
2172 attr->ud_cm_handler = ibcm_local_cm_handler;
2173 attr->ud_cm_private = local_handler_priv;
2174 }
2175
2176 /* In order to get DQPN and Destination QKey, perform SIDR */
2177 retval = ibcm_ud_get_dqpn(attr, mode, ud_ret_args);
2178 if (retval != IBT_SUCCESS) {
2179 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: "
2180 "Failed to get DQPN: %d", retval);
2181
2182 /* Free memory allocated for local handler's private data. */
2183 if (local_handler_priv != NULL)
2184 kmem_free(local_handler_priv,
2185 sizeof (*local_handler_priv));
2186 return (retval);
2187 }
2188
2189 /*
2190 * Fill in the dqpn and dqkey as obtained from ud_ret_args,
2191 * values will be valid only on BLOCKING mode.
2192 */
2193 if (mode == IBT_BLOCKING) {
2194 ud_destp->ud_dst_qpn = ud_ret_args->ud_dqpn;
2195 ud_destp->ud_qkey = ud_ret_args->ud_qkey;
2196 }
2197
2198 return (retval);
2199 }
2200
2201 /*
2202 * Function:
2203 * ibt_ud_get_dqpn
2204 * Input:
2205 * attr A pointer to an ibt_ud_dest_attr_t struct that are
2206 * required for SIDR REQ message. Not specified attributes
2207 * should be set to "NULL" or "0".
2208 * ud_sid, ud_addr and ud_pkt_lt must be specified.
2209 * mode This function can execute in blocking or non blocking
2210 * modes.
2211 * Output:
2212 * returns If the function is called in blocking mode, returns
2213 * should be a pointer to an ibt_ud_returns_t struct.
2214 * Return:
2215 * IBT_SUCCESS on success or respective failure on error.
2216 * Description:
2217 * Finds the destination QPN at the specified destination that the
2218 * specified service can be reached on. The IBTF CM initiates the
2219 * service ID resolution protocol (SIDR) to determine a destination QPN.
2220 *
2221 * NOTE: SIDR_REQ is initiated from active side.
2222 */
2223 ibt_status_t
2224 ibt_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode,
2225 ibt_ud_returns_t *returns)
2226 {
2227 ibt_status_t retval;
2228
2229 IBTF_DPRINTF_L3(cmlog, "ibt_ud_get_dqpn(%p, %x, %p)",
2230 attr, mode, returns);
2231
2232 retval = ibcm_validate_dqpn_data(attr, mode, returns);
2233 if (retval != IBT_SUCCESS) {
2234 return (retval);
2235 }
2236
2237 return (ibcm_ud_get_dqpn(attr, mode, returns));
2238 }
2239
2240
2241 /*
2242 * ibt_cm_delay:
2243 * A client CM handler function can call this function
2244 * to extend its response time to a CM event.
2245 * INPUTS:
2246 * flags Indicates what CM message processing is being delayed
2247 * by the CM handler, valid values are:
2248 * IBT_CM_DELAY_REQ
2249 * IBT_CM_DELAY_REP
2250 * IBT_CM_DELAY_LAP
2251 * cm_session_id The session ID that was passed to client srv_handler
2252 * by the CM
2253 * service_time The extended service time
2254 * priv_data Vendor specific data to be sent in the CM generated
2255 * MRA message. Should be NULL if not specified.
2256 * len The number of bytes of data specified by priv_data.
2257 *
2258 * RETURN VALUES:
2259 * IBT_SUCCESS on success (or respective failure on error)
2260 */
2261 ibt_status_t
2262 ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id,
2263 clock_t service_time, void *priv_data, ibt_priv_data_len_t len)
2264 {
2265 uint8_t msg_typ = 0;
2266 ibcm_mra_msg_t *mra_msgp;
2267 ibcm_state_data_t *statep;
2268 ibt_status_t status;
2269
2270 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay(0x%x, %p, 0x%x)",
2271 flags, cm_session_id, service_time);
2272
2273 /*
2274 * Make sure channel is associated with a statep
2275 */
2276 statep = (ibcm_state_data_t *)cm_session_id;
2277
2278 if (statep == NULL) {
2279 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: statep NULL");
2280 return (IBT_INVALID_PARAM);
2281 }
2282
2283 IBTF_DPRINTF_L4(cmlog, "ibt_cm_delay: statep %p", statep);
2284
2285 /* Allocate an ibmf msg for mra, if not allocated yet */
2286 if (statep->mra_msg == NULL) {
2287 if ((status = ibcm_alloc_out_msg(
2288 statep->stored_reply_addr.ibmf_hdl, &statep->mra_msg,
2289 MAD_METHOD_SEND)) != IBT_SUCCESS) {
2290 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: chan 0x%p"
2291 "IBMF MSG allocation failed", statep->channel);
2292 return (status);
2293 }
2294 }
2295
2296 mra_msgp = (ibcm_mra_msg_t *)IBCM_OUT_MSGP(statep->mra_msg);
2297 mra_msgp->mra_local_comm_id = h2b32(statep->local_comid);
2298 mra_msgp->mra_remote_comm_id = h2b32(statep->remote_comid);
2299
2300 /* fill in rest of MRA's fields - Message MRAed and Service Timeout */
2301 if (flags == IBT_CM_DELAY_REQ) {
2302 msg_typ = IBT_CM_MRA_TYPE_REQ;
2303 } else if (flags == IBT_CM_DELAY_REP) {
2304 msg_typ = IBT_CM_MRA_TYPE_REP;
2305 } else if (flags == IBT_CM_DELAY_LAP) {
2306 msg_typ = IBT_CM_MRA_TYPE_LAP;
2307 }
2308
2309 mra_msgp->mra_message_type_plus = msg_typ << 6;
2310 mra_msgp->mra_service_timeout_plus = ibt_usec2ib(service_time) << 3;
2311
2312 len = min(len, IBT_MRA_PRIV_DATA_SZ);
2313 if (priv_data && (len > 0))
2314 bcopy(priv_data, mra_msgp->mra_private_data, len);
2315
2316 IBCM_OUT_HDRP(statep->mra_msg)->AttributeID =
2317 h2b16(IBCM_INCOMING_MRA + IBCM_ATTR_BASE_ID);
2318
2319 mutex_enter(&statep->state_mutex);
2320
2321 if ((statep->mode == IBCM_ACTIVE_MODE) &&
2322 (statep->state == IBCM_STATE_REP_RCVD)) {
2323 statep->state = IBCM_STATE_MRA_REP_SENT;
2324 } else if (statep->mode == IBCM_PASSIVE_MODE) {
2325 if (statep->state == IBCM_STATE_REQ_RCVD) {
2326 statep->state = IBCM_STATE_MRA_SENT;
2327 } else if (statep->ap_state == IBCM_AP_STATE_LAP_RCVD) {
2328 statep->ap_state = IBCM_AP_STATE_MRA_LAP_RCVD;
2329 } else {
2330 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state "
2331 "/ap_state/mode %x, %x, %x", statep->state,
2332 statep->ap_state, statep->mode);
2333 mutex_exit(&statep->state_mutex);
2334 return (IBT_CHAN_STATE_INVALID);
2335 }
2336 } else {
2337 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state "
2338 "/ap_state/mode %x, %x, %x", statep->state,
2339 statep->ap_state, statep->mode);
2340 mutex_exit(&statep->state_mutex);
2341
2342 return (IBT_CHAN_STATE_INVALID);
2343 }
2344 /* service time is usecs, stale_clock is nsecs */
2345 statep->stale_clock = gethrtime() +
2346 (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 *
2347 statep->max_cm_retries);
2348
2349 statep->send_mad_flags |= IBCM_MRA_POST_BUSY;
2350 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_mra_complete */
2351 mutex_exit(&statep->state_mutex);
2352
2353 IBCM_OUT_HDRP(statep->mra_msg)->TransactionID =
2354 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID;
2355
2356 /* post the MRA mad in blocking mode, as no timers involved */
2357 ibcm_post_rc_mad(statep, statep->mra_msg, ibcm_post_mra_complete,
2358 statep);
2359 ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_MRA);
2360 /* If this message isn't seen then ibt_cm_delay failed */
2361 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay: done !!");
2362
2363 return (IBT_SUCCESS);
2364 }
2365
2366
2367 /*
2368 * ibt_register_service()
2369 * Register a service with the IBCM
2370 *
2371 * INPUTS:
2372 * ibt_hdl The IBT client handle returned to the client
2373 * on an ibt_attach() call.
2374 *
2375 * srv The address of a ibt_srv_desc_t that describes
2376 * the service, containing the following:
2377 *
2378 * sd_ud_handler The Service CM UD event Handler.
2379 * sd_handler The Service CM RC/UC/RD event Handler.
2380 * sd_flags Service flags (peer-to-peer, or not).
2381 *
2382 * sid This tells CM if the service is local (sid is 0) or
2383 * wellknown (sid is the starting service id of the range).
2384 *
2385 * num_sids The number of contiguous service-ids to reserve.
2386 *
2387 * srv_hdl The address of a service identification handle, used
2388 * to deregister a service, and to bind GIDs to.
2389 *
2390 * ret_sid The address to store the Service ID return value.
2391 * If num_sids > 1, ret_sid is the first Service ID
2392 * in the range.
2393 *
2394 * ibt_register_service() returns:
2395 * IBT_SUCCESS - added a service successfully.
2396 * IBT_INVALID_PARAM - invalid input parameter.
2397 * IBT_CM_FAILURE - failed to add the service.
2398 * IBT_CM_SERVICE_EXISTS - service already exists.
2399 * IBT_INSUFF_KERNEL_RESOURCE - ran out of local service ids (should
2400 * never happen).
2401 */
2402 ibt_status_t
2403 ibt_register_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_desc_t *srv,
2404 ib_svc_id_t sid, int num_sids, ibt_srv_hdl_t *srv_hdl, ib_svc_id_t *ret_sid)
2405 {
2406 ibcm_svc_info_t *svcinfop;
2407
2408 IBTF_DPRINTF_L2(cmlog, "ibt_register_service(%p (%s), %p, 0x%llX, %d)",
2409 ibt_hdl, ibtl_cm_get_clnt_name(ibt_hdl), srv, (longlong_t)sid,
2410 num_sids);
2411
2412 *srv_hdl = NULL;
2413
2414 if (num_sids <= 0) {
2415 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: "
2416 "Invalid number of service-ids specified (%d)", num_sids);
2417 return (IBT_INVALID_PARAM);
2418 }
2419
2420 if (sid == 0) {
2421 if (ret_sid == NULL)
2422 return (IBT_INVALID_PARAM);
2423 sid = ibcm_alloc_local_sids(num_sids);
2424 if (sid == 0)
2425 return (IBT_INSUFF_KERNEL_RESOURCE);
2426
2427 /* Make sure that the ServiceId specified is not of LOCAL AGN type. */
2428 } else if ((sid & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) {
2429 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: "
2430 "Invalid non-LOCAL SID specified: 0x%llX",
2431 (longlong_t)sid);
2432 return (IBT_INVALID_PARAM);
2433 }
2434
2435 svcinfop = ibcm_create_svc_entry(sid, num_sids);
2436
2437 if (svcinfop == NULL) {
2438 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: "
2439 "Service-ID 0x%llx already registered", (longlong_t)sid);
2440 return (IBT_CM_SERVICE_EXISTS);
2441 }
2442
2443 /*
2444 * 'sid' and 'num_sids' are filled in ibcm_create_svc_entry()
2445 */
2446 svcinfop->svc_flags = srv->sd_flags;
2447 svcinfop->svc_rc_handler = srv->sd_handler;
2448 svcinfop->svc_ud_handler = srv->sd_ud_handler;
2449
2450 if (ret_sid != NULL)
2451 *ret_sid = sid;
2452
2453 *srv_hdl = svcinfop;
2454
2455 ibtl_cm_change_service_cnt(ibt_hdl, num_sids);
2456
2457 /* If this message isn't seen, then ibt_register_service failed. */
2458 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: done (%p, %llX)",
2459 svcinfop, sid);
2460
2461 return (IBT_SUCCESS);
2462 }
2463
2464
2465 static ibt_status_t
2466 ibcm_write_service_record(ibmf_saa_handle_t saa_handle,
2467 sa_service_record_t *srv_recp, ibmf_saa_access_type_t saa_type)
2468 {
2469 int rval;
2470 int retry;
2471
2472 ibcm_sa_access_enter();
2473 for (retry = 0; retry < ibcm_max_sa_retries; retry++) {
2474 rval = ibmf_saa_update_service_record(
2475 saa_handle, srv_recp, saa_type, 0);
2476 if (rval != IBMF_TRANS_TIMEOUT) {
2477 break;
2478 }
2479 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: "
2480 "ibmf_saa_update_service_record timed out"
2481 " SID = %llX, rval = %d, saa_type = %d",
2482 (longlong_t)srv_recp->ServiceID, rval, saa_type);
2483 delay(ibcm_sa_timeout_delay);
2484 }
2485 ibcm_sa_access_exit();
2486
2487 if (rval != IBMF_SUCCESS) {
2488 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: "
2489 "ibmf_saa_update_service_record() : Failed - %d", rval);
2490 return (ibcm_ibmf_analyze_error(rval));
2491 } else
2492 return (IBT_SUCCESS);
2493 }
2494
2495
2496 static void
2497 ibcm_rem_stale_srec(ibmf_saa_handle_t saa_handle, sa_service_record_t *srec)
2498 {
2499 ibt_status_t retval;
2500 uint_t num_found;
2501 size_t length;
2502 sa_service_record_t *srv_resp;
2503 void *results_p;
2504 uint_t i;
2505 uint64_t component_mask;
2506 ibmf_saa_access_args_t access_args;
2507
2508 component_mask =
2509 SA_SR_COMPMASK_PKEY | SA_SR_COMPMASK_NAME | SA_SR_COMPMASK_GID;
2510
2511 /* Call in SA Access retrieve routine to get Service Records. */
2512 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID;
2513 access_args.sq_access_type = IBMF_SAA_RETRIEVE;
2514 access_args.sq_component_mask = component_mask;
2515 access_args.sq_template = srec;
2516 access_args.sq_template_length = sizeof (sa_service_record_t);
2517 access_args.sq_callback = NULL;
2518 access_args.sq_callback_arg = NULL;
2519
2520 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length,
2521 &results_p);
2522 if (retval != IBT_SUCCESS) {
2523 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: "
2524 "SA Access Failure");
2525 return;
2526 }
2527
2528 num_found = length / sizeof (sa_service_record_t);
2529
2530 if (num_found)
2531 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: "
2532 "Found %d matching Service Records.", num_found);
2533
2534 /* Validate the returned number of records. */
2535 if ((results_p != NULL) && (num_found > 0)) {
2536
2537 /* Remove all the records. */
2538 for (i = 0; i < num_found; i++) {
2539
2540 srv_resp = (sa_service_record_t *)
2541 ((uchar_t *)results_p +
2542 i * sizeof (sa_service_record_t));
2543
2544 /*
2545 * Found some matching records, but check out whether
2546 * this Record is really stale or just happens to match
2547 * the current session records. If yes, don't remove it.
2548 */
2549 mutex_enter(&ibcm_svc_info_lock);
2550 if (ibcm_find_svc_entry(srv_resp->ServiceID) != NULL) {
2551 /* This record is NOT STALE. */
2552 mutex_exit(&ibcm_svc_info_lock);
2553 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: "
2554 "This is not Stale, it's an active record");
2555 continue;
2556 }
2557 mutex_exit(&ibcm_svc_info_lock);
2558
2559 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: "
2560 "Removing Stale Rec: %s, %llX",
2561 srv_resp->ServiceName, srv_resp->ServiceID);
2562
2563 IBCM_DUMP_SERVICE_REC(srv_resp);
2564
2565 /*
2566 * Remove the Service Record Entry from SA.
2567 *
2568 * Get ServiceID info from Response Buf, other
2569 * attributes are already filled-in.
2570 */
2571
2572 srec->ServiceID = srv_resp->ServiceID;
2573
2574 (void) ibcm_write_service_record(saa_handle, srec,
2575 IBMF_SAA_DELETE);
2576 }
2577
2578 /* Deallocate the memory for results_p. */
2579 kmem_free(results_p, length);
2580 }
2581 }
2582
2583
2584
2585 /*
2586 * ibt_bind_service()
2587 * Register a service with the IBCM
2588 *
2589 * INPUTS:
2590 * srv_hdl The service id handle returned to the client
2591 * on an ibt_service_register() call.
2592 *
2593 * gid The GID to which to bind the service.
2594 *
2595 * srv_bind The address of a ibt_srv_bind_t that describes
2596 * the service record. This should be NULL if there
2597 * is to be no service record. This contains:
2598 *
2599 * sb_lease Lease period
2600 * sb_pkey Partition
2601 * sb_name pointer to ASCII string Service Name,
2602 * NULL terminated.
2603 * sb_key[] Key to secure the service record.
2604 * sb_data Service Data structure (64-byte)
2605 *
2606 * cm_private First argument of Service handler.
2607 *
2608 * sb_hdl_p The address of a service bind handle, used
2609 * to undo the service binding.
2610 *
2611 * ibt_bind_service() returns:
2612 * IBT_SUCCESS - added a service successfully.
2613 * IBT_INVALID_PARAM - invalid input parameter.
2614 * IBT_CM_FAILURE - failed to add the service.
2615 * IBT_CM_SERVICE_EXISTS - service already exists.
2616 */
2617 ibt_status_t
2618 ibt_bind_service(ibt_srv_hdl_t srv_hdl, ib_gid_t gid, ibt_srv_bind_t *srv_bind,
2619 void *cm_private, ibt_sbind_hdl_t *sb_hdl_p)
2620 {
2621 ibt_status_t status;
2622 ibtl_cm_hca_port_t port;
2623 ibcm_svc_bind_t *sbindp, *sbp;
2624 ibcm_hca_info_t *hcap;
2625 ib_svc_id_t sid, start_sid, end_sid;
2626 ibmf_saa_handle_t saa_handle;
2627 sa_service_record_t srv_rec;
2628 uint16_t pkey_ix;
2629
2630 if (sb_hdl_p != NULL)
2631 *sb_hdl_p = NULL; /* return value for error cases */
2632
2633 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: srv_hdl %p, gid (%llX:%llX)",
2634 srv_hdl, (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid);
2635
2636 /* Call ibtl_cm_get_hca_port to get the port number and the HCA GUID. */
2637 if ((status = ibtl_cm_get_hca_port(gid, 0, &port)) != IBT_SUCCESS) {
2638 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: "
2639 "ibtl_cm_get_hca_port failed: %d", status);
2640 return (status);
2641 }
2642 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: Port:%d HCA GUID:%llX",
2643 port.hp_port, port.hp_hca_guid);
2644
2645 hcap = ibcm_find_hca_entry(port.hp_hca_guid);
2646 if (hcap == NULL) {
2647 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: NO HCA found");
2648 return (IBT_HCA_BUSY_DETACHING);
2649 }
2650 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: hcap = %p", hcap);
2651
2652 if (srv_bind != NULL) {
2653 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port);
2654 if (saa_handle == NULL) {
2655 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: "
2656 "saa_handle is NULL");
2657 ibcm_dec_hca_acc_cnt(hcap);
2658 return (IBT_HCA_PORT_NOT_ACTIVE);
2659 }
2660 if (srv_bind->sb_pkey == 0) {
2661 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: "
2662 "P_Key must not be 0");
2663 ibcm_dec_hca_acc_cnt(hcap);
2664 return (IBT_INVALID_PARAM);
2665 }
2666 if (strlen(srv_bind->sb_name) >= IB_SVC_NAME_LEN) {
2667 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: "
2668 "Service Name is too long");
2669 ibcm_dec_hca_acc_cnt(hcap);
2670 return (IBT_INVALID_PARAM);
2671 } else
2672 IBTF_DPRINTF_L3(cmlog, "ibt_bind_service: "
2673 "Service Name='%s'", srv_bind->sb_name);
2674 status = ibt_pkey2index_byguid(port.hp_hca_guid,
2675 port.hp_port, srv_bind->sb_pkey, &pkey_ix);
2676 if (status != IBT_SUCCESS) {
2677 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: "
2678 "P_Key 0x%x not found in P_Key_Table",
2679 srv_bind->sb_pkey);
2680 ibcm_dec_hca_acc_cnt(hcap);
2681 return (status);
2682 }
2683 }
2684
2685 /* assume success - allocate before locking */
2686 sbindp = kmem_zalloc(sizeof (*sbindp), KM_SLEEP);
2687 sbindp->sbind_cm_private = cm_private;
2688 sbindp->sbind_gid = gid;
2689 sbindp->sbind_hcaguid = port.hp_hca_guid;
2690 sbindp->sbind_port = port.hp_port;
2691
2692 mutex_enter(&ibcm_svc_info_lock);
2693
2694 sbp = srv_hdl->svc_bind_list;
2695 while (sbp != NULL) {
2696 if (sbp->sbind_gid.gid_guid == gid.gid_guid &&
2697 sbp->sbind_gid.gid_prefix == gid.gid_prefix) {
2698 if (srv_bind == NULL ||
2699 srv_bind->sb_pkey == sbp->sbind_pkey) {
2700 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: "
2701 "failed: GID %llX:%llX and PKEY %x is "
2702 "already bound", gid.gid_prefix,
2703 gid.gid_guid, sbp->sbind_pkey);
2704 mutex_exit(&ibcm_svc_info_lock);
2705 ibcm_dec_hca_acc_cnt(hcap);
2706 kmem_free(sbindp, sizeof (*sbindp));
2707 return (IBT_CM_SERVICE_EXISTS);
2708 }
2709 }
2710 sbp = sbp->sbind_link;
2711 }
2712 /* no entry found */
2713
2714 sbindp->sbind_link = srv_hdl->svc_bind_list;
2715 srv_hdl->svc_bind_list = sbindp;
2716
2717 mutex_exit(&ibcm_svc_info_lock);
2718
2719 if (srv_bind != NULL) {
2720 bzero(&srv_rec, sizeof (srv_rec));
2721
2722 srv_rec.ServiceLease =
2723 sbindp->sbind_lease = srv_bind->sb_lease;
2724 srv_rec.ServiceP_Key =
2725 sbindp->sbind_pkey = srv_bind->sb_pkey;
2726 srv_rec.ServiceKey_hi =
2727 sbindp->sbind_key[0] = srv_bind->sb_key[0];
2728 srv_rec.ServiceKey_lo =
2729 sbindp->sbind_key[1] = srv_bind->sb_key[1];
2730 (void) strcpy(sbindp->sbind_name, srv_bind->sb_name);
2731 (void) strcpy((char *)srv_rec.ServiceName, srv_bind->sb_name);
2732 srv_rec.ServiceGID = gid;
2733
2734 /*
2735 * Find out whether we have any stale Local Service records
2736 * matching the current attributes. If yes, we shall try to
2737 * remove them from SA using the current request's ServiceKey.
2738 *
2739 * We will perform this operation only for Local Services, as
2740 * it is handled by SA automatically for WellKnown Services.
2741 *
2742 * Ofcourse, clients can specify NOT to do this clean-up by
2743 * setting IBT_SBIND_NO_CLEANUP flag (srv_bind->sb_flag).
2744 */
2745 if ((srv_hdl->svc_id & IB_SID_AGN_LOCAL) &&
2746 (!(srv_bind->sb_flag & IBT_SBIND_NO_CLEANUP))) {
2747 ibcm_rem_stale_srec(saa_handle, &srv_rec);
2748 }
2749
2750 /* Handle endianess for service data. */
2751 ibcm_swizzle_from_srv(&srv_bind->sb_data, sbindp->sbind_data);
2752
2753 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN);
2754
2755 /* insert srv record into the SA */
2756 start_sid = srv_hdl->svc_id;
2757 end_sid = start_sid + srv_hdl->svc_num_sids - 1;
2758 for (sid = start_sid; sid <= end_sid; sid++) {
2759
2760 srv_rec.ServiceID = sid;
2761
2762 IBCM_DUMP_SERVICE_REC(&srv_rec);
2763
2764 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: "
2765 "ibmf_saa_write_service_record, SvcId = %llX",
2766 (longlong_t)sid);
2767
2768 status = ibcm_write_service_record(saa_handle, &srv_rec,
2769 IBMF_SAA_UPDATE);
2770 if (status != IBT_SUCCESS) {
2771 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service:"
2772 " ibcm_write_service_record fails %d, "
2773 "sid %llX", status, (longlong_t)sid);
2774
2775 if (sid != start_sid) {
2776 /*
2777 * Bind failed while bind SID other than
2778 * first in the sid_range. So we need
2779 * to unbind those, which are passed.
2780 *
2781 * Need to increment svc count to
2782 * compensate for ibt_unbind_service().
2783 */
2784 ibcm_inc_hca_svc_cnt(hcap);
2785 ibcm_dec_hca_acc_cnt(hcap);
2786
2787 (void) ibt_unbind_service(srv_hdl,
2788 sbindp);
2789 } else {
2790 ibcm_svc_bind_t **sbpp;
2791
2792 /*
2793 * Bind failed for the first SID or the
2794 * only SID in question, then no need
2795 * to unbind, just free memory and
2796 * return error.
2797 */
2798 mutex_enter(&ibcm_svc_info_lock);
2799
2800 sbpp = &srv_hdl->svc_bind_list;
2801 sbp = *sbpp;
2802 while (sbp != NULL) {
2803 if (sbp == sbindp) {
2804 *sbpp = sbp->sbind_link;
2805 break;
2806 }
2807 sbpp = &sbp->sbind_link;
2808 sbp = *sbpp;
2809 }
2810 mutex_exit(&ibcm_svc_info_lock);
2811 ibcm_dec_hca_acc_cnt(hcap);
2812
2813 kmem_free(sbindp, sizeof (*sbindp));
2814 }
2815 return (status);
2816 }
2817 }
2818 }
2819 ibcm_inc_hca_svc_cnt(hcap);
2820 ibcm_dec_hca_acc_cnt(hcap);
2821
2822 /* If this message isn't seen then ibt_bind_service failed */
2823 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: DONE (%p, %llX:%llX)",
2824 srv_hdl, gid.gid_prefix, gid.gid_guid);
2825
2826 if (sb_hdl_p != NULL)
2827 *sb_hdl_p = sbindp;
2828
2829 return (IBT_SUCCESS);
2830 }
2831
2832 ibt_status_t
2833 ibt_unbind_service(ibt_srv_hdl_t srv_hdl, ibt_sbind_hdl_t sbindp)
2834 {
2835 ib_svc_id_t sid, end_sid;
2836 ibt_status_t rval;
2837 ibcm_hca_info_t *hcap;
2838 ibcm_svc_bind_t *sbp, **sbpp;
2839
2840 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service(%p, %p)",
2841 srv_hdl, sbindp);
2842
2843 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid);
2844
2845 /* If there is a service on hca, respective hcap cannot go away */
2846 ASSERT(hcap != NULL);
2847
2848 mutex_enter(&ibcm_svc_info_lock);
2849
2850 sbpp = &srv_hdl->svc_bind_list;
2851 sbp = *sbpp;
2852 while (sbp != NULL) {
2853 if (sbp == sbindp) {
2854 *sbpp = sbp->sbind_link;
2855 break;
2856 }
2857 sbpp = &sbp->sbind_link;
2858 sbp = *sbpp;
2859 }
2860 sid = srv_hdl->svc_id;
2861 end_sid = srv_hdl->svc_id + srv_hdl->svc_num_sids - 1;
2862 if (sbp != NULL)
2863 while (sbp->sbind_rewrite_state == IBCM_REWRITE_BUSY)
2864 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock);
2865 mutex_exit(&ibcm_svc_info_lock);
2866
2867 if (sbp == NULL) {
2868 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: "
2869 "service binding not found: srv_hdl %p, srv_bind %p",
2870 srv_hdl, sbindp);
2871 ibcm_dec_hca_acc_cnt(hcap);
2872 return (IBT_INVALID_PARAM);
2873 }
2874
2875 if (sbindp->sbind_pkey != 0) { /* Are there service records? */
2876 ibtl_cm_hca_port_t port;
2877 sa_service_record_t srv_rec;
2878 ibmf_saa_handle_t saa_handle;
2879 ibt_status_t status;
2880
2881 /* get the default SGID of the port */
2882 if ((status = ibtl_cm_get_hca_port(sbindp->sbind_gid, 0, &port))
2883 != IBT_SUCCESS) {
2884 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: "
2885 "ibtl_cm_get_hca_port failed: %d", status);
2886 /* we're done, but there may be stale service records */
2887 goto done;
2888 }
2889
2890 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port);
2891 if (saa_handle == NULL) {
2892 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: "
2893 "saa_handle is NULL");
2894 /* we're done, but there may be stale service records */
2895 goto done;
2896 }
2897
2898 /* Fill in fields of srv_rec */
2899 bzero(&srv_rec, sizeof (srv_rec));
2900
2901 srv_rec.ServiceP_Key = sbindp->sbind_pkey;
2902 srv_rec.ServiceKey_hi = sbindp->sbind_key[0];
2903 srv_rec.ServiceKey_lo = sbindp->sbind_key[1];
2904 srv_rec.ServiceGID = sbindp->sbind_gid;
2905 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name);
2906
2907 while (sid <= end_sid) {
2908
2909 srv_rec.ServiceID = sid;
2910 IBCM_DUMP_SERVICE_REC(&srv_rec);
2911
2912 rval = ibcm_write_service_record(saa_handle, &srv_rec,
2913 IBMF_SAA_DELETE);
2914
2915 IBTF_DPRINTF_L4(cmlog, "ibt_unbind_service: "
2916 "ibcm_write_service_record rval = %d, SID %llx",
2917 rval, sid);
2918 if (rval != IBT_SUCCESS) {
2919 /* this is not considered a reason to fail */
2920 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: "
2921 "ibcm_write_service_record fails %d, "
2922 "sid %llx", rval, sid);
2923 }
2924 sid++;
2925 }
2926 }
2927 done:
2928 ibcm_dec_hca_svc_cnt(hcap);
2929 ibcm_dec_hca_acc_cnt(hcap);
2930 kmem_free(sbindp, sizeof (*sbindp));
2931
2932 /* If this message isn't seen then ibt_unbind_service failed */
2933 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: done !!");
2934
2935 return (IBT_SUCCESS);
2936 }
2937
2938 /*
2939 * Simply pull off each binding from the list and unbind it.
2940 * If any of the unbind calls fail, we fail.
2941 */
2942 ibt_status_t
2943 ibt_unbind_all_services(ibt_srv_hdl_t srv_hdl)
2944 {
2945 ibt_status_t status;
2946 ibcm_svc_bind_t *sbp;
2947
2948 mutex_enter(&ibcm_svc_info_lock);
2949 sbp = NULL;
2950
2951 /* this compare keeps the loop from being infinite */
2952 while (sbp != srv_hdl->svc_bind_list) {
2953 sbp = srv_hdl->svc_bind_list;
2954 mutex_exit(&ibcm_svc_info_lock);
2955 status = ibt_unbind_service(srv_hdl, sbp);
2956 if (status != IBT_SUCCESS)
2957 return (status);
2958 mutex_enter(&ibcm_svc_info_lock);
2959 if (srv_hdl->svc_bind_list == NULL)
2960 break;
2961 }
2962 mutex_exit(&ibcm_svc_info_lock);
2963 return (IBT_SUCCESS);
2964 }
2965
2966 /*
2967 * ibt_deregister_service()
2968 * Deregister a service with the IBCM
2969 *
2970 * INPUTS:
2971 * ibt_hdl The IBT client handle returned to the client
2972 * on an ibt_attach() call.
2973 *
2974 * srv_hdl The address of a service identification handle, used
2975 * to de-register a service.
2976 * RETURN VALUES:
2977 * IBT_SUCCESS on success (or respective failure on error)
2978 */
2979 ibt_status_t
2980 ibt_deregister_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_hdl_t srv_hdl)
2981 {
2982 ibcm_svc_info_t *svcp;
2983 ibcm_svc_lookup_t svc;
2984
2985 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(%p (%s), %p)",
2986 ibt_hdl, ibtl_cm_get_clnt_name(ibt_hdl), srv_hdl);
2987
2988 mutex_enter(&ibcm_svc_info_lock);
2989
2990 if (srv_hdl->svc_bind_list != NULL) {
2991 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service:"
2992 " srv_hdl %p still has bindings", srv_hdl);
2993 mutex_exit(&ibcm_svc_info_lock);
2994 return (IBT_CM_SERVICE_BUSY);
2995 }
2996 svc.sid = srv_hdl->svc_id;
2997 svc.num_sids = 1;
2998 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_service: SID 0x%llX, numsids %d",
2999 srv_hdl->svc_id, srv_hdl->svc_num_sids);
3000
3001 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL);
3002 if (svcp != srv_hdl) {
3003 mutex_exit(&ibcm_svc_info_lock);
3004 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(): "
3005 "srv_hdl %p not found", srv_hdl);
3006 return (IBT_INVALID_PARAM);
3007 }
3008 avl_remove(&ibcm_svc_avl_tree, svcp);
3009
3010 /* wait for active REQ/SREQ handling to be done */
3011 svcp->svc_to_delete = 1;
3012 while (svcp->svc_ref_cnt != 0)
3013 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock);
3014
3015 mutex_exit(&ibcm_svc_info_lock);
3016
3017 if ((srv_hdl->svc_id & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL)
3018 ibcm_free_local_sids(srv_hdl->svc_id, srv_hdl->svc_num_sids);
3019
3020 ibtl_cm_change_service_cnt(ibt_hdl, -srv_hdl->svc_num_sids);
3021 kmem_free(srv_hdl, sizeof (*srv_hdl));
3022
3023 /* If this message isn't seen then ibt_deregister_service failed */
3024 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service: done !!");
3025
3026 return (IBT_SUCCESS);
3027 }
3028
3029 ibcm_status_t
3030 ibcm_ar_init(void)
3031 {
3032 ib_svc_id_t sid = IBCM_DAPL_ATS_SID;
3033 ibcm_svc_info_t *tmp_svcp;
3034
3035 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init()");
3036
3037 /* remove this special SID from the pool of available SIDs */
3038 if ((tmp_svcp = ibcm_create_svc_entry(sid, 1)) == NULL) {
3039 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init: "
3040 "DAPL ATS SID 0x%llx already registered", (longlong_t)sid);
3041 return (IBCM_FAILURE);
3042 }
3043 mutex_enter(&ibcm_svc_info_lock);
3044 ibcm_ar_svcinfop = tmp_svcp;
3045 ibcm_ar_list = NULL; /* no address records registered yet */
3046 mutex_exit(&ibcm_svc_info_lock);
3047 return (IBCM_SUCCESS);
3048 }
3049
3050 ibcm_status_t
3051 ibcm_ar_fini(void)
3052 {
3053 ibcm_ar_t *ar_list;
3054 ibcm_svc_info_t *tmp_svcp;
3055
3056 mutex_enter(&ibcm_svc_info_lock);
3057 ar_list = ibcm_ar_list;
3058
3059 if (ar_list == NULL &&
3060 avl_numnodes(&ibcm_svc_avl_tree) == 1 &&
3061 avl_first(&ibcm_svc_avl_tree) == ibcm_ar_svcinfop) {
3062 avl_remove(&ibcm_svc_avl_tree, ibcm_ar_svcinfop);
3063 tmp_svcp = ibcm_ar_svcinfop;
3064 mutex_exit(&ibcm_svc_info_lock);
3065 kmem_free(tmp_svcp, sizeof (*ibcm_ar_svcinfop));
3066 return (IBCM_SUCCESS);
3067 }
3068 mutex_exit(&ibcm_svc_info_lock);
3069 return (IBCM_FAILURE);
3070 }
3071
3072
3073 /*
3074 * Return to the caller:
3075 * IBT_SUCCESS Found a perfect match.
3076 * *arpp is set to the record.
3077 * IBT_INCONSISTENT_AR Found a record that's inconsistent.
3078 * IBT_AR_NOT_REGISTERED Found no record with same GID/pkey and
3079 * found no record with same data.
3080 */
3081 static ibt_status_t
3082 ibcm_search_ar(ibt_ar_t *arp, ibcm_ar_t **arpp)
3083 {
3084 ibcm_ar_t *tmp;
3085 int i;
3086
3087 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock));
3088 tmp = ibcm_ar_list;
3089 while (tmp != NULL) {
3090 if (tmp->ar.ar_gid.gid_prefix == arp->ar_gid.gid_prefix &&
3091 tmp->ar.ar_gid.gid_guid == arp->ar_gid.gid_guid &&
3092 tmp->ar.ar_pkey == arp->ar_pkey) {
3093 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++)
3094 if (tmp->ar.ar_data[i] != arp->ar_data[i])
3095 return (IBT_INCONSISTENT_AR);
3096 *arpp = tmp;
3097 return (IBT_SUCCESS);
3098 } else {
3099 /* if all the data bytes match, we have inconsistency */
3100 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++)
3101 if (tmp->ar.ar_data[i] != arp->ar_data[i])
3102 break;
3103 if (i == IBCM_DAPL_ATS_NBYTES)
3104 return (IBT_INCONSISTENT_AR);
3105 /* try next address record */
3106 }
3107 tmp = tmp->ar_link;
3108 }
3109 return (IBT_AR_NOT_REGISTERED);
3110 }
3111
3112 ibt_status_t
3113 ibt_register_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp)
3114 {
3115 ibcm_ar_t *found;
3116 ibcm_ar_t *tmp;
3117 ibt_status_t status;
3118 ibt_status_t s1, s2;
3119 char *s;
3120 ibcm_ar_ref_t *hdlp;
3121 ibcm_ar_t *new;
3122 ibcm_ar_t **linkp;
3123 ibtl_cm_hca_port_t cm_port;
3124 uint16_t pkey_ix;
3125 ibcm_hca_info_t *hcap;
3126 ibmf_saa_handle_t saa_handle;
3127 sa_service_record_t *srv_recp;
3128 uint64_t gid_ored;
3129
3130 IBTF_DPRINTF_L3(cmlog, "ibt_register_ar: PKey 0x%X GID %llX:%llX",
3131 arp->ar_pkey, (longlong_t)arp->ar_gid.gid_prefix,
3132 (longlong_t)arp->ar_gid.gid_guid);
3133
3134 /*
3135 * If P_Key is 0, but GID is not, this query is invalid.
3136 * If GID is 0, but P_Key is not, this query is invalid.
3137 */
3138 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix;
3139 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) ||
3140 (arp->ar_pkey != 0 && gid_ored == 0ULL)) {
3141 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: "
3142 "GID/P_Key is not valid");
3143 return (IBT_INVALID_PARAM);
3144 }
3145
3146 /* assume success, so these might be needed */
3147 hdlp = kmem_alloc(sizeof (*hdlp), KM_SLEEP);
3148 new = kmem_zalloc(sizeof (*new), KM_SLEEP);
3149
3150 mutex_enter(&ibcm_svc_info_lock);
3151 /* search for existing GID/pkey (there can be at most 1) */
3152 status = ibcm_search_ar(arp, &found);
3153 if (status == IBT_INCONSISTENT_AR) {
3154 mutex_exit(&ibcm_svc_info_lock);
3155 kmem_free(new, sizeof (*new));
3156 kmem_free(hdlp, sizeof (*hdlp));
3157 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: "
3158 "address record is inconsistent with a known one");
3159 return (IBT_INCONSISTENT_AR);
3160 } else if (status == IBT_SUCCESS) {
3161 if (found->ar_flags == IBCM_AR_INITING) {
3162 found->ar_waiters++;
3163 cv_wait(&found->ar_cv, &ibcm_svc_info_lock);
3164 found->ar_waiters--;
3165 }
3166 if (found->ar_flags == IBCM_AR_FAILED) {
3167 if (found->ar_waiters == 0) {
3168 cv_destroy(&found->ar_cv);
3169 kmem_free(found, sizeof (*found));
3170 }
3171 mutex_exit(&ibcm_svc_info_lock);
3172 kmem_free(new, sizeof (*new));
3173 kmem_free(hdlp, sizeof (*hdlp));
3174 return (ibt_get_module_failure(IBT_FAILURE_IBCM, 0));
3175 }
3176 hdlp->ar_ibt_hdl = ibt_hdl;
3177 hdlp->ar_ref_link = found->ar_ibt_hdl_list;
3178 found->ar_ibt_hdl_list = hdlp;
3179 mutex_exit(&ibcm_svc_info_lock);
3180 kmem_free(new, sizeof (*new));
3181 ibtl_cm_change_service_cnt(ibt_hdl, 1);
3182 return (IBT_SUCCESS);
3183 } else {
3184 ASSERT(status == IBT_AR_NOT_REGISTERED);
3185 }
3186 hdlp->ar_ref_link = NULL;
3187 hdlp->ar_ibt_hdl = ibt_hdl;
3188 new->ar_ibt_hdl_list = hdlp;
3189 new->ar = *arp;
3190 new->ar_flags = IBCM_AR_INITING;
3191 new->ar_waiters = 0;
3192 cv_init(&new->ar_cv, NULL, CV_DEFAULT, NULL);
3193 new->ar_link = ibcm_ar_list;
3194 ibcm_ar_list = new;
3195
3196 /* verify GID/pkey is valid for a local port, etc. */
3197 hcap = NULL;
3198 if ((s1 = ibtl_cm_get_hca_port(arp->ar_gid, 0, &cm_port))
3199 != IBT_SUCCESS ||
3200 (s2 = ibt_pkey2index_byguid(cm_port.hp_hca_guid, cm_port.hp_port,
3201 arp->ar_pkey, &pkey_ix)) != IBT_SUCCESS ||
3202 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL) {
3203 cv_destroy(&new->ar_cv);
3204 ibcm_ar_list = new->ar_link;
3205 mutex_exit(&ibcm_svc_info_lock);
3206 kmem_free(new, sizeof (*new));
3207 kmem_free(hdlp, sizeof (*hdlp));
3208 status = IBT_INVALID_PARAM;
3209 if (s1 == IBT_HCA_PORT_NOT_ACTIVE) {
3210 s = "PORT DOWN";
3211 status = IBT_HCA_PORT_NOT_ACTIVE;
3212 } else if (s1 != IBT_SUCCESS)
3213 s = "GID not found";
3214 else if (s2 != IBT_SUCCESS)
3215 s = "PKEY not found";
3216 else
3217 s = "CM could not find its HCA entry";
3218 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: %s, status = %d",
3219 s, status);
3220 return (status);
3221 }
3222 mutex_exit(&ibcm_svc_info_lock);
3223 saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port);
3224
3225 /* create service record */
3226 srv_recp = kmem_zalloc(sizeof (*srv_recp), KM_SLEEP);
3227 srv_recp->ServiceLease = 0xFFFFFFFF; /* infinite */
3228 srv_recp->ServiceP_Key = arp->ar_pkey;
3229 srv_recp->ServiceKey_hi = 0xDA410000ULL; /* DAPL */
3230 srv_recp->ServiceKey_lo = 0xA7500000ULL; /* ATS */
3231 (void) strcpy((char *)srv_recp->ServiceName, IBCM_DAPL_ATS_NAME);
3232 srv_recp->ServiceGID = arp->ar_gid;
3233 bcopy(arp->ar_data, srv_recp->ServiceData, IBCM_DAPL_ATS_NBYTES);
3234 srv_recp->ServiceID = IBCM_DAPL_ATS_SID;
3235
3236 /* insert service record into the SA */
3237
3238 IBCM_DUMP_SERVICE_REC(srv_recp);
3239
3240 if (saa_handle != NULL)
3241 status = ibcm_write_service_record(saa_handle, srv_recp,
3242 IBMF_SAA_UPDATE);
3243 else
3244 status = IBT_HCA_PORT_NOT_ACTIVE;
3245
3246 if (status != IBT_SUCCESS) {
3247 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: sa access fails %d, "
3248 "sid %llX", status, (longlong_t)srv_recp->ServiceID);
3249 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: FAILED for gid "
3250 "%llX:%llX pkey 0x%X", (longlong_t)arp->ar_gid.gid_prefix,
3251 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey);
3252
3253 kmem_free(srv_recp, sizeof (*srv_recp));
3254 kmem_free(hdlp, sizeof (*hdlp));
3255
3256 mutex_enter(&ibcm_svc_info_lock);
3257 linkp = &ibcm_ar_list;
3258 tmp = *linkp;
3259 while (tmp != NULL) {
3260 if (tmp == new) {
3261 *linkp = new->ar_link;
3262 break;
3263 }
3264 linkp = &tmp->ar_link;
3265 tmp = *linkp;
3266 }
3267 if (new->ar_waiters > 0) {
3268 new->ar_flags = IBCM_AR_FAILED;
3269 cv_broadcast(&new->ar_cv);
3270 mutex_exit(&ibcm_svc_info_lock);
3271 } else {
3272 cv_destroy(&new->ar_cv);
3273 mutex_exit(&ibcm_svc_info_lock);
3274 kmem_free(new, sizeof (*new));
3275 }
3276 ibcm_dec_hca_acc_cnt(hcap);
3277 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: "
3278 "IBMF_SAA failed to write address record");
3279 } else { /* SUCCESS */
3280 uint8_t *b;
3281
3282 IBTF_DPRINTF_L3(cmlog, "ibt_register_ar: SUCCESS for gid "
3283 "%llx:%llx pkey %x", (longlong_t)arp->ar_gid.gid_prefix,
3284 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey);
3285 b = arp->ar_data;
3286
3287 IBTF_DPRINTF_L3(cmlog, "ibt_register_ar:"
3288 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d",
3289 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9],
3290 b[10], b[11], b[12], b[13], b[14], b[15]);
3291 mutex_enter(&ibcm_svc_info_lock);
3292 new->ar_srv_recp = srv_recp;
3293 new->ar_saa_handle = saa_handle;
3294 new->ar_port = cm_port.hp_port;
3295 new->ar_hcap = hcap;
3296 new->ar_flags = IBCM_AR_SUCCESS;
3297 if (new->ar_waiters > 0)
3298 cv_broadcast(&new->ar_cv);
3299 mutex_exit(&ibcm_svc_info_lock);
3300 ibtl_cm_change_service_cnt(ibt_hdl, 1);
3301 /* do not call ibcm_dec_hca_acc_cnt(hcap) until deregister */
3302 }
3303 return (status);
3304 }
3305
3306 ibt_status_t
3307 ibt_deregister_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp)
3308 {
3309 ibcm_ar_t *found;
3310 ibcm_ar_t *tmp;
3311 ibcm_ar_t **linkp;
3312 ibcm_ar_ref_t *hdlp;
3313 ibcm_ar_ref_t **hdlpp;
3314 ibt_status_t status;
3315 ibmf_saa_handle_t saa_handle;
3316 sa_service_record_t *srv_recp;
3317 uint64_t gid_ored;
3318
3319 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_ar: pkey %x", arp->ar_pkey);
3320 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_ar: gid %llx:%llx",
3321 (longlong_t)arp->ar_gid.gid_prefix,
3322 (longlong_t)arp->ar_gid.gid_guid);
3323
3324 /*
3325 * If P_Key is 0, but GID is not, this query is invalid.
3326 * If GID is 0, but P_Key is not, this query is invalid.
3327 */
3328 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix;
3329 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) ||
3330 (arp->ar_pkey != 0 && gid_ored == 0ULL)) {
3331 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: "
3332 "GID/P_Key is not valid");
3333 return (IBT_INVALID_PARAM);
3334 }
3335
3336 mutex_enter(&ibcm_svc_info_lock);
3337 /* search for existing GID/pkey (there can be at most 1) */
3338 status = ibcm_search_ar(arp, &found);
3339 if (status == IBT_INCONSISTENT_AR || status == IBT_AR_NOT_REGISTERED) {
3340 mutex_exit(&ibcm_svc_info_lock);
3341 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: "
3342 "address record not found");
3343 return (IBT_AR_NOT_REGISTERED);
3344 }
3345 ASSERT(status == IBT_SUCCESS);
3346
3347 hdlpp = &found->ar_ibt_hdl_list;
3348 hdlp = *hdlpp;
3349 while (hdlp != NULL) {
3350 if (hdlp->ar_ibt_hdl == ibt_hdl)
3351 break;
3352 hdlpp = &hdlp->ar_ref_link;
3353 hdlp = *hdlpp;
3354 }
3355 if (hdlp == NULL) { /* could not find ibt_hdl on list */
3356 mutex_exit(&ibcm_svc_info_lock);
3357 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: "
3358 "address record found, but not for this client");
3359 return (IBT_AR_NOT_REGISTERED);
3360 }
3361 *hdlpp = hdlp->ar_ref_link; /* remove ref for this client */
3362 if (found->ar_ibt_hdl_list == NULL && found->ar_waiters == 0) {
3363 /* last entry was removed */
3364 found->ar_flags = IBCM_AR_INITING; /* hold off register_ar */
3365 saa_handle = found->ar_saa_handle;
3366 srv_recp = found->ar_srv_recp;
3367
3368 /* wait if this service record is being rewritten */
3369 while (found->ar_rewrite_state == IBCM_REWRITE_BUSY)
3370 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock);
3371 mutex_exit(&ibcm_svc_info_lock);
3372
3373 /* remove service record */
3374 status = ibcm_write_service_record(saa_handle, srv_recp,
3375 IBMF_SAA_DELETE);
3376 if (status != IBT_SUCCESS)
3377 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: "
3378 "IBMF_SAA failed to delete address record");
3379 mutex_enter(&ibcm_svc_info_lock);
3380 if (found->ar_waiters == 0) { /* still no waiters */
3381 linkp = &ibcm_ar_list;
3382 tmp = *linkp;
3383 while (tmp != found) {
3384 linkp = &tmp->ar_link;
3385 tmp = *linkp;
3386 }
3387 *linkp = tmp->ar_link;
3388 ibcm_dec_hca_acc_cnt(found->ar_hcap);
3389 kmem_free(srv_recp, sizeof (*srv_recp));
3390 cv_destroy(&found->ar_cv);
3391 kmem_free(found, sizeof (*found));
3392 } else {
3393 /* add service record back in for the waiters */
3394 mutex_exit(&ibcm_svc_info_lock);
3395 status = ibcm_write_service_record(saa_handle, srv_recp,
3396 IBMF_SAA_UPDATE);
3397 mutex_enter(&ibcm_svc_info_lock);
3398 if (status == IBT_SUCCESS)
3399 found->ar_flags = IBCM_AR_SUCCESS;
3400 else {
3401 found->ar_flags = IBCM_AR_FAILED;
3402 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: "
3403 "IBMF_SAA failed to write address record");
3404 }
3405 cv_broadcast(&found->ar_cv);
3406 }
3407 }
3408 mutex_exit(&ibcm_svc_info_lock);
3409 kmem_free(hdlp, sizeof (*hdlp));
3410 ibtl_cm_change_service_cnt(ibt_hdl, -1);
3411 return (status);
3412 }
3413
3414 ibt_status_t
3415 ibt_query_ar(ib_gid_t *sgid, ibt_ar_t *queryp, ibt_ar_t *resultp)
3416 {
3417 sa_service_record_t svcrec_req;
3418 sa_service_record_t *svcrec_resp;
3419 void *results_p;
3420 uint64_t component_mask = 0;
3421 uint64_t gid_ored;
3422 size_t length;
3423 int num_rec;
3424 int i;
3425 ibmf_saa_access_args_t access_args;
3426 ibt_status_t retval;
3427 ibtl_cm_hca_port_t cm_port;
3428 ibcm_hca_info_t *hcap;
3429 ibmf_saa_handle_t saa_handle;
3430
3431 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar(%p, %p)", queryp, resultp);
3432 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: sgid %llx:%llx",
3433 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid);
3434 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_pkey %x", queryp->ar_pkey);
3435 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_gid %llx:%llx",
3436 (longlong_t)queryp->ar_gid.gid_prefix,
3437 (longlong_t)queryp->ar_gid.gid_guid);
3438
3439 /*
3440 * If P_Key is 0, but GID is not, this query is invalid.
3441 * If GID is 0, but P_Key is not, this query is invalid.
3442 */
3443 gid_ored = queryp->ar_gid.gid_guid | queryp->ar_gid.gid_prefix;
3444 if ((queryp->ar_pkey == 0 && gid_ored != 0ULL) ||
3445 (queryp->ar_pkey != 0 && gid_ored == 0ULL)) {
3446 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: GID/P_Key is not valid");
3447 return (IBT_INVALID_PARAM);
3448 }
3449
3450 hcap = NULL;
3451 if (ibtl_cm_get_hca_port(*sgid, 0, &cm_port) != IBT_SUCCESS ||
3452 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL ||
3453 (saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port)) == NULL) {
3454 if (hcap != NULL)
3455 ibcm_dec_hca_acc_cnt(hcap);
3456 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: sgid is not valid");
3457 return (IBT_INVALID_PARAM);
3458 }
3459
3460 bzero(&svcrec_req, sizeof (svcrec_req));
3461
3462 /* Is GID/P_Key Specified. */
3463 if (queryp->ar_pkey != 0) { /* GID is non-zero from check above */
3464 svcrec_req.ServiceP_Key = queryp->ar_pkey;
3465 component_mask |= SA_SR_COMPMASK_PKEY;
3466 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: P_Key %X",
3467 queryp->ar_pkey);
3468 svcrec_req.ServiceGID = queryp->ar_gid;
3469 component_mask |= SA_SR_COMPMASK_GID;
3470 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: GID %llX:%llX",
3471 (longlong_t)queryp->ar_gid.gid_prefix,
3472 (longlong_t)queryp->ar_gid.gid_guid);
3473 }
3474
3475 /* Is ServiceData Specified. */
3476 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) {
3477 if (queryp->ar_data[i] != 0) {
3478 bcopy(queryp->ar_data, svcrec_req.ServiceData,
3479 IBCM_DAPL_ATS_NBYTES);
3480 component_mask |= 0xFFFF << 7; /* all 16 Data8 */
3481 /* components */
3482 break;
3483 }
3484 }
3485
3486 /* Service Name */
3487 (void) strcpy((char *)svcrec_req.ServiceName, IBCM_DAPL_ATS_NAME);
3488 component_mask |= SA_SR_COMPMASK_NAME;
3489
3490 svcrec_req.ServiceID = IBCM_DAPL_ATS_SID;
3491 component_mask |= SA_SR_COMPMASK_ID;
3492
3493 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: "
3494 "Perform SA Access: Mask: 0x%X", component_mask);
3495
3496 /*
3497 * Call in SA Access retrieve routine to get Service Records.
3498 *
3499 * SA Access framework allocated memory for the "results_p".
3500 * Make sure to deallocate once we are done with the results_p.
3501 * The size of the buffer allocated will be as returned in
3502 * "length" field.
3503 */
3504 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID;
3505 access_args.sq_access_type = IBMF_SAA_RETRIEVE;
3506 access_args.sq_component_mask = component_mask;
3507 access_args.sq_template = &svcrec_req;
3508 access_args.sq_template_length = sizeof (sa_service_record_t);
3509 access_args.sq_callback = NULL;
3510 access_args.sq_callback_arg = NULL;
3511
3512 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length,
3513 &results_p);
3514
3515 ibcm_dec_hca_acc_cnt(hcap);
3516 if (retval != IBT_SUCCESS) {
3517 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: SA Access Failed");
3518 return (retval);
3519 }
3520
3521 num_rec = length / sizeof (sa_service_record_t);
3522
3523 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: "
3524 "Found %d Service Records.", num_rec);
3525
3526 /* Validate the returned number of records. */
3527 if ((results_p != NULL) && (num_rec > 0)) {
3528 uint8_t *b;
3529
3530 /* Just return info from the first service record. */
3531 svcrec_resp = (sa_service_record_t *)results_p;
3532
3533 /* The Service GID and Service ID */
3534 resultp->ar_gid = svcrec_resp->ServiceGID;
3535 resultp->ar_pkey = svcrec_resp->ServiceP_Key;
3536 bcopy(svcrec_resp->ServiceData,
3537 resultp->ar_data, IBCM_DAPL_ATS_NBYTES);
3538
3539 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: "
3540 "Found: pkey %x dgid %llX:%llX", resultp->ar_pkey,
3541 (longlong_t)resultp->ar_gid.gid_prefix,
3542 (longlong_t)resultp->ar_gid.gid_guid);
3543 b = resultp->ar_data;
3544 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar:"
3545 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d",
3546 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9],
3547 b[10], b[11], b[12], b[13], b[14], b[15]);
3548
3549 /* Deallocate the memory for results_p. */
3550 kmem_free(results_p, length);
3551 if (num_rec > 1)
3552 retval = IBT_MULTIPLE_AR;
3553 else
3554 retval = IBT_SUCCESS;
3555 } else {
3556 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: "
3557 "ibmf_sa_access found 0 matching records");
3558 retval = IBT_AR_NOT_REGISTERED;
3559 }
3560 return (retval);
3561 }
3562
3563 /* mark all ATS service records associated with the port */
3564 static void
3565 ibcm_mark_ar(ib_guid_t hca_guid, uint8_t port)
3566 {
3567 ibcm_ar_t *tmp;
3568
3569 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock));
3570 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) {
3571 if (tmp->ar_hcap == NULL)
3572 continue;
3573 if (tmp->ar_hcap->hca_guid == hca_guid &&
3574 tmp->ar_port == port) {
3575 /* even if it's busy, we mark it for rewrite */
3576 tmp->ar_rewrite_state = IBCM_REWRITE_NEEDED;
3577 }
3578 }
3579 }
3580
3581 /* rewrite all ATS service records */
3582 static int
3583 ibcm_rewrite_ar(void)
3584 {
3585 ibcm_ar_t *tmp;
3586 ibmf_saa_handle_t saa_handle;
3587 sa_service_record_t *srv_recp;
3588 ibt_status_t rval;
3589 int did_something = 0;
3590
3591 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock));
3592 check_for_work:
3593 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) {
3594 if (tmp->ar_rewrite_state == IBCM_REWRITE_NEEDED) {
3595 tmp->ar_rewrite_state = IBCM_REWRITE_BUSY;
3596 saa_handle = tmp->ar_saa_handle;
3597 srv_recp = tmp->ar_srv_recp;
3598 mutex_exit(&ibcm_svc_info_lock);
3599 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_ar: "
3600 "rewriting ar @ %p", tmp);
3601 did_something = 1;
3602 rval = ibcm_write_service_record(saa_handle, srv_recp,
3603 IBMF_SAA_UPDATE);
3604 if (rval != IBT_SUCCESS)
3605 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_ar: "
3606 "ibcm_write_service_record failed: "
3607 "status = %d", rval);
3608 mutex_enter(&ibcm_svc_info_lock);
3609 /* if it got marked again, then we want to rewrite */
3610 if (tmp->ar_rewrite_state == IBCM_REWRITE_BUSY)
3611 tmp->ar_rewrite_state = IBCM_REWRITE_IDLE;
3612 /* in case there was a waiter... */
3613 cv_broadcast(&ibcm_svc_info_cv);
3614 goto check_for_work;
3615 }
3616 }
3617 return (did_something);
3618 }
3619
3620 static void
3621 ibcm_rewrite_svc_record(ibcm_svc_info_t *srv_hdl, ibcm_svc_bind_t *sbindp)
3622 {
3623 ibcm_hca_info_t *hcap;
3624 ib_svc_id_t sid, start_sid, end_sid;
3625 ibmf_saa_handle_t saa_handle;
3626 sa_service_record_t srv_rec;
3627 ibt_status_t rval;
3628
3629 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid);
3630 if (hcap == NULL) {
3631 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: "
3632 "NO HCA found for HCA GUID %llX", sbindp->sbind_hcaguid);
3633 return;
3634 }
3635
3636 saa_handle = ibcm_get_saa_handle(hcap, sbindp->sbind_port);
3637 if (saa_handle == NULL) {
3638 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: "
3639 "saa_handle is NULL");
3640 ibcm_dec_hca_acc_cnt(hcap);
3641 return;
3642 }
3643
3644 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_svc_record: "
3645 "rewriting svc '%s', port_guid = %llX", sbindp->sbind_name,
3646 sbindp->sbind_gid.gid_guid);
3647
3648 bzero(&srv_rec, sizeof (srv_rec));
3649
3650 srv_rec.ServiceLease = sbindp->sbind_lease;
3651 srv_rec.ServiceP_Key = sbindp->sbind_pkey;
3652 srv_rec.ServiceKey_hi = sbindp->sbind_key[0];
3653 srv_rec.ServiceKey_lo = sbindp->sbind_key[1];
3654 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name);
3655 srv_rec.ServiceGID = sbindp->sbind_gid;
3656
3657 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN);
3658
3659 /* insert srv record into the SA */
3660 start_sid = srv_hdl->svc_id;
3661 end_sid = start_sid + srv_hdl->svc_num_sids - 1;
3662 for (sid = start_sid; sid <= end_sid; sid++) {
3663 srv_rec.ServiceID = sid;
3664
3665 rval = ibcm_write_service_record(saa_handle, &srv_rec,
3666 IBMF_SAA_UPDATE);
3667
3668 IBTF_DPRINTF_L4(cmlog, "ibcm_rewrite_svc_record: "
3669 "ibcm_write_service_record, SvcId = %llX, "
3670 "rval = %d", (longlong_t)sid, rval);
3671 if (rval != IBT_SUCCESS) {
3672 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record:"
3673 " ibcm_write_service_record fails %d sid %llX",
3674 rval, (longlong_t)sid);
3675 }
3676 }
3677 ibcm_dec_hca_acc_cnt(hcap);
3678 }
3679
3680 /*
3681 * Task to mark all service records as needing to be rewritten to the SM/SA.
3682 * This task does not return until all of them have been rewritten.
3683 */
3684 void
3685 ibcm_service_record_rewrite_task(void *arg)
3686 {
3687 ibcm_port_up_t *pup = (ibcm_port_up_t *)arg;
3688 ib_guid_t hca_guid = pup->pup_hca_guid;
3689 uint8_t port = pup->pup_port;
3690 ibcm_svc_info_t *svcp;
3691 ibcm_svc_bind_t *sbp;
3692 avl_tree_t *avl_tree = &ibcm_svc_avl_tree;
3693 static int task_is_running = 0;
3694
3695 IBTF_DPRINTF_L3(cmlog, "ibcm_service_record_rewrite_task STARTED "
3696 "for hca_guid %llX, port %d", hca_guid, port);
3697
3698 mutex_enter(&ibcm_svc_info_lock);
3699 ibcm_mark_ar(hca_guid, port);
3700 for (svcp = avl_first(avl_tree); svcp != NULL;
3701 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) {
3702 sbp = svcp->svc_bind_list;
3703 while (sbp != NULL) {
3704 if (sbp->sbind_pkey != 0 &&
3705 sbp->sbind_port == port &&
3706 sbp->sbind_hcaguid == hca_guid) {
3707 /* even if it's busy, we mark it for rewrite */
3708 sbp->sbind_rewrite_state = IBCM_REWRITE_NEEDED;
3709 }
3710 sbp = sbp->sbind_link;
3711 }
3712 }
3713 if (task_is_running) {
3714 /* let the other task thread finish the work */
3715 mutex_exit(&ibcm_svc_info_lock);
3716 return;
3717 }
3718 task_is_running = 1;
3719
3720 (void) ibcm_rewrite_ar();
3721
3722 check_for_work:
3723 for (svcp = avl_first(avl_tree); svcp != NULL;
3724 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) {
3725 sbp = svcp->svc_bind_list;
3726 while (sbp != NULL) {
3727 if (sbp->sbind_rewrite_state == IBCM_REWRITE_NEEDED) {
3728 sbp->sbind_rewrite_state = IBCM_REWRITE_BUSY;
3729 mutex_exit(&ibcm_svc_info_lock);
3730 ibcm_rewrite_svc_record(svcp, sbp);
3731 mutex_enter(&ibcm_svc_info_lock);
3732 /* if it got marked again, we want to rewrite */
3733 if (sbp->sbind_rewrite_state ==
3734 IBCM_REWRITE_BUSY)
3735 sbp->sbind_rewrite_state =
3736 IBCM_REWRITE_IDLE;
3737 /* in case there was a waiter... */
3738 cv_broadcast(&ibcm_svc_info_cv);
3739 goto check_for_work;
3740 }
3741 sbp = sbp->sbind_link;
3742 }
3743 }
3744 /*
3745 * If there were no service records to write, and we failed to
3746 * have to rewrite any more ATS service records, then we're done.
3747 */
3748 if (ibcm_rewrite_ar() != 0)
3749 goto check_for_work;
3750 task_is_running = 0;
3751 mutex_exit(&ibcm_svc_info_lock);
3752
3753 IBTF_DPRINTF_L3(cmlog, "ibcm_service_record_rewrite_task DONE");
3754 kmem_free(pup, sizeof (ibcm_port_up_t));
3755 }
3756
3757 ibt_status_t
3758 ibt_ofuvcm_get_req_data(void *session_id, ibt_ofuvcm_req_data_t *req_data)
3759 {
3760 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id;
3761 ibcm_req_msg_t *req_msgp;
3762
3763 IBTF_DPRINTF_L3(cmlog, "ibt_get_ofuvcm_req_data: session_id %p",
3764 session_id);
3765 mutex_enter(&statep->state_mutex);
3766 if ((statep->state != IBCM_STATE_REQ_RCVD) &&
3767 (statep->state != IBCM_STATE_MRA_SENT)) {
3768 IBTF_DPRINTF_L2(cmlog, "ibt_get_ofuvcm_req_data: Invalid "
3769 "State %x", statep->state);
3770 mutex_exit(&statep->state_mutex);
3771 return (IBT_CHAN_STATE_INVALID);
3772 }
3773 if (statep->mode == IBCM_ACTIVE_MODE) {
3774 IBTF_DPRINTF_L2(cmlog, "ibt_get_ofuvcm_req_data: Active mode "
3775 "not supported");
3776 mutex_exit(&statep->state_mutex);
3777 return (IBT_INVALID_PARAM);
3778 }
3779 ASSERT(statep->req_msgp);
3780
3781 /*
3782 * Fill in the additional req message values reqired for
3783 * RTR transition.
3784 * Should the PSN be same as the active side??
3785 */
3786 req_msgp = (ibcm_req_msg_t *)statep->req_msgp;
3787 req_data->req_rnr_nak_time = ibcm_default_rnr_nak_time;
3788 req_data->req_path_mtu = req_msgp->req_mtu_plus >> 4;
3789 req_data->req_rq_psn = b2h32(req_msgp->req_starting_psn_plus) >> 8;
3790 mutex_exit(&statep->state_mutex);
3791 return (IBT_SUCCESS);
3792 }
3793
3794 ibt_status_t
3795 ibt_ofuvcm_proceed(ibt_cm_event_type_t event, void *session_id,
3796 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data,
3797 void *priv_data, ibt_priv_data_len_t priv_data_len)
3798 {
3799 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id;
3800 ibt_status_t ret;
3801
3802 IBTF_DPRINTF_L3(cmlog, "ibt_ofuvcm_proceed chan 0x%p event %x "
3803 "status %x session_id %p", statep->channel, event, status,
3804 session_id);
3805
3806 IBTF_DPRINTF_L5(cmlog, "ibt_ofuvcm_proceed chan 0x%p "
3807 "cm_event_data %p, priv_data %p priv_data_len %x",
3808 statep->channel, cm_event_data, priv_data, priv_data_len);
3809
3810 /* validate session_id and status */
3811 if ((statep == NULL) || (status == IBT_CM_DEFER)) {
3812 IBTF_DPRINTF_L2(cmlog, "ibt_ofuvcm_proceed : Invalid Args");
3813 return (IBT_INVALID_PARAM);
3814 }
3815
3816 if (event != IBT_CM_EVENT_REQ_RCV) {
3817 IBTF_DPRINTF_L2(cmlog, "ibt_ofuvcm_proceed : only for REQ_RCV");
3818 return (IBT_INVALID_PARAM);
3819 }
3820 mutex_enter(&statep->state_mutex);
3821 statep->is_this_ofuv_chan = B_TRUE;
3822 mutex_exit(&statep->state_mutex);
3823
3824 ret = ibt_cm_proceed(event, session_id, status, cm_event_data,
3825 priv_data, priv_data_len);
3826 return (ret);
3827 }
3828
3829 /*
3830 * Function:
3831 * ibt_cm_proceed
3832 *
3833 * Verifies the arguments and dispatches the cm state machine processing
3834 * via taskq
3835 */
3836
3837 ibt_status_t
3838 ibt_cm_proceed(ibt_cm_event_type_t event, void *session_id,
3839 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data,
3840 void *priv_data, ibt_priv_data_len_t priv_data_len)
3841 {
3842 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id;
3843 ibcm_proceed_targs_t *proceed_targs;
3844 ibcm_proceed_error_t proceed_error;
3845
3846 IBTF_DPRINTF_L3(cmlog, "ibt_cm_proceed chan 0x%p event %x status %x "
3847 "session_id %p", statep->channel, event, status, session_id);
3848
3849 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed chan 0x%p cm_event_data %p, "
3850 "priv_data %p priv_data_len %x", statep->channel, cm_event_data,
3851 priv_data, priv_data_len);
3852
3853 /* validate session_id and status */
3854 if ((statep == NULL) || (status == IBT_CM_DEFER)) {
3855 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : Invalid Args");
3856 return (IBT_INVALID_PARAM);
3857 }
3858
3859 /* If priv data len specified, then priv_data cannot be NULL */
3860 if ((priv_data_len > 0) && (priv_data == NULL))
3861 return (IBT_INVALID_PARAM);
3862
3863 proceed_error = IBCM_PROCEED_INVALID_NONE;
3864
3865 mutex_enter(&statep->state_mutex);
3866 if (event == IBT_CM_EVENT_REQ_RCV) {
3867
3868 if ((statep->state != IBCM_STATE_REQ_RCVD) &&
3869 (statep->state != IBCM_STATE_MRA_SENT))
3870 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE;
3871 else if (priv_data_len > IBT_REP_PRIV_DATA_SZ)
3872 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ;
3873
3874 } else if (event == IBT_CM_EVENT_REP_RCV) {
3875 if ((statep->state != IBCM_STATE_REP_RCVD) &&
3876 (statep->state != IBCM_STATE_MRA_REP_SENT))
3877 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE;
3878 else if (priv_data_len > IBT_RTU_PRIV_DATA_SZ)
3879 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ;
3880 } else if (event == IBT_CM_EVENT_LAP_RCV) {
3881 if ((statep->ap_state != IBCM_AP_STATE_LAP_RCVD) &&
3882 (statep->ap_state != IBCM_AP_STATE_MRA_LAP_SENT))
3883 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE;
3884 else if (priv_data_len > IBT_APR_PRIV_DATA_SZ)
3885 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ;
3886 } else if (event == IBT_CM_EVENT_CONN_CLOSED) {
3887 if (statep->state != IBCM_STATE_DREQ_RCVD)
3888 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE;
3889 else if (priv_data_len > IBT_DREP_PRIV_DATA_SZ)
3890 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ;
3891 } else {
3892 proceed_error = IBCM_PROCEED_INVALID_EVENT;
3893 }
3894
3895 /* if there is an error, print an error message and return */
3896 if (proceed_error != IBCM_PROCEED_INVALID_NONE) {
3897 mutex_exit(&statep->state_mutex);
3898 if (proceed_error == IBCM_PROCEED_INVALID_EVENT_STATE) {
3899 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p"
3900 "Invalid Event/State combination specified",
3901 statep->channel);
3902 return (IBT_INVALID_PARAM);
3903 } else if (proceed_error == IBCM_PROCEED_INVALID_PRIV_SZ) {
3904 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p"
3905 "Invalid Event/priv len combination specified",
3906 statep->channel);
3907 return (IBT_INVALID_PARAM);
3908 } else if (proceed_error == IBCM_PROCEED_INVALID_EVENT) {
3909 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p"
3910 "Invalid Event specified", statep->channel);
3911 return (IBT_INVALID_PARAM);
3912 } else {
3913 ASSERT(proceed_error == IBCM_PROCEED_INVALID_LAP);
3914 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p"
3915 "IBT_CM_EVENT_LAP_RCV not supported",
3916 statep->channel);
3917 /* UNTIL HCA DRIVER ENABLES AP SUPPORT, FAIL THE CALL */
3918 return (IBT_APM_NOT_SUPPORTED);
3919 }
3920 }
3921
3922
3923 /* wait until client's CM handler returns DEFER status back to CM */
3924
3925 while (statep->clnt_proceed == IBCM_BLOCK) {
3926 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed : chan 0x%p blocked for "
3927 "return of client's cm handler", statep->channel);
3928 cv_wait(&statep->block_client_cv, &statep->state_mutex);
3929 }
3930
3931 if (statep->clnt_proceed == IBCM_FAIL) {
3932 mutex_exit(&statep->state_mutex);
3933 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p Failed as "
3934 "client returned non-DEFER status from cm handler",
3935 statep->channel);
3936 return (IBT_CHAN_STATE_INVALID);
3937 }
3938
3939 ASSERT(statep->clnt_proceed == IBCM_UNBLOCK);
3940 statep->clnt_proceed = IBCM_FAIL;
3941 mutex_exit(&statep->state_mutex);
3942
3943 /* the state machine processing is done in a separate thread */
3944
3945 /* proceed_targs is freed in ibcm_proceed_via_taskq */
3946 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t),
3947 KM_SLEEP);
3948
3949 proceed_targs->event = event;
3950 proceed_targs->status = status;
3951 proceed_targs->priv_data_len = priv_data_len;
3952
3953 bcopy(priv_data, proceed_targs->priv_data, priv_data_len);
3954
3955 proceed_targs->tst.rc.statep = statep;
3956 bcopy(cm_event_data, &proceed_targs->tst.rc.rc_cm_event_data,
3957 sizeof (ibt_cm_proceed_reply_t));
3958
3959 (void) taskq_dispatch(ibcm_taskq, ibcm_proceed_via_taskq,
3960 proceed_targs, TQ_SLEEP);
3961
3962 return (IBT_SUCCESS);
3963 }
3964
3965 /*
3966 * Function:
3967 * ibcm_proceed_via_taskq
3968 *
3969 * Called from taskq, dispatched by ibt_cm_proceed
3970 * Completes the cm state processing for ibt_cm_proceed
3971 */
3972 void
3973 ibcm_proceed_via_taskq(void *targs)
3974 {
3975 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs;
3976 ibcm_state_data_t *statep = proceed_targs->tst.rc.statep;
3977 ibt_cm_reason_t reject_reason;
3978 uint8_t arej_len;
3979 ibcm_status_t response;
3980 ibcm_clnt_reply_info_t clnt_info;
3981
3982 clnt_info.reply_event = &proceed_targs->tst.rc.rc_cm_event_data;
3983 clnt_info.priv_data = proceed_targs->priv_data;
3984 clnt_info.priv_data_len = proceed_targs->priv_data_len;
3985
3986 IBTF_DPRINTF_L4(cmlog, "ibcm_proceed_via_taskq chan 0x%p targs %x",
3987 statep->channel, targs);
3988
3989 if (proceed_targs->event == IBT_CM_EVENT_REQ_RCV) {
3990 response =
3991 ibcm_process_cep_req_cm_hdlr(statep, proceed_targs->status,
3992 &clnt_info, &reject_reason, &arej_len,
3993 (ibcm_req_msg_t *)statep->defer_cm_msg);
3994
3995 ibcm_handle_cep_req_response(statep, response, reject_reason,
3996 arej_len);
3997
3998 } else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) {
3999 response =
4000 ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status,
4001 &clnt_info, &reject_reason, &arej_len,
4002 (ibcm_rep_msg_t *)statep->defer_cm_msg);
4003
4004 ibcm_handle_cep_rep_response(statep, response, reject_reason,
4005 arej_len, (ibcm_rep_msg_t *)statep->defer_cm_msg);
4006
4007 } else if (proceed_targs->event == IBT_CM_EVENT_LAP_RCV) {
4008 ibcm_process_cep_lap_cm_hdlr(statep, proceed_targs->status,
4009 &clnt_info, (ibcm_lap_msg_t *)statep->defer_cm_msg,
4010 (ibcm_apr_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg));
4011
4012 ibcm_post_apr_mad(statep);
4013
4014 } else {
4015 ASSERT(proceed_targs->event == IBT_CM_EVENT_CONN_CLOSED);
4016 ibcm_handle_cep_dreq_response(statep, proceed_targs->priv_data,
4017 proceed_targs->priv_data_len);
4018 }
4019
4020 kmem_free(targs, sizeof (ibcm_proceed_targs_t));
4021 }
4022
4023 /*
4024 * Function:
4025 * ibt_cm_ud_proceed
4026 *
4027 * Verifies the arguments and dispatches the cm state machine processing
4028 * via taskq
4029 */
4030 ibt_status_t
4031 ibt_cm_ud_proceed(void *session_id, ibt_channel_hdl_t ud_channel,
4032 ibt_cm_status_t status, ibt_redirect_info_t *redirect_infop,
4033 void *priv_data, ibt_priv_data_len_t priv_data_len)
4034 {
4035 ibcm_ud_state_data_t *ud_statep = (ibcm_ud_state_data_t *)session_id;
4036 ibcm_proceed_targs_t *proceed_targs;
4037 ibt_qp_query_attr_t qp_attr;
4038 ibt_status_t retval;
4039
4040 IBTF_DPRINTF_L3(cmlog, "ibt_cm_ud_proceed session_id %p "
4041 "ud_channel %p ", session_id, ud_channel);
4042
4043 IBTF_DPRINTF_L4(cmlog, "ibt_cm_ud_proceed status %x priv_data %p "
4044 "priv_data_len %x", status, priv_data, priv_data_len);
4045
4046 /* validate session_id and status */
4047 if ((ud_statep == NULL) || (status == IBT_CM_DEFER)) {
4048 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid Args");
4049 return (IBT_INVALID_PARAM);
4050 }
4051
4052 /* If priv data len specified, then priv_data cannot be NULL */
4053 if ((priv_data_len > 0) && (priv_data == NULL))
4054 return (IBT_INVALID_PARAM);
4055
4056 if (priv_data_len > IBT_SIDR_REP_PRIV_DATA_SZ)
4057 return (IBT_INVALID_PARAM);
4058
4059 /* retrieve qpn and qkey from ud channel */
4060
4061 /* validate event and statep's state */
4062
4063 if (status == IBT_CM_ACCEPT) {
4064 retval = ibt_query_qp(ud_channel, &qp_attr);
4065 if ((retval != IBT_SUCCESS) ||
4066 (qp_attr.qp_info.qp_trans != IBT_UD_SRV)) {
4067 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed: "
4068 "Failed to retrieve QPN from the channel: %d",
4069 retval);
4070 return (IBT_INVALID_PARAM);
4071 }
4072 }
4073
4074
4075 mutex_enter(&ud_statep->ud_state_mutex);
4076
4077 if (ud_statep->ud_state != IBCM_STATE_SIDR_REQ_RCVD) {
4078 mutex_exit(&ud_statep->ud_state_mutex);
4079 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid State "
4080 "specified");
4081 return (IBT_INVALID_PARAM);
4082 }
4083
4084 /* wait until client's CM handler returns DEFER status back to CM */
4085
4086 while (ud_statep->ud_clnt_proceed == IBCM_BLOCK) {
4087 IBTF_DPRINTF_L5(cmlog, "ibt_cm_ud_proceed : Blocked for return"
4088 " of client's ud cm handler");
4089 cv_wait(&ud_statep->ud_block_client_cv,
4090 &ud_statep->ud_state_mutex);
4091 }
4092
4093 if (ud_statep->ud_clnt_proceed == IBCM_FAIL) {
4094 mutex_exit(&ud_statep->ud_state_mutex);
4095 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Failed as client "
4096 "returned non-DEFER status from cm handler");
4097 return (IBT_INVALID_PARAM);
4098 }
4099
4100 ASSERT(ud_statep->ud_clnt_proceed == IBCM_UNBLOCK);
4101 ud_statep->ud_clnt_proceed = IBCM_FAIL;
4102 mutex_exit(&ud_statep->ud_state_mutex);
4103
4104 /* the state machine processing is done in a separate thread */
4105
4106 /* proceed_targs is freed in ibcm_proceed_via_taskq */
4107 proceed_targs = kmem_zalloc(sizeof (ibcm_proceed_targs_t),
4108 KM_SLEEP);
4109
4110 proceed_targs->status = status;
4111 proceed_targs->priv_data_len = priv_data_len;
4112
4113 bcopy(priv_data, proceed_targs->priv_data, priv_data_len);
4114
4115 if (status == IBT_CM_ACCEPT) {
4116 proceed_targs->tst.ud.ud_qkey =
4117 qp_attr.qp_info.qp_transport.ud.ud_qkey;
4118 proceed_targs->tst.ud.ud_qpn = qp_attr.qp_qpn;
4119 }
4120
4121 proceed_targs->tst.ud.ud_statep = ud_statep;
4122
4123 /* copy redirect info based on status */
4124 if (status == IBT_CM_REDIRECT)
4125 bcopy(redirect_infop, &proceed_targs->tst.ud.ud_redirect_info,
4126 sizeof (ibt_redirect_info_t));
4127
4128 (void) taskq_dispatch(ibcm_taskq, ibcm_ud_proceed_via_taskq,
4129 proceed_targs, TQ_SLEEP);
4130
4131 return (IBT_SUCCESS);
4132 }
4133
4134 /*
4135 * Function:
4136 * ibcm_ud_proceed_via_taskq
4137 *
4138 * Called from taskq, dispatched by ibt_cm_ud_proceed
4139 * Completes the cm state processing for ibt_cm_ud_proceed
4140 */
4141 void
4142 ibcm_ud_proceed_via_taskq(void *targs)
4143 {
4144 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs;
4145 ibcm_ud_state_data_t *ud_statep = proceed_targs->tst.ud.ud_statep;
4146 ibcm_ud_clnt_reply_info_t ud_clnt_info;
4147 ibt_sidr_status_t sidr_status;
4148
4149 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_proceed_via_taskq(%p)", targs);
4150
4151 ud_clnt_info.ud_qpn = proceed_targs->tst.ud.ud_qpn;
4152 ud_clnt_info.ud_qkey = proceed_targs->tst.ud.ud_qkey;
4153 ud_clnt_info.priv_data = proceed_targs->priv_data;
4154 ud_clnt_info.priv_data_len = proceed_targs->priv_data_len;
4155 ud_clnt_info.redirect_infop = &proceed_targs->tst.ud.ud_redirect_info;
4156
4157 /* validate event and statep's state */
4158 ibcm_process_sidr_req_cm_hdlr(ud_statep, proceed_targs->status,
4159 &ud_clnt_info, &sidr_status,
4160 (ibcm_sidr_rep_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg));
4161
4162 ibcm_post_sidr_rep_mad(ud_statep, sidr_status);
4163
4164 /* decr the statep ref cnt incremented in ibcm_process_sidr_req_msg */
4165 mutex_enter(&ud_statep->ud_state_mutex);
4166 IBCM_UD_REF_CNT_DECR(ud_statep);
4167 mutex_exit(&ud_statep->ud_state_mutex);
4168
4169 kmem_free(targs, sizeof (ibcm_proceed_targs_t));
4170 }
4171
4172 /*
4173 * Function:
4174 * ibt_set_alt_path
4175 * Input:
4176 * channel Channel handle returned from ibt_alloc_rc_channel(9F).
4177 *
4178 * mode Execute in blocking or non blocking mode.
4179 *
4180 * alt_path A pointer to an ibt_alt_path_info_t as returned from an
4181 * ibt_get_alt_path(9F) call that specifies the new
4182 * alternate path.
4183 *
4184 * priv_data A pointer to a buffer specified by caller for the
4185 * private data in the outgoing CM Load Alternate Path
4186 * (LAP) message sent to the remote host. This can be NULL
4187 * if no private data is available to communicate to the
4188 * remote node.
4189 *
4190 * priv_data_len Length of valid data in priv_data, this should be less
4191 * than or equal to IBT_LAP_PRIV_DATA_SZ.
4192 *
4193 * Output:
4194 * ret_args If called in blocking mode, points to a return argument
4195 * structure of type ibt_ap_returns_t.
4196 *
4197 * Returns:
4198 * IBT_SUCCESS on Success else appropriate error.
4199 * Description:
4200 * Load the specified alternate path. Causes the CM to send an LAP message
4201 * to the remote node.
4202 * Can only be called on a previously opened RC channel.
4203 */
4204 ibt_status_t
4205 ibt_set_alt_path(ibt_channel_hdl_t channel, ibt_execution_mode_t mode,
4206 ibt_alt_path_info_t *alt_path, void *priv_data,
4207 ibt_priv_data_len_t priv_data_len, ibt_ap_returns_t *ret_args)
4208 {
4209 ibmf_handle_t ibmf_hdl;
4210 ibt_status_t status = IBT_SUCCESS;
4211 ibcm_lap_msg_t *lap_msgp;
4212 ibcm_hca_info_t *hcap;
4213 ibcm_state_data_t *statep;
4214 uint8_t port_no;
4215 ib_lid_t alternate_slid;
4216 ibt_priv_data_len_t len;
4217 ib_lid_t base_lid;
4218 boolean_t alt_grh;
4219
4220 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path(%p, %x, %p, %p, %x, %p)",
4221 channel, mode, alt_path, priv_data, priv_data_len, ret_args);
4222
4223 /* validate channel */
4224 if (IBCM_INVALID_CHANNEL(channel)) {
4225 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: invalid channel");
4226 return (IBT_CHAN_HDL_INVALID);
4227 }
4228
4229 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) {
4230 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4231 "Invalid Channel type: Applicable only to RC Channel");
4232 return (IBT_CHAN_SRV_TYPE_INVALID);
4233 }
4234
4235 if (mode == IBT_NONBLOCKING) {
4236 if (ret_args != NULL) {
4237 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4238 "ret_args should be NULL when called in "
4239 "non-blocking mode");
4240 return (IBT_INVALID_PARAM);
4241 }
4242 } else if (mode == IBT_BLOCKING) {
4243 if (ret_args == NULL) {
4244 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4245 "ret_args should be Non-NULL when called in "
4246 "blocking mode");
4247 return (IBT_INVALID_PARAM);
4248 }
4249 if (ret_args->ap_priv_data_len > IBT_APR_PRIV_DATA_SZ) {
4250 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4251 "expected private data length is too large");
4252 return (IBT_INVALID_PARAM);
4253 }
4254 if ((ret_args->ap_priv_data_len > 0) &&
4255 (ret_args->ap_priv_data == NULL)) {
4256 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4257 "apr_priv_data_len > 0, but apr_priv_data NULL");
4258 return (IBT_INVALID_PARAM);
4259 }
4260 } else { /* any other mode is not valid for ibt_set_alt_path */
4261 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4262 "invalid mode %x specified", mode);
4263 return (IBT_INVALID_PARAM);
4264 }
4265
4266 if ((port_no = alt_path->ap_alt_cep_path.cep_hca_port_num) == 0)
4267 return (IBT_INVALID_PARAM);
4268
4269 /* get the statep */
4270 IBCM_GET_CHAN_PRIVATE(channel, statep);
4271 if (statep == NULL) {
4272 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: statep NULL");
4273 return (IBT_CM_FAILURE);
4274 }
4275
4276 mutex_enter(&statep->state_mutex);
4277 IBCM_RELEASE_CHAN_PRIVATE(channel);
4278 IBCM_REF_CNT_INCR(statep);
4279 mutex_exit(&statep->state_mutex);
4280
4281 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: statep %p", statep);
4282
4283 hcap = statep->hcap;
4284
4285 /* HCA must have been in active state. If not, it's a client bug */
4286 if (!IBCM_ACCESS_HCA_OK(hcap))
4287 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: hca in error state");
4288
4289 ASSERT(statep->cm_handler != NULL);
4290
4291 /* Check Alternate port */
4292 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, NULL,
4293 &base_lid);
4294 if (status != IBT_SUCCESS) {
4295 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4296 "ibt_get_port_state_byguid status %d ", status);
4297 mutex_enter(&statep->state_mutex);
4298 IBCM_REF_CNT_DECR(statep);
4299 mutex_exit(&statep->state_mutex);
4300 return (status);
4301 }
4302
4303 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) &&
4304 ((status = ibcm_hca_reinit_port(hcap, port_no - 1))
4305 != IBT_SUCCESS)) {
4306 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4307 "ibmf reg or callback setup failed during re-initialize");
4308 mutex_enter(&statep->state_mutex);
4309 IBCM_REF_CNT_DECR(statep);
4310 mutex_exit(&statep->state_mutex);
4311 return (status);
4312 }
4313
4314 ibmf_hdl = statep->stored_reply_addr.ibmf_hdl;
4315
4316 alternate_slid = base_lid +
4317 alt_path->ap_alt_cep_path.cep_adds_vect.av_src_path;
4318
4319 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: alternate SLID = %x",
4320 h2b16(alternate_slid));
4321
4322 ibcm_lapr_enter(); /* limit how many run simultaneously */
4323
4324 /* Allocate MAD for LAP */
4325 if (statep->lapr_msg == NULL)
4326 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &statep->lapr_msg,
4327 MAD_METHOD_SEND)) != IBT_SUCCESS) {
4328 ibcm_lapr_exit();
4329 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: "
4330 "chan 0x%p ibcm_alloc_out_msg failed", channel);
4331 mutex_enter(&statep->state_mutex);
4332 IBCM_REF_CNT_DECR(statep);
4333 mutex_exit(&statep->state_mutex);
4334 return (status);
4335 }
4336
4337 mutex_enter(&statep->state_mutex);
4338
4339 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: connection state is"
4340 " %x", statep->state);
4341
4342 /* Check state */
4343 if ((statep->state != IBCM_STATE_ESTABLISHED) ||
4344 (statep->ap_state != IBCM_AP_STATE_IDLE)) {
4345 IBCM_REF_CNT_DECR(statep);
4346 mutex_exit(&statep->state_mutex);
4347 (void) ibcm_free_out_msg(ibmf_hdl, &statep->lapr_msg);
4348 ibcm_lapr_exit();
4349 return (IBT_CHAN_STATE_INVALID);
4350 } else {
4351 /* Set to LAP Sent state */
4352 statep->ap_state = IBCM_AP_STATE_LAP_SENT;
4353 statep->ap_done = B_FALSE;
4354 statep->remaining_retry_cnt = statep->max_cm_retries;
4355 statep->timer_stored_state = statep->state;
4356 statep->timer_stored_ap_state = statep->ap_state;
4357 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_lap_complete */
4358 }
4359
4360 mutex_exit(&statep->state_mutex);
4361
4362 /* No more failure returns below */
4363
4364 /* Allocate MAD for LAP */
4365 IBTF_DPRINTF_L5(cmlog, "ibt_set_alt_path:"
4366 " statep's mad addr = 0x%p", IBCM_OUT_HDRP(statep->lapr_msg));
4367
4368 lap_msgp = (ibcm_lap_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg);
4369
4370 lap_msgp->lap_alt_l_port_lid = h2b16(alternate_slid);
4371 lap_msgp->lap_alt_r_port_lid =
4372 h2b16(alt_path->ap_alt_cep_path.cep_adds_vect.av_dlid);
4373
4374 /* Fill in remote port gid */
4375 lap_msgp->lap_alt_r_port_gid.gid_prefix =
4376 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_prefix);
4377 lap_msgp->lap_alt_r_port_gid.gid_guid =
4378 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_guid);
4379
4380 /* Fill in local port gid */
4381 lap_msgp->lap_alt_l_port_gid.gid_prefix =
4382 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_prefix);
4383 lap_msgp->lap_alt_l_port_gid.gid_guid =
4384 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_guid);
4385
4386 alt_grh = alt_path->ap_alt_cep_path.cep_adds_vect.av_send_grh;
4387
4388 /* alternate_flow_label, and alternate srate, alternate traffic class */
4389 lap_msgp->lap_alt_srate_plus =
4390 alt_path->ap_alt_cep_path.cep_adds_vect.av_srate & 0x3f;
4391 lap_msgp->lap_alt_flow_label_plus = h2b32(((alt_grh == B_TRUE) ?
4392 (alt_path->ap_alt_cep_path.cep_adds_vect.av_flow << 12) : 0) |
4393 alt_path->ap_alt_cep_path.cep_adds_vect.av_tclass);
4394
4395 /* Alternate hop limit, service level */
4396 lap_msgp->lap_alt_hop_limit = (alt_grh == B_TRUE) ?
4397 alt_path->ap_alt_cep_path.cep_adds_vect.av_hop : 1;
4398 lap_msgp->lap_alt_sl_plus =
4399 alt_path->ap_alt_cep_path.cep_adds_vect.av_srvl << 4 |
4400 ((alt_grh == B_FALSE) ? 0x8 : 0);
4401
4402 lap_msgp->lap_alt_local_acktime_plus = ibt_usec2ib(
4403 (2 * statep->rc_alt_pkt_lt) +
4404 ibt_ib2usec(hcap->hca_ack_delay)) << 3;
4405
4406 lap_msgp->lap_local_comm_id = h2b32(statep->local_comid);
4407 lap_msgp->lap_remote_comm_id = h2b32(statep->remote_comid);
4408
4409 lap_msgp->lap_remote_qpn_eecn_plus =
4410 h2b32((statep->remote_qpn << 8) |
4411 ibt_usec2ib(ibcm_remote_response_time) << 3);
4412
4413 len = min(priv_data_len, IBT_LAP_PRIV_DATA_SZ);
4414 if ((len > 0) && priv_data) {
4415 bcopy(priv_data, lap_msgp->lap_private_data, len);
4416 }
4417
4418 /* only rc_alt_pkt_lt and ap_return_data fields are initialized */
4419 statep->rc_alt_pkt_lt = ibt_ib2usec(alt_path->ap_alt_pkt_lt);
4420
4421 /* return_data is filled up in the state machine code */
4422 statep->ap_return_data = ret_args;
4423
4424 IBCM_OUT_HDRP(statep->lapr_msg)->AttributeID =
4425 h2b16(IBCM_INCOMING_LAP + IBCM_ATTR_BASE_ID);
4426
4427 IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID =
4428 h2b64(ibcm_generate_tranid(IBCM_INCOMING_LAP, statep->local_comid,
4429 0));
4430 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path: statep %p, tid %llx",
4431 statep, IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID);
4432
4433 /* Send LAP */
4434 ibcm_post_rc_mad(statep, statep->lapr_msg, ibcm_post_lap_complete,
4435 statep);
4436
4437 mutex_enter(&statep->state_mutex);
4438
4439 if (mode == IBT_BLOCKING) {
4440 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: blocking");
4441
4442 /* wait for APR */
4443 while (statep->ap_done != B_TRUE) {
4444 cv_wait(&statep->block_client_cv,
4445 &statep->state_mutex);
4446 }
4447
4448 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done blocking");
4449
4450 /*
4451 * In the case that ibt_set_alt_path fails,
4452 * change retval to IBT_CM_FAILURE
4453 */
4454 if (statep->ap_return_data->ap_status != IBT_CM_AP_LOADED)
4455 status = IBT_CM_FAILURE;
4456
4457 }
4458
4459 /* decrement the ref-count before leaving here */
4460 IBCM_REF_CNT_DECR(statep);
4461
4462 mutex_exit(&statep->state_mutex);
4463
4464 ibcm_lapr_exit();
4465
4466 /* If this message isn't seen then ibt_set_alt_path failed */
4467 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done");
4468
4469 return (status);
4470 }
4471
4472
4473 #ifdef DEBUG
4474
4475 /*
4476 * ibcm_query_classport_info:
4477 * Query classportinfo
4478 *
4479 * INPUTS:
4480 * channel - Channel that is associated with a statep
4481 *
4482 * RETURN VALUE: NONE
4483 * This function is currently used to generate a valid get method classport
4484 * info, and test CM functionality. There is no ibtl client interface to
4485 * generate a classportinfo. It is possible that CM may use classportinfo
4486 * from other nodes in the future, and most of the code below could be re-used.
4487 */
4488 void
4489 ibcm_query_classport_info(ibt_channel_hdl_t channel)
4490 {
4491 ibcm_state_data_t *statep;
4492 ibmf_msg_t *msgp;
4493
4494 IBTF_DPRINTF_L3(cmlog, "ibcm_query_classport_info(%p)", channel);
4495
4496 /* validate channel, first */
4497 if (IBCM_INVALID_CHANNEL(channel)) {
4498 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: "
4499 "invalid channel (%p)", channel);
4500 return;
4501 }
4502
4503 /* get the statep */
4504 IBCM_GET_CHAN_PRIVATE(channel, statep);
4505
4506 /*
4507 * This can happen, if the statep is already gone by a DREQ from
4508 * the remote side
4509 */
4510 if (statep == NULL) {
4511 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: "
4512 "statep NULL");
4513 return;
4514 }
4515
4516 mutex_enter(&statep->state_mutex);
4517 IBCM_RELEASE_CHAN_PRIVATE(channel);
4518 IBCM_REF_CNT_INCR(statep);
4519 mutex_exit(&statep->state_mutex);
4520
4521 /* Debug/test code, so don't care about return status */
4522 (void) ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp,
4523 MAD_METHOD_GET);
4524
4525 IBCM_OUT_HDRP(msgp)->TransactionID = h2b64(ibcm_generate_tranid(
4526 MAD_ATTR_ID_CLASSPORTINFO, statep->local_comid, 0));
4527 IBCM_OUT_HDRP(msgp)->AttributeID = h2b16(MAD_ATTR_ID_CLASSPORTINFO);
4528
4529 (void) ibcm_post_mad(msgp, &statep->stored_reply_addr, NULL, NULL);
4530
4531 IBTF_DPRINTF_L3(cmlog, "ibcm_query_classport_info(%p) "
4532 "Get method MAD posted ", channel);
4533
4534 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp);
4535
4536 mutex_enter(&statep->state_mutex);
4537 IBCM_REF_CNT_DECR(statep);
4538 mutex_exit(&statep->state_mutex);
4539 }
4540
4541 static void
4542 ibcm_print_reply_addr(ibt_channel_hdl_t channel, ibcm_mad_addr_t *cm_reply_addr)
4543 {
4544 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: chan 0x%p, SLID %x, "
4545 "DLID %x", channel, cm_reply_addr->rcvd_addr.ia_local_lid,
4546 cm_reply_addr->rcvd_addr.ia_remote_lid);
4547
4548 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: QKEY %x, PKEY %x, "
4549 "RQPN %x SL %x", cm_reply_addr->rcvd_addr.ia_q_key,
4550 cm_reply_addr->rcvd_addr.ia_p_key,
4551 cm_reply_addr->rcvd_addr.ia_remote_qno,
4552 cm_reply_addr->rcvd_addr.ia_service_level);
4553
4554 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM SGID %llX:%llX ",
4555 cm_reply_addr->grh_hdr.ig_sender_gid.gid_prefix,
4556 cm_reply_addr->grh_hdr.ig_sender_gid.gid_guid);
4557
4558 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM DGID %llX:%llX",
4559 cm_reply_addr->grh_hdr.ig_recver_gid.gid_prefix,
4560 cm_reply_addr->grh_hdr.ig_recver_gid.gid_guid);
4561
4562 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM FL %x TC %x HL %x",
4563 cm_reply_addr->grh_hdr.ig_flow_label,
4564 cm_reply_addr->grh_hdr.ig_tclass,
4565 cm_reply_addr->grh_hdr.ig_hop_limit);
4566 }
4567
4568 #endif
4569
4570 /* For MCG List search */
4571 typedef struct ibcm_mcg_list_s {
4572 struct ibcm_mcg_list_s *ml_next;
4573 ib_gid_t ml_sgid;
4574 ib_gid_t ml_mgid;
4575 ib_pkey_t ml_pkey;
4576 ib_qkey_t ml_qkey;
4577 uint_t ml_refcnt;
4578 uint8_t ml_jstate;
4579 } ibcm_mcg_list_t;
4580
4581 ibcm_mcg_list_t *ibcm_mcglist = NULL;
4582
4583 typedef struct ibcm_join_mcg_tqarg_s {
4584 ib_gid_t rgid;
4585 ibt_mcg_attr_t mcg_attr;
4586 ibt_mcg_info_t *mcg_infop;
4587 ibt_mcg_handler_t func;
4588 void *arg;
4589 } ibcm_join_mcg_tqarg_t;
4590
4591 void
4592 ibcm_add_incr_mcg_entry(sa_mcmember_record_t *mcg_req,
4593 sa_mcmember_record_t *mcg_resp)
4594 {
4595 ibcm_mcg_list_t *new = NULL;
4596 ibcm_mcg_list_t *head = NULL;
4597
4598 IBTF_DPRINTF_L3(cmlog, "ibcm_add_incr_mcg_entry: MGID %llX:%llX"
4599 "\n SGID %llX:%llX, JState %X)", mcg_req->MGID.gid_prefix,
4600 mcg_req->MGID.gid_guid, mcg_req->PortGID.gid_prefix,
4601 mcg_req->PortGID.gid_guid, mcg_req->JoinState);
4602
4603 mutex_enter(&ibcm_mcglist_lock);
4604 head = ibcm_mcglist;
4605
4606 while (head != NULL) {
4607 if ((head->ml_mgid.gid_guid == mcg_resp->MGID.gid_guid) &&
4608 (head->ml_mgid.gid_prefix == mcg_resp->MGID.gid_prefix) &&
4609 (head->ml_sgid.gid_guid == mcg_resp->PortGID.gid_guid)) {
4610 /* Increment the count */
4611 head->ml_refcnt++;
4612 /* OR the join_state value, we need this during leave */
4613 head->ml_jstate |= mcg_req->JoinState;
4614
4615 IBTF_DPRINTF_L3(cmlog, "ibcm_add_incr_mcg_entry: Entry "
4616 "FOUND: refcnt %d JState %X", head->ml_refcnt,
4617 head->ml_jstate);
4618
4619 mutex_exit(&ibcm_mcglist_lock);
4620 return;
4621 }
4622 head = head->ml_next;
4623 }
4624 mutex_exit(&ibcm_mcglist_lock);
4625
4626 IBTF_DPRINTF_L3(cmlog, "ibcm_add_incr_mcg_entry: Create NEW Entry ");
4627
4628 /* If we are here, either list is empty or match couldn't be found */
4629 new = kmem_zalloc(sizeof (ibcm_mcg_list_t), KM_SLEEP);
4630
4631 mutex_enter(&ibcm_mcglist_lock);
4632 /* Initialize the fields */
4633 new->ml_sgid = mcg_resp->PortGID;
4634 new->ml_mgid = mcg_resp->MGID;
4635 new->ml_qkey = mcg_req->Q_Key;
4636 new->ml_pkey = mcg_req->P_Key;
4637 new->ml_refcnt = 1; /* As this is the first entry */
4638 new->ml_jstate = mcg_req->JoinState;
4639 new->ml_next = NULL;
4640
4641 new->ml_next = ibcm_mcglist;
4642 ibcm_mcglist = new;
4643 mutex_exit(&ibcm_mcglist_lock);
4644 }
4645
4646 /*
4647 * ibcm_del_decr_mcg_entry
4648 *
4649 * Return value:
4650 * IBCM_SUCCESS Entry found and ref_cnt is now zero. So go-ahead and
4651 * leave the MCG group. The return arg *jstate will have
4652 * a valid join_state value that needed to be used by
4653 * xxx_leave_mcg().
4654 * IBCM_LOOKUP_EXISTS Entry found and ref_cnt is decremented but is NOT zero.
4655 * So do not leave the MCG group yet.
4656 * IBCM_LOOKUP_FAIL Entry is NOT found.
4657 */
4658 ibcm_status_t
4659 ibcm_del_decr_mcg_entry(sa_mcmember_record_t *mcg_req, uint8_t *jstate)
4660 {
4661 ibcm_mcg_list_t *head, *prev;
4662
4663 IBTF_DPRINTF_L3(cmlog, "ibcm_del_decr_mcg_entry: MGID %llX:%llX"
4664 "\n SGID %llX:%llX, JState %X)", mcg_req->MGID.gid_prefix,
4665 mcg_req->MGID.gid_guid, mcg_req->PortGID.gid_prefix,
4666 mcg_req->PortGID.gid_guid, mcg_req->JoinState);
4667
4668 *jstate = 0;
4669
4670 mutex_enter(&ibcm_mcglist_lock);
4671 head = ibcm_mcglist;
4672 prev = NULL;
4673
4674 while (head != NULL) {
4675 if ((head->ml_mgid.gid_guid == mcg_req->MGID.gid_guid) &&
4676 (head->ml_mgid.gid_prefix == mcg_req->MGID.gid_prefix) &&
4677 (head->ml_sgid.gid_guid == mcg_req->PortGID.gid_guid)) {
4678 if (!(head->ml_jstate & mcg_req->JoinState)) {
4679 IBTF_DPRINTF_L2(cmlog, "ibcm_del_decr_mcg_entry"
4680 ": JoinState mismatch %X %X)",
4681 head->ml_jstate, mcg_req->JoinState);
4682 }
4683 /* Decrement the count */
4684 head->ml_refcnt--;
4685
4686 if (head->ml_refcnt == 0) {
4687 *jstate = head->ml_jstate;
4688
4689 IBTF_DPRINTF_L3(cmlog, "ibcm_del_decr_mcg_entry"
4690 ": refcnt is ZERO, so delete the entry ");
4691 if ((head == ibcm_mcglist) || (prev == NULL)) {
4692 ibcm_mcglist = head->ml_next;
4693 } else if (prev != NULL) {
4694 prev->ml_next = head->ml_next;
4695 }
4696 mutex_exit(&ibcm_mcglist_lock);
4697
4698 kmem_free(head, sizeof (ibcm_mcg_list_t));
4699 return (IBCM_SUCCESS);
4700 }
4701 mutex_exit(&ibcm_mcglist_lock);
4702 return (IBCM_LOOKUP_EXISTS);
4703 }
4704 prev = head;
4705 head = head->ml_next;
4706 }
4707 mutex_exit(&ibcm_mcglist_lock);
4708
4709 /*
4710 * If we are here, something went wrong, we don't have the entry
4711 * for that MCG being joined.
4712 */
4713 IBTF_DPRINTF_L2(cmlog, "ibcm_del_decr_mcg_entry: Match NOT "
4714 "Found ");
4715
4716 return (IBCM_LOOKUP_FAIL);
4717 }
4718
4719
4720 /*
4721 * Function:
4722 * ibt_join_mcg
4723 * Input:
4724 * rgid The request GID that defines the HCA port from which a
4725 * contact to SA Access is performed to add the specified
4726 * endport GID ((mcg_attr->mc_pgid) to a multicast group.
4727 * If mcg_attr->mc_pgid is null, then this (rgid) will be
4728 * treated as endport GID that is to be added to the
4729 * multicast group.
4730 *
4731 * mcg_attr A pointer to an ibt_mcg_attr_t structure that defines
4732 * the attributes of the desired multicast group to be
4733 * created or joined.
4734 *
4735 * func NULL or a pointer to a function to call when
4736 * ibt_join_mcg() completes. If 'func' is not NULL then
4737 * ibt_join_mcg() will return as soon as possible after
4738 * initiating the multicast group join/create process.
4739 * 'func' is then called when the process completes.
4740 *
4741 * arg Argument to the 'func'.
4742 *
4743 * Output:
4744 * mcg_info_p A pointer to the ibt_mcg_info_t structure, allocated
4745 * by the caller, where the attributes of the created or
4746 * joined multicast group are copied.
4747 * Returns:
4748 * IBT_SUCCESS
4749 * IBT_INVALID_PARAM
4750 * IBT_MCG_RECORDS_NOT_FOUND
4751 * IBT_INSUFF_RESOURCE
4752 * Description:
4753 * Join a multicast group. The first full member "join" causes the MCG
4754 * to be created.
4755 */
4756 ibt_status_t
4757 ibt_join_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr,
4758 ibt_mcg_info_t *mcg_info_p, ibt_mcg_handler_t func, void *arg)
4759 {
4760 ibcm_join_mcg_tqarg_t *mcg_tq;
4761 int flag = ((func == NULL) ? KM_SLEEP : KM_NOSLEEP);
4762
4763 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg(%llX:%llX, %p)", rgid.gid_prefix,
4764 rgid.gid_guid, mcg_attr);
4765
4766 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) {
4767 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Request GID is required");
4768 return (IBT_INVALID_PARAM);
4769 }
4770
4771 if ((mcg_attr->mc_pkey == IB_PKEY_INVALID_LIMITED) ||
4772 (mcg_attr->mc_pkey == IB_PKEY_INVALID_FULL)) {
4773 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Invalid P_Key specified");
4774 return (IBT_INVALID_PARAM);
4775 }
4776
4777 if (mcg_attr->mc_join_state == 0) {
4778 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: JoinState not specified");
4779 return (IBT_INVALID_PARAM);
4780 }
4781
4782 if (mcg_info_p == NULL) {
4783 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: mcg_info_p is NULL");
4784 return (IBT_INVALID_PARAM);
4785 }
4786
4787 mcg_tq = kmem_alloc(sizeof (ibcm_join_mcg_tqarg_t), flag);
4788 if (mcg_tq == NULL) {
4789 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: "
4790 "Unable to allocate memory for local usage.");
4791 return (IBT_INSUFF_KERNEL_RESOURCE);
4792 }
4793
4794 mcg_tq->rgid = rgid;
4795 bcopy(mcg_attr, &mcg_tq->mcg_attr, sizeof (ibt_mcg_attr_t));
4796 mcg_tq->mcg_infop = mcg_info_p;
4797 mcg_tq->func = func;
4798 mcg_tq->arg = arg;
4799
4800 if (func != NULL) { /* Non-Blocking */
4801 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg: Non-Blocking Call");
4802 if (taskq_dispatch(ibcm_taskq, ibcm_process_async_join_mcg,
4803 mcg_tq, TQ_NOSLEEP) == 0) {
4804 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Failed to "
4805 "Dispatch the TaskQ");
4806 kmem_free(mcg_tq, sizeof (ibcm_join_mcg_tqarg_t));
4807 return (IBT_INSUFF_KERNEL_RESOURCE);
4808 } else
4809 return (IBT_SUCCESS);
4810 } else { /* Blocking */
4811 return (ibcm_process_join_mcg(mcg_tq));
4812 }
4813 }
4814
4815 static void
4816 ibcm_process_async_join_mcg(void *tq_arg)
4817 {
4818 (void) ibcm_process_join_mcg(tq_arg);
4819 }
4820
4821 static ibt_status_t
4822 ibcm_process_join_mcg(void *taskq_arg)
4823 {
4824 sa_mcmember_record_t mcg_req;
4825 sa_mcmember_record_t *mcg_resp;
4826 ibmf_saa_access_args_t access_args;
4827 ibmf_saa_handle_t saa_handle;
4828 uint64_t component_mask = 0;
4829 ibt_status_t retval;
4830 ibtl_cm_hca_port_t hca_port;
4831 uint_t num_records;
4832 size_t length;
4833 ibcm_hca_info_t *hcap;
4834 ibcm_join_mcg_tqarg_t *mcg_arg = (ibcm_join_mcg_tqarg_t *)taskq_arg;
4835 ibt_mcg_info_t *mcg_info_p = mcg_arg->mcg_infop;
4836
4837 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg(%p)", mcg_arg);
4838
4839 retval = ibtl_cm_get_hca_port(mcg_arg->rgid, 0, &hca_port);
4840 if (retval != IBT_SUCCESS) {
4841 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed to get "
4842 "port info from specified RGID: status = %d", retval);
4843 goto ibcm_join_mcg_exit1;
4844 }
4845
4846 bzero(&mcg_req, sizeof (sa_mcmember_record_t));
4847
4848 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix == 0) ||
4849 (mcg_arg->mcg_attr.mc_pgid.gid_guid == 0)) {
4850 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: "
4851 "Request GID is Port GID");
4852 mcg_req.PortGID = mcg_arg->rgid;
4853 } else {
4854 mcg_req.PortGID = mcg_arg->mcg_attr.mc_pgid;
4855 }
4856 component_mask |= SA_MC_COMPMASK_PORTGID;
4857
4858 mcg_req.Q_Key = mcg_arg->mcg_attr.mc_qkey;
4859 mcg_req.P_Key = mcg_arg->mcg_attr.mc_pkey;
4860 mcg_req.JoinState = mcg_arg->mcg_attr.mc_join_state;
4861 mcg_req.TClass = mcg_arg->mcg_attr.mc_tclass;
4862 mcg_req.FlowLabel = mcg_arg->mcg_attr.mc_flow;
4863 mcg_req.SL = mcg_arg->mcg_attr.mc_sl;
4864
4865 component_mask |= SA_MC_COMPMASK_QKEY | SA_MC_COMPMASK_PKEY |
4866 SA_MC_COMPMASK_JOINSTATE | SA_MC_COMPMASK_TCLASS |
4867 SA_MC_COMPMASK_FLOWLABEL | SA_MC_COMPMASK_SL;
4868
4869 /* If client has specified MGID, use it else SA will assign one. */
4870 if ((mcg_arg->mcg_attr.mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) {
4871 mcg_req.MGID = mcg_arg->mcg_attr.mc_mgid;
4872 component_mask |= SA_MC_COMPMASK_MGID;
4873 }
4874
4875 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: ");
4876 IBTF_DPRINTF_L3(cmlog, "PGID=%016llX:%016llX, ",
4877 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid);
4878 IBTF_DPRINTF_L3(cmlog, "MGID=%016llX:%016llX",
4879 mcg_req.MGID.gid_prefix, mcg_req.MGID.gid_guid);
4880 IBTF_DPRINTF_L3(cmlog, "JoinState = %X",
4881 mcg_arg->mcg_attr.mc_join_state);
4882 IBTF_DPRINTF_L5(cmlog, "QKey %lX, PKey %lX",
4883 mcg_arg->mcg_attr.mc_qkey, mcg_arg->mcg_attr.mc_pkey);
4884 IBTF_DPRINTF_L5(cmlog, "Scope %X, MLID %X",
4885 mcg_arg->mcg_attr.mc_scope, mcg_arg->mcg_attr.mc_mlid);
4886
4887 /* Is MTU specified. */
4888 if (mcg_arg->mcg_attr.mc_mtu_req.r_mtu) {
4889 mcg_req.MTU = mcg_arg->mcg_attr.mc_mtu_req.r_mtu;
4890 mcg_req.MTUSelector = mcg_arg->mcg_attr.mc_mtu_req.r_selector;
4891
4892 component_mask |= SA_MC_COMPMASK_MTUSELECTOR |
4893 SA_MC_COMPMASK_MTU;
4894 }
4895
4896 /* Is RATE specified. */
4897 if (mcg_arg->mcg_attr.mc_rate_req.r_srate) {
4898 mcg_req.Rate = mcg_arg->mcg_attr.mc_rate_req.r_srate;
4899 mcg_req.RateSelector =
4900 mcg_arg->mcg_attr.mc_rate_req.r_selector;
4901
4902 component_mask |= SA_MC_COMPMASK_RATESELECTOR |
4903 SA_MC_COMPMASK_RATE;
4904 }
4905
4906 /* Is Packet Life Time specified. */
4907 if (mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt) {
4908 mcg_req.Rate = mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt;
4909 mcg_req.RateSelector =
4910 mcg_arg->mcg_attr.mc_pkt_lt_req.p_selector;
4911
4912 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR |
4913 SA_MC_COMPMASK_PKTLT;
4914 }
4915
4916 if (mcg_arg->mcg_attr.mc_hop) {
4917 mcg_req.HopLimit = mcg_arg->mcg_attr.mc_hop;
4918 component_mask |= SA_MC_COMPMASK_HOPLIMIT;
4919 }
4920
4921 if (mcg_arg->mcg_attr.mc_scope) {
4922 mcg_req.Scope = mcg_arg->mcg_attr.mc_scope;
4923 component_mask |= SA_MC_COMPMASK_SCOPE;
4924 }
4925
4926 if (mcg_arg->mcg_attr.mc_mlid) {
4927 mcg_req.MLID = mcg_arg->mcg_attr.mc_mlid;
4928 component_mask |= SA_MC_COMPMASK_MLID;
4929 }
4930
4931 /* Get SA Access Handle. */
4932 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid);
4933 if (hcap == NULL) {
4934 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: NO HCA found");
4935
4936 retval = IBT_HCA_BUSY_DETACHING;
4937 goto ibcm_join_mcg_exit1;
4938 }
4939
4940 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port);
4941 if (saa_handle == NULL) {
4942 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: SA Handle NULL");
4943
4944 retval = IBT_HCA_PORT_NOT_ACTIVE;
4945 goto ibcm_join_mcg_exit;
4946 }
4947
4948 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix != 0) &&
4949 (mcg_arg->mcg_attr.mc_pgid.gid_guid != 0)) {
4950 retval = ibtl_cm_get_hca_port(mcg_arg->mcg_attr.mc_pgid, 0,
4951 &hca_port);
4952 if (retval != IBT_SUCCESS) {
4953 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed "
4954 "to get PortInfo of specified PGID: status = %d",
4955 retval);
4956 goto ibcm_join_mcg_exit1;
4957 }
4958 }
4959
4960 /* Contact SA Access */
4961 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID;
4962 access_args.sq_access_type = IBMF_SAA_UPDATE;
4963 access_args.sq_component_mask = component_mask;
4964 access_args.sq_template = &mcg_req;
4965 access_args.sq_template_length = sizeof (sa_mcmember_record_t);
4966 access_args.sq_callback = NULL;
4967 access_args.sq_callback_arg = NULL;
4968
4969 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length,
4970 (void **)&mcg_resp);
4971 if (retval != IBT_SUCCESS) {
4972 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: "
4973 "SA Access Failed");
4974 goto ibcm_join_mcg_exit;
4975 }
4976
4977 num_records = length/sizeof (sa_mcmember_record_t);
4978
4979 IBTF_DPRINTF_L4(cmlog, "ibcm_process_join_mcg: "
4980 "Found %d MCMember Records", num_records);
4981
4982 /* Validate the returned number of records. */
4983 if ((mcg_resp != NULL) && (num_records > 0)) {
4984 /* Update the return values. */
4985 mcg_info_p->mc_adds_vect.av_dgid = mcg_resp->MGID;
4986 mcg_info_p->mc_adds_vect.av_sgid = mcg_resp->PortGID;
4987 mcg_info_p->mc_adds_vect.av_srate = mcg_resp->Rate;
4988 mcg_info_p->mc_adds_vect.av_srvl = mcg_resp->SL;
4989 mcg_info_p->mc_adds_vect.av_flow = mcg_resp->FlowLabel;
4990 mcg_info_p->mc_adds_vect.av_tclass = mcg_resp->TClass;
4991 mcg_info_p->mc_adds_vect.av_hop = mcg_resp->HopLimit;
4992 mcg_info_p->mc_adds_vect.av_send_grh = B_TRUE;
4993 mcg_info_p->mc_adds_vect.av_dlid = mcg_resp->MLID;
4994 mcg_info_p->mc_mtu = mcg_resp->MTU;
4995 mcg_info_p->mc_qkey = mcg_resp->Q_Key;
4996
4997 retval = ibt_pkey2index_byguid(hca_port.hp_hca_guid,
4998 hca_port.hp_port, mcg_resp->P_Key, &mcg_info_p->mc_pkey_ix);
4999 if (retval != IBT_SUCCESS) {
5000 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: "
5001 "Pkey2Index Conversion failed<%d>", retval);
5002 mcg_info_p->mc_pkey_ix = 0;
5003 }
5004
5005 mcg_info_p->mc_scope = mcg_resp->Scope;
5006 mcg_info_p->mc_pkt_lt = mcg_resp->PacketLifeTime;
5007
5008 mcg_info_p->mc_adds_vect.av_port_num = hca_port.hp_port;
5009 mcg_info_p->mc_adds_vect.av_sgid_ix = hca_port.hp_sgid_ix;
5010 mcg_info_p->mc_adds_vect.av_src_path = 0;
5011
5012 /* Add or Incr the matching MCG entry. */
5013 ibcm_add_incr_mcg_entry(&mcg_req, mcg_resp);
5014 /* Deallocate the memory allocated by SA for mcg_resp. */
5015 kmem_free(mcg_resp, length);
5016
5017 retval = IBT_SUCCESS;
5018 } else {
5019 retval = IBT_MCG_RECORDS_NOT_FOUND;
5020 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: "
5021 "MCG RECORDS NOT FOUND");
5022 }
5023
5024 ibcm_join_mcg_exit:
5025 ibcm_dec_hca_acc_cnt(hcap);
5026
5027 ibcm_join_mcg_exit1:
5028 if (mcg_arg->func)
5029 (*(mcg_arg->func))(mcg_arg->arg, retval, mcg_info_p);
5030
5031 kmem_free(mcg_arg, sizeof (ibcm_join_mcg_tqarg_t));
5032
5033 return (retval);
5034 }
5035
5036
5037 /*
5038 * Function:
5039 * ibt_leave_mcg
5040 * Input:
5041 * rgid The request GID that defines the HCA port upon which
5042 * to send the request to the Subnet Administrator, to
5043 * remove the specified port (port_gid) from the multicast
5044 * group. If 'port_gid' is the Reserved GID (i.e.
5045 * port_gid.gid_prefix = 0 and port_gid.gid_guid = 0),
5046 * then the end-port associated with 'rgid' is removed
5047 * from the multicast group.
5048 *
5049 * mc_gid A multicast group GID as returned from ibt_join_mcg()
5050 * call. This is optional, if not specified (i.e.
5051 * mc_gid.gid_prefix has 0xFF in its upper 8 bits to
5052 * identify this as being a multicast GID), then the
5053 * port is removed from all the multicast groups of
5054 * which it is a member.
5055 *
5056 * port_gid This is optional, if not the Reserved GID (gid_prefix
5057 * and gid_guid not equal to 0), then this specifies the
5058 * endport GID of the multicast group member being deleted
5059 * from the group. If it is the Reserved GID (gid_prefix
5060 * and gid_guid equal to 0) then the member endport GID is
5061 * determined from 'rgid'.
5062 *
5063 * mc_join_state The Join State attribute used when the group was joined
5064 * using ibt_join_mcg(). This Join State component must
5065 * contains at least one bit set to 1 in the same position
5066 * as that used during ibt_join_mcg(). i.e. the logical
5067 * AND of the two JoinState components is not all zeros.
5068 * This Join State component must not have some bits set
5069 * which are not set using ibt_join_mcg().
5070 * Output:
5071 * None.
5072 * Returns:
5073 * IBT_SUCCESS
5074 * IBT_INVALID_PARAM
5075 * IBT_MC_GROUP_INVALID
5076 * IBT_INSUFF_RESOURCE
5077 * Description:
5078 * The port associated with the port GID shall be removed from the
5079 * multicast group specified by MGID (mc_gid) or from all the multicast
5080 * groups of which it is a member if the MGID (mc_gid) is not specified.
5081 *
5082 * The last full member to leave causes the destruction of the Multicast
5083 * Group.
5084 */
5085 ibt_status_t
5086 ibt_leave_mcg(ib_gid_t rgid, ib_gid_t mc_gid, ib_gid_t port_gid,
5087 uint8_t mc_join_state)
5088 {
5089 sa_mcmember_record_t mcg_req;
5090 ibmf_saa_access_args_t access_args;
5091 ibmf_saa_handle_t saa_handle;
5092 uint64_t component_mask = 0;
5093 int sa_retval;
5094 ibt_status_t retval;
5095 ibcm_status_t ret;
5096 ibtl_cm_hca_port_t hca_port;
5097 size_t length;
5098 void *results_p;
5099 ibcm_hca_info_t *hcap;
5100 uint8_t jstate = 0;
5101
5102 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, %llX:%llX)",
5103 rgid.gid_prefix, rgid.gid_guid, mc_gid.gid_prefix, mc_gid.gid_guid);
5104
5105 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, 0x%X)",
5106 port_gid.gid_prefix, port_gid.gid_guid, mc_join_state);
5107
5108 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) {
5109 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: RequestGID is required");
5110 return (IBT_INVALID_PARAM);
5111 }
5112
5113 bzero(&mcg_req, sizeof (sa_mcmember_record_t));
5114
5115 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: MGID: %llX%llX",
5116 mc_gid.gid_prefix, mc_gid.gid_guid);
5117
5118 /* Validate MGID */
5119 if ((mc_gid.gid_prefix >> 56ULL & 0xFF) == 0xFF) {
5120 mcg_req.MGID = mc_gid;
5121 component_mask |= SA_MC_COMPMASK_MGID;
5122 } else if ((mc_gid.gid_prefix != 0) || (mc_gid.gid_guid != 0)) {
5123 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Invalid MGID specified");
5124 return (IBT_MC_MGID_INVALID);
5125 }
5126
5127 if ((port_gid.gid_prefix == 0) || (port_gid.gid_guid == 0)) {
5128 mcg_req.PortGID = rgid;
5129 } else {
5130 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Performing PROXY Leave");
5131 mcg_req.PortGID = port_gid;
5132 }
5133 component_mask |= SA_MC_COMPMASK_PORTGID;
5134
5135 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Port GID <%llX:%llX>",
5136 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid);
5137
5138 /* Join State */
5139 mcg_req.JoinState = mc_join_state;
5140 component_mask |= SA_MC_COMPMASK_JOINSTATE;
5141
5142 ret = ibcm_del_decr_mcg_entry(&mcg_req, &jstate);
5143 if (ret == IBCM_LOOKUP_EXISTS) {
5144 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Multiple JoinMCG record "
5145 " still exists, we shall leave for last leave_mcg call");
5146 return (IBT_SUCCESS);
5147 } else if (ret == IBCM_LOOKUP_FAIL) {
5148 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: No Record found, "
5149 "continue with leave_mcg call");
5150 } else if ((ret == IBCM_SUCCESS) && (jstate != 0)) {
5151 /*
5152 * Update with cached "jstate", as this will be OR'ed of
5153 * all ibt_join_mcg() calls for this record.
5154 */
5155 mcg_req.JoinState = jstate;
5156 }
5157
5158 retval = ibtl_cm_get_hca_port(rgid, 0, &hca_port);
5159 if (retval != IBT_SUCCESS) {
5160 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: Failed to get port info "
5161 "from specified RGID : status = %d", retval);
5162 return (retval);
5163 }
5164
5165 /* Get SA Access Handle. */
5166 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid);
5167 if (hcap == NULL) {
5168 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: "
5169 "NO HCA found");
5170 return (IBT_HCA_BUSY_DETACHING);
5171 }
5172
5173 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port);
5174 if (saa_handle == NULL) {
5175 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: saa_handle is NULL");
5176 ibcm_dec_hca_acc_cnt(hcap);
5177 return (IBT_HCA_PORT_NOT_ACTIVE);
5178 }
5179
5180 /* Contact SA Access */
5181 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID;
5182 access_args.sq_access_type = IBMF_SAA_DELETE;
5183 access_args.sq_component_mask = component_mask;
5184 access_args.sq_template = &mcg_req;
5185 access_args.sq_template_length = sizeof (sa_mcmember_record_t);
5186 access_args.sq_callback = NULL;
5187 access_args.sq_callback_arg = NULL;
5188
5189 ibcm_sa_access_enter();
5190
5191 sa_retval = ibmf_sa_access(saa_handle, &access_args, 0, &length,
5192 &results_p);
5193 if (sa_retval != IBMF_SUCCESS) {
5194 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: SA access Failed: %d",
5195 sa_retval);
5196 (void) ibcm_ibmf_analyze_error(sa_retval);
5197 retval = IBT_MC_GROUP_INVALID;
5198 }
5199
5200 ibcm_sa_access_exit();
5201
5202 ibcm_dec_hca_acc_cnt(hcap);
5203
5204 return (retval);
5205 }
5206
5207
5208 /*
5209 * Function:
5210 * ibt_query_mcg
5211 * Input:
5212 * rgid The request GID that defines the HCA port upon which
5213 * to send the request to the Subnet Administrator, to
5214 * retrieve Multicast Records matching attributes as
5215 * specified through 'mcg_attr' argument.
5216 *
5217 * mcg_attr NULL or a pointer to an ibt_mcg_attr_t structure that
5218 * specifies MCG attributes that are to be matched.
5219 * Attributes that are not required can be wild carded
5220 * by specifying as '0'.
5221 *
5222 * mcgs_max_num The maximum number of matching multicast groups to
5223 * return. If zero, then all available matching multicast
5224 * groups are returned.
5225 * Output:
5226 * mcgs_info_p The address of an ibt_mcg_info_t pointer, where
5227 * multicast group information is returned. The actual
5228 * number of entries filled in the array is returned in
5229 * entries_p.
5230 *
5231 * entries_p The number of ibt_mcg_attr_t entries returned.
5232 * Returns:
5233 * IBT_SUCCESS
5234 * IBT_INVALID_PARAM
5235 * IBT_MCG_RECORDS_NOT_FOUND
5236 * Description:
5237 * Request information on multicast groups that match the parameters
5238 * specified in mcg_attr. Information on each multicast group is returned
5239 * to the caller in the form of an array of ibt_mcg_info_t.
5240 * ibt_query_mcg() allocates the memory for this array and returns a
5241 * pointer to the array (mcgs_p) and the number of entries in the array
5242 * (entries_p). This memory should be freed by the client using
5243 * ibt_free_mcg_info().
5244 */
5245 ibt_status_t
5246 ibt_query_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, uint_t mcgs_max_num,
5247 ibt_mcg_info_t **mcgs_info_p, uint_t *entries_p)
5248 {
5249 sa_mcmember_record_t mcg_req;
5250 sa_mcmember_record_t *mcg_resp;
5251 ibt_mcg_info_t *mcg_infop;
5252 ibmf_saa_access_args_t access_args;
5253 ibmf_saa_handle_t saa_handle;
5254 uint64_t component_mask = 0;
5255 ibt_status_t retval;
5256 ibtl_cm_hca_port_t hport;
5257 uint_t num_records;
5258 size_t length;
5259 void *results_p;
5260 ib_gid_t port_gid;
5261 ibcm_hca_info_t *hcap;
5262
5263 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg(%p, %d)", mcg_attr, mcgs_max_num);
5264
5265 if ((entries_p == NULL) || (mcgs_info_p == NULL)) {
5266 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: "
5267 "entries_p or mcgs_info_p is NULL");
5268 return (IBT_INVALID_PARAM);
5269 }
5270
5271 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) {
5272 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: RequestGID is required");
5273 return (IBT_INVALID_PARAM);
5274 }
5275 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Request GID <%llX:%llX>",
5276 rgid.gid_prefix, rgid.gid_guid);
5277
5278 bzero(&mcg_req, sizeof (sa_mcmember_record_t));
5279 port_gid.gid_prefix = port_gid.gid_guid = 0;
5280
5281 if (mcg_attr != NULL) {
5282 port_gid = mcg_attr->mc_pgid;
5283
5284 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) {
5285 mcg_req.PortGID = mcg_attr->mc_pgid;
5286 component_mask |= SA_MC_COMPMASK_PORTGID;
5287
5288 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: PGID %llX:%llX",
5289 port_gid.gid_prefix, port_gid.gid_guid);
5290 }
5291
5292 /* Is Q_Key specified. */
5293 if (mcg_attr->mc_qkey != 0) {
5294 mcg_req.Q_Key = mcg_attr->mc_qkey;
5295 component_mask |= SA_MC_COMPMASK_QKEY;
5296 }
5297
5298 /* Is P_Key specified. */
5299 if (mcg_attr->mc_pkey != 0) {
5300 mcg_req.P_Key = mcg_attr->mc_pkey;
5301 component_mask |= SA_MC_COMPMASK_PKEY;
5302 }
5303
5304 /* Is MGID specified. */
5305 if ((mcg_attr->mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) {
5306 mcg_req.MGID = mcg_attr->mc_mgid;
5307 component_mask |= SA_MC_COMPMASK_MGID;
5308 }
5309
5310 /* Is MTU specified. */
5311 if (mcg_attr->mc_mtu_req.r_mtu) {
5312 mcg_req.MTU = mcg_attr->mc_mtu_req.r_mtu;
5313 mcg_req.MTUSelector = mcg_attr->mc_mtu_req.r_selector;
5314
5315 component_mask |= SA_MC_COMPMASK_MTUSELECTOR |
5316 SA_MC_COMPMASK_MTU;
5317 }
5318
5319 if (mcg_attr->mc_tclass) {
5320 mcg_req.TClass = mcg_attr->mc_tclass;
5321 component_mask |= SA_MC_COMPMASK_TCLASS;
5322 }
5323
5324 /* Is RATE specified. */
5325 if (mcg_attr->mc_rate_req.r_srate) {
5326 mcg_req.Rate = mcg_attr->mc_rate_req.r_srate;
5327 mcg_req.RateSelector = mcg_attr->mc_rate_req.r_selector;
5328
5329 component_mask |= SA_MC_COMPMASK_RATESELECTOR |
5330 SA_MC_COMPMASK_RATE;
5331 }
5332
5333 /* Is Packet Life Time specified. */
5334 if (mcg_attr->mc_pkt_lt_req.p_pkt_lt) {
5335 mcg_req.Rate = mcg_attr->mc_pkt_lt_req.p_pkt_lt;
5336 mcg_req.RateSelector =
5337 mcg_attr->mc_pkt_lt_req.p_selector;
5338
5339 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR |
5340 SA_MC_COMPMASK_PKTLT;
5341 }
5342
5343 if (mcg_attr->mc_hop) {
5344 mcg_req.HopLimit = mcg_attr->mc_hop;
5345 component_mask |= SA_MC_COMPMASK_HOPLIMIT;
5346 }
5347
5348 if (mcg_attr->mc_flow) {
5349 mcg_req.FlowLabel = mcg_attr->mc_flow;
5350 component_mask |= SA_MC_COMPMASK_FLOWLABEL;
5351 }
5352
5353 if (mcg_attr->mc_sl) {
5354 mcg_req.SL = mcg_attr->mc_sl;
5355 component_mask |= SA_MC_COMPMASK_SL;
5356 }
5357
5358 if (mcg_attr->mc_scope) {
5359 mcg_req.Scope = mcg_attr->mc_scope;
5360 component_mask |= SA_MC_COMPMASK_SCOPE;
5361 }
5362
5363 if (mcg_attr->mc_join_state) {
5364 mcg_req.JoinState = mcg_attr->mc_join_state;
5365 component_mask |= SA_MC_COMPMASK_JOINSTATE;
5366 }
5367
5368 if (mcg_attr->mc_mlid) {
5369 mcg_req.MLID = mcg_attr->mc_mlid;
5370 component_mask |= SA_MC_COMPMASK_MLID;
5371 }
5372 }
5373
5374 retval = ibtl_cm_get_hca_port(rgid, 0, &hport);
5375 if (retval != IBT_SUCCESS) {
5376 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: Failed to get port info "
5377 "from specified RGID : status = %d", retval);
5378 return (retval);
5379 }
5380
5381 /* Get SA Access Handle. */
5382 hcap = ibcm_find_hca_entry(hport.hp_hca_guid);
5383 if (hcap == NULL) {
5384 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: NO HCA found");
5385 return (IBT_HCA_BUSY_DETACHING);
5386 }
5387
5388 saa_handle = ibcm_get_saa_handle(hcap, hport.hp_port);
5389 if (saa_handle == NULL) {
5390 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: saa_handle is NULL");
5391 ibcm_dec_hca_acc_cnt(hcap);
5392 return (IBT_HCA_PORT_NOT_ACTIVE);
5393 }
5394
5395 /* Contact SA Access */
5396 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID;
5397 access_args.sq_access_type = IBMF_SAA_RETRIEVE;
5398 access_args.sq_component_mask = component_mask;
5399 access_args.sq_template = &mcg_req;
5400 access_args.sq_template_length = sizeof (sa_mcmember_record_t);
5401 access_args.sq_callback = NULL;
5402 access_args.sq_callback_arg = NULL;
5403
5404 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length,
5405 &results_p);
5406 if (retval != IBT_SUCCESS) {
5407 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: SA access Failed");
5408 ibcm_dec_hca_acc_cnt(hcap);
5409 return (retval);
5410 }
5411
5412 num_records = length/sizeof (sa_mcmember_record_t);
5413
5414 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Found %d MCMember Records",
5415 num_records);
5416
5417 /* Validate the returned number of records. */
5418 if ((results_p != NULL) && (num_records > 0)) {
5419 uint_t i;
5420
5421 /*
5422 * If mcgs_max_num is zero, then return all records else
5423 * return only requested number of records
5424 */
5425 if ((mcgs_max_num != 0) && (num_records > mcgs_max_num)) {
5426 /* we are interested in only mcgs_max_num records */
5427 num_records = mcgs_max_num;
5428 }
5429
5430 /*
5431 * The SGID returned in "mcg_info_p" buffer should be PortGID,
5432 * (mcg_attr->mc_pgid), if 'mcg_attr->mc_pgid' was specified,
5433 * else RequestGID (rgid) should be returned.
5434 */
5435 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) {
5436
5437 /* Get sgid_ix and port number of 'port_gid' */
5438 retval = ibtl_cm_get_hca_port(port_gid, 0, &hport);
5439 if (retval != IBT_SUCCESS) {
5440 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: "
5441 "Failed to Get Portinfo for PortGID :"
5442 "status = %d", retval);
5443 return (retval);
5444 }
5445 } else {
5446 /*
5447 * The sgid_ix and port number related to RequestGID
5448 * are already obtained at the beginning.
5449 */
5450 port_gid = rgid;
5451 }
5452
5453 /*
5454 * Allocate memory for return buffer, to be freed in
5455 * ibt_free_mcg_info().
5456 */
5457 mcg_infop = kmem_alloc((num_records * sizeof (ibt_mcg_info_t)),
5458 KM_SLEEP);
5459
5460 *mcgs_info_p = mcg_infop;
5461 *entries_p = num_records;
5462
5463 /* Update the return values. */
5464 for (i = 0; i < num_records; i++) {
5465
5466 mcg_resp = (sa_mcmember_record_t *)((uchar_t *)
5467 results_p + i * sizeof (sa_mcmember_record_t));
5468
5469 mcg_infop[i].mc_adds_vect.av_dgid = mcg_resp->MGID;
5470 mcg_infop[i].mc_adds_vect.av_sgid = port_gid;
5471 mcg_infop[i].mc_adds_vect.av_srate = mcg_resp->Rate;
5472 mcg_infop[i].mc_adds_vect.av_srvl = mcg_resp->SL;
5473 mcg_infop[i].mc_adds_vect.av_flow = mcg_resp->FlowLabel;
5474 mcg_infop[i].mc_adds_vect.av_tclass = mcg_resp->TClass;
5475 mcg_infop[i].mc_adds_vect.av_hop = mcg_resp->HopLimit;
5476 mcg_infop[i].mc_adds_vect.av_port_num = hport.hp_port;
5477 mcg_infop[i].mc_adds_vect.av_send_grh = B_TRUE;
5478 mcg_infop[i].mc_adds_vect.av_dlid = mcg_resp->MLID;
5479 mcg_infop[i].mc_adds_vect.av_sgid_ix = hport.hp_sgid_ix;
5480 mcg_infop[i].mc_adds_vect.av_src_path = 0;
5481 mcg_infop[i].mc_mtu = mcg_resp->MTU;
5482 mcg_infop[i].mc_qkey = mcg_resp->Q_Key;
5483 mcg_infop[i].mc_scope = mcg_resp->Scope;
5484 mcg_infop[i].mc_pkt_lt = mcg_resp->PacketLifeTime;
5485
5486 if (ibt_pkey2index_byguid(hport.hp_hca_guid,
5487 hport.hp_port, mcg_resp->P_Key,
5488 &mcg_infop[i].mc_pkey_ix) != IBT_SUCCESS) {
5489 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: "
5490 "Pkey2Index Conversion failed");
5491 mcg_infop[i].mc_pkey_ix = 0;
5492 }
5493 }
5494
5495 /*
5496 * Deallocate the memory allocated by SA for results_p.
5497 */
5498 kmem_free(results_p, length);
5499 retval = IBT_SUCCESS;
5500
5501 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: returning %d MCGRecords",
5502 num_records);
5503
5504 } else {
5505 retval = IBT_MCG_RECORDS_NOT_FOUND;
5506 *entries_p = 0;
5507
5508 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: MCG RECORDS NOT FOUND");
5509 }
5510
5511 ibcm_dec_hca_acc_cnt(hcap);
5512
5513 return (retval);
5514 }
5515
5516
5517 /*
5518 * ibt_free_mcg_info()
5519 * Free the memory allocated by successful ibt_query_mcg()
5520 *
5521 * mcgs_info Pointer returned by ibt_query_mcg().
5522 *
5523 * entries The number of ibt_mcg_info_t entries to free.
5524 */
5525 void
5526 ibt_free_mcg_info(ibt_mcg_info_t *mcgs_info, uint_t entries)
5527 {
5528 IBTF_DPRINTF_L3(cmlog, "ibt_free_mcg_info: "
5529 "Free <%d> entries from 0x%p", entries, mcgs_info);
5530
5531 if ((mcgs_info != NULL) && (entries > 0))
5532 kmem_free(mcgs_info, entries * sizeof (ibt_mcg_info_t));
5533 else
5534 IBTF_DPRINTF_L2(cmlog, "ibt_free_mcg_info: "
5535 "ERROR: NULL buf pointer or length specified.");
5536 }
5537
5538
5539 /*
5540 * Function:
5541 * ibt_gid_to_node_info()
5542 * Input:
5543 * gid Identifies the IB Node and port for which to obtain
5544 * Node information.
5545 * Output:
5546 * node_info_p A pointer to an ibt_node_info_t structure (allocated
5547 * by the caller) in which to return the node information.
5548 * Returns:
5549 * IBT_SUCCESS
5550 * IBT_INVALID_PARAM
5551 * IBT_NODE_RECORDS_NOT_FOUND
5552 * IBT_NO_HCAS_AVAILABLE
5553 * Description:
5554 * Retrieve Node Information for the specified GID.
5555 */
5556 ibt_status_t
5557 ibt_gid_to_node_info(ib_gid_t gid, ibt_node_info_t *node_info_p)
5558 {
5559 sa_node_record_t nr_req, *nr_resp;
5560 ibmf_saa_handle_t saa_handle;
5561 ibt_status_t retval;
5562 ibcm_hca_info_t *hcap;
5563 ibtl_cm_hca_port_t hport;
5564 int i, j;
5565 uint_t num_rec;
5566 ib_guid_t *guid_array = NULL;
5567 sa_path_record_t *path;
5568 size_t len;
5569 uint8_t npaths;
5570 uint32_t num_hcas = 0;
5571 ib_lid_t node_lid;
5572 boolean_t local_node = B_FALSE;
5573 void *res_p;
5574 uint8_t num_ports = 0;
5575
5576
5577 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info(%llX:%llX, %p)",
5578 gid.gid_prefix, gid.gid_guid, node_info_p);
5579
5580 if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) {
5581 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: GID is required");
5582 return (IBT_INVALID_PARAM);
5583 }
5584
5585 if (node_info_p == NULL) {
5586 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: "
5587 "Return Buf (node_info_p) is NULL.");
5588 return (IBT_INVALID_PARAM);
5589 }
5590
5591 /*
5592 * If 'gid' is on local node, then get node lid (i.e. base lid of the
5593 * associated port) info via ibtl_cm_get_hca_port() call.
5594 */
5595 bzero(&hport, sizeof (ibtl_cm_hca_port_t));
5596 if (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS) {
5597
5598 hcap = ibcm_find_hca_entry(hport.hp_hca_guid);
5599 if (hcap == NULL) {
5600 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: "
5601 "HCA(%llX) info not found", hport.hp_hca_guid);
5602 return (IBT_NO_HCAS_AVAILABLE);
5603 }
5604 num_ports = 1;
5605 num_hcas = 1;
5606 node_lid = hport.hp_base_lid;
5607 local_node = B_TRUE;
5608 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: Local Node: "
5609 "LID = 0x%X", node_lid);
5610 } else {
5611 /* Get the number of HCAs and their GUIDs */
5612 num_hcas = ibt_get_hca_list(&guid_array);
5613 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: ibt_get_hca_list "
5614 "returned %d hcas", num_hcas);
5615
5616 if (num_hcas == 0) {
5617 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: "
5618 "NO HCA's Found on this system");
5619 return (IBT_NO_HCAS_AVAILABLE);
5620 }
5621 }
5622
5623 for (i = 0; i < num_hcas; i++) {
5624 if (local_node == B_FALSE) {
5625 hcap = ibcm_find_hca_entry(guid_array[i]);
5626 if (hcap == NULL) {
5627 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: "
5628 "HCA(%llX) info not found", guid_array[i]);
5629 retval = IBT_NO_HCAS_AVAILABLE;
5630 continue;
5631 }
5632 num_ports = hcap->hca_num_ports;
5633 }
5634
5635 for (j = 0; j < num_ports; j++) {
5636 uint8_t port = 0;
5637
5638 if (local_node == B_TRUE)
5639 port = hport.hp_port;
5640 else
5641 port = j + 1;
5642
5643 /* Get SA Access Handle. */
5644 saa_handle = ibcm_get_saa_handle(hcap, port);
5645 if (saa_handle == NULL) {
5646 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: "
5647 "Port %d of HCA (%llX) is NOT ACTIVE",
5648 port, hport.hp_hca_guid);
5649 retval = IBT_NODE_RECORDS_NOT_FOUND;
5650 continue;
5651 }
5652
5653 if (local_node == B_FALSE) {
5654 ib_gid_t sgid;
5655 int sa_ret;
5656
5657 /*
5658 * Check whether 'gid' and this port has same
5659 * subnet prefix. If not, then there is no use
5660 * in searching from this port.
5661 */
5662 sgid = hcap->hca_port_info[j].port_sgid0;
5663 if (gid.gid_prefix != sgid.gid_prefix) {
5664 IBTF_DPRINTF_L3(cmlog,
5665 "ibt_gid_to_node_info:Sn_Prefix of "
5666 "GID(%llX) and Port's(%llX) differ",
5667 gid.gid_prefix, sgid.gid_prefix);
5668 retval = IBT_NODE_RECORDS_NOT_FOUND;
5669 continue;
5670 }
5671
5672 /*
5673 * First Get Path Records for the specified DGID
5674 * from this port (SGID). From Path Records,
5675 * note down DLID, then use this DLID as Input
5676 * attribute to get NodeRecords from SA Access.
5677 */
5678 npaths = 1;
5679 path = NULL;
5680
5681 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle,
5682 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len,
5683 &path);
5684 if (sa_ret != IBMF_SUCCESS) {
5685 IBTF_DPRINTF_L2(cmlog,
5686 "ibt_gid_to_node_info: "
5687 "ibmf_saa_gid_to_pathrecords() "
5688 "returned error: %d ", sa_ret);
5689 retval =
5690 ibcm_ibmf_analyze_error(sa_ret);
5691 continue;
5692 } else if ((npaths == 0) || (path == NULL)) {
5693 IBTF_DPRINTF_L3(cmlog,
5694 "ibt_gid_to_node_info: failed (%d) "
5695 "to get path records for the DGID "
5696 "0x%llX from SGID 0x%llX", sa_ret,
5697 gid.gid_guid, sgid.gid_guid);
5698 retval = IBT_NODE_RECORDS_NOT_FOUND;
5699 continue;
5700 }
5701 node_lid = path->DLID; /* LID */
5702
5703 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: "
5704 "Remote Node: LID = 0x%X", node_lid);
5705
5706 /* Free SA_Access memory for path record. */
5707 kmem_free(path, len);
5708 }
5709
5710 /* Retrieve Node Records from SA Access. */
5711 bzero(&nr_req, sizeof (sa_node_record_t));
5712
5713 nr_req.LID = node_lid; /* LID */
5714
5715 retval = ibcm_get_node_rec(saa_handle, &nr_req,
5716 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len);
5717 if (retval == IBT_NODE_RECORDS_NOT_FOUND) {
5718 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: "
5719 "failed (%d) to get Node records", retval);
5720 continue;
5721 } else if (retval != IBT_SUCCESS) {
5722 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: "
5723 "failed (%d) to get Node records", retval);
5724 ibcm_dec_hca_acc_cnt(hcap);
5725 goto gid_to_ni_exit;
5726 }
5727
5728 num_rec = len/sizeof (sa_node_record_t);
5729 nr_resp = (sa_node_record_t *)(uchar_t *)res_p;
5730
5731 /* Validate the returned number of records. */
5732 if ((nr_resp != NULL) && (num_rec > 0)) {
5733
5734 IBCM_DUMP_NODE_REC(nr_resp);
5735
5736 node_info_p->n_sys_img_guid =
5737 nr_resp->NodeInfo.SystemImageGUID;
5738 node_info_p->n_node_guid =
5739 nr_resp->NodeInfo.NodeGUID;
5740 node_info_p->n_port_guid =
5741 nr_resp->NodeInfo.PortGUID;
5742 node_info_p->n_dev_id =
5743 nr_resp->NodeInfo.DeviceID;
5744 node_info_p->n_revision =
5745 nr_resp->NodeInfo.Revision;
5746 node_info_p->n_vendor_id =
5747 nr_resp->NodeInfo.VendorID;
5748 node_info_p->n_num_ports =
5749 nr_resp->NodeInfo.NumPorts;
5750 node_info_p->n_port_num =
5751 nr_resp->NodeInfo.LocalPortNum;
5752 node_info_p->n_node_type =
5753 nr_resp->NodeInfo.NodeType;
5754 (void) strncpy(node_info_p->n_description,
5755 (char *)&nr_resp->NodeDescription, 64);
5756
5757 /*
5758 * Deallocate the memory allocated by SA for
5759 * 'nr_resp'.
5760 */
5761 ibcm_dec_hca_acc_cnt(hcap);
5762 kmem_free(nr_resp, len);
5763 retval = IBT_SUCCESS;
5764
5765 goto gid_to_ni_exit;
5766 } else {
5767 retval = IBT_NODE_RECORDS_NOT_FOUND;
5768 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: "
5769 "Node Records NOT found - PortGUID %016llX",
5770 gid.gid_guid);
5771 }
5772 }
5773 ibcm_dec_hca_acc_cnt(hcap);
5774
5775 if (local_node == B_TRUE)
5776 break;
5777 }
5778
5779 gid_to_ni_exit:
5780 if (guid_array)
5781 ibt_free_hca_list(guid_array, num_hcas);
5782
5783 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: done. Status %d", retval);
5784
5785 return (retval);
5786 }
5787
5788
5789 ibt_status_t
5790 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req,
5791 uint64_t component_mask, void *result_p, size_t *len)
5792 {
5793 ibmf_saa_access_args_t args;
5794 size_t length;
5795 ibt_status_t retval;
5796
5797 args.sq_attr_id = SA_NODERECORD_ATTRID;
5798 args.sq_template = nr_req;
5799 args.sq_access_type = IBMF_SAA_RETRIEVE;
5800 args.sq_template_length = sizeof (sa_node_record_t);
5801 args.sq_component_mask = component_mask;
5802 args.sq_callback = NULL;
5803 args.sq_callback_arg = NULL;
5804
5805 retval = ibcm_contact_sa_access(saa_handle, &args, &length, result_p);
5806 if (retval != IBT_SUCCESS) {
5807 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: SA Call Failed");
5808 return (retval);
5809 }
5810
5811 *len = length;
5812
5813 /* Validate the returned number of records. */
5814 if ((result_p != NULL) && (length > 0)) {
5815 IBTF_DPRINTF_L3(cmlog, "ibcm_get_node_rec: Node Records FOUND");
5816
5817 /* Got it, done!. */
5818 return (IBT_SUCCESS);
5819 } else {
5820 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: Node Rec NOT found");
5821 return (IBT_NODE_RECORDS_NOT_FOUND);
5822 }
5823 }
5824
5825
5826 /*
5827 * Function:
5828 * ibt_lid_to_node_info()
5829 * Input:
5830 * lid Identifies the IB Node and port for which to obtain
5831 * Node information.
5832 * Output:
5833 * node_info_p A pointer to an ibt_node_info_t structure (allocated
5834 * by the caller) in which to return the node information.
5835 * Returns:
5836 * IBT_SUCCESS
5837 * IBT_INVALID_PARAM
5838 * IBT_NODE_RECORDS_NOT_FOUND
5839 * IBT_NO_HCAS_AVAILABLE
5840 * Description:
5841 * Retrieve Node Information for the specified LID.
5842 */
5843 ibt_status_t
5844 ibt_lid_to_node_info(ib_lid_t lid, ibt_node_info_t *node_info_p)
5845 {
5846 ibt_status_t retval;
5847 ibcm_hca_info_t *hcap;
5848 uint8_t i, j;
5849 ib_guid_t *guid_array = NULL;
5850 uint_t num_hcas = 0;
5851
5852
5853 IBTF_DPRINTF_L4(cmlog, "ibt_lid_to_node_info(0x%lX, %p)",
5854 lid, node_info_p);
5855
5856 if ((lid == 0) || (node_info_p == NULL)) {
5857 IBTF_DPRINTF_L2(cmlog, "ibt_lid_to_node_info: "
5858 "Lid is zero, or node_info_p is NULL.");
5859 return (IBT_INVALID_PARAM);
5860 }
5861
5862 /* Get the number of HCAs and their GUIDs */
5863 num_hcas = ibt_get_hca_list(&guid_array);
5864 IBTF_DPRINTF_L4(cmlog, "ibt_lid_to_node_info: ibt_get_hca_list "
5865 "returned %d hcas", num_hcas);
5866
5867 if (num_hcas == 0) {
5868 IBTF_DPRINTF_L2(cmlog, "ibt_lid_to_node_info: "
5869 "NO HCA's Found on this system");
5870 return (IBT_NO_HCAS_AVAILABLE);
5871 }
5872
5873 for (i = 0; i < num_hcas; i++) {
5874 hcap = ibcm_find_hca_entry(guid_array[i]);
5875 if (hcap == NULL) {
5876 IBTF_DPRINTF_L3(cmlog, "ibt_lid_to_node_info: "
5877 "HCA(%llX) info not found", guid_array[i]);
5878 retval = IBT_NO_HCAS_AVAILABLE;
5879 continue;
5880 }
5881
5882 for (j = 0; j < hcap->hca_num_ports; j++) {
5883 uint8_t port;
5884 ibmf_saa_handle_t saa_handle;
5885 uint_t num_rec;
5886 size_t len;
5887 void *res_p;
5888 sa_node_record_t nr_req, *nr_resp;
5889
5890 port = j + 1;
5891
5892 /* Get SA Access Handle. */
5893 saa_handle = ibcm_get_saa_handle(hcap, port);
5894 if (saa_handle == NULL) {
5895 IBTF_DPRINTF_L3(cmlog, "ibt_lid_to_node_info: "
5896 "Port %d of HCA (%llX) is NOT ACTIVE",
5897 port, guid_array[i]);
5898 retval = IBT_NODE_RECORDS_NOT_FOUND;
5899 continue;
5900 }
5901
5902 /* Retrieve Node Records from SA Access. */
5903 bzero(&nr_req, sizeof (sa_node_record_t));
5904
5905 nr_req.LID = lid; /* LID */
5906
5907 retval = ibcm_get_node_rec(saa_handle, &nr_req,
5908 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len);
5909 if (retval == IBT_NODE_RECORDS_NOT_FOUND) {
5910 IBTF_DPRINTF_L2(cmlog, "ibt_lid_to_node_info: "
5911 "failed (%d) to get Node records", retval);
5912 continue;
5913 } else if (retval != IBT_SUCCESS) {
5914 IBTF_DPRINTF_L2(cmlog, "ibt_lid_to_node_info: "
5915 "failed (%d) to get Node records", retval);
5916 ibcm_dec_hca_acc_cnt(hcap);
5917 goto lid_to_ni_exit;
5918 }
5919
5920 num_rec = len/sizeof (sa_node_record_t);
5921 nr_resp = (sa_node_record_t *)(uchar_t *)res_p;
5922
5923 /* Validate the returned number of records. */
5924 if ((nr_resp != NULL) && (num_rec > 0)) {
5925
5926 IBCM_DUMP_NODE_REC(nr_resp);
5927
5928 node_info_p->n_sys_img_guid =
5929 nr_resp->NodeInfo.SystemImageGUID;
5930 node_info_p->n_node_guid =
5931 nr_resp->NodeInfo.NodeGUID;
5932 node_info_p->n_port_guid =
5933 nr_resp->NodeInfo.PortGUID;
5934 node_info_p->n_dev_id =
5935 nr_resp->NodeInfo.DeviceID;
5936 node_info_p->n_revision =
5937 nr_resp->NodeInfo.Revision;
5938 node_info_p->n_vendor_id =
5939 nr_resp->NodeInfo.VendorID;
5940 node_info_p->n_num_ports =
5941 nr_resp->NodeInfo.NumPorts;
5942 node_info_p->n_port_num =
5943 nr_resp->NodeInfo.LocalPortNum;
5944 node_info_p->n_node_type =
5945 nr_resp->NodeInfo.NodeType;
5946 (void) strncpy(node_info_p->n_description,
5947 (char *)&nr_resp->NodeDescription, 64);
5948
5949 /*
5950 * Deallocate the memory allocated by SA for
5951 * 'nr_resp'.
5952 */
5953 ibcm_dec_hca_acc_cnt(hcap);
5954 kmem_free(nr_resp, len);
5955 retval = IBT_SUCCESS;
5956
5957 goto lid_to_ni_exit;
5958 } else {
5959 retval = IBT_NODE_RECORDS_NOT_FOUND;
5960 IBTF_DPRINTF_L3(cmlog, "ibt_lid_to_node_info: "
5961 "Node Records NOT found - LID 0x%lX",
5962 lid);
5963 }
5964 }
5965 ibcm_dec_hca_acc_cnt(hcap);
5966 }
5967
5968 lid_to_ni_exit:
5969 if (guid_array)
5970 ibt_free_hca_list(guid_array, num_hcas);
5971
5972 IBTF_DPRINTF_L3(cmlog, "ibt_lid_to_node_info: done. Status %d", retval);
5973
5974 return (retval);
5975 }
5976
5977 /*
5978 * Function:
5979 * ibt_get_companion_port_gids()
5980 * Description:
5981 * Get list of GID's available on a companion port(s) of the specified
5982 * GID or list of GIDs available on a specified Node GUID/SystemImage GUID.
5983 */
5984 ibt_status_t
5985 ibt_get_companion_port_gids(ib_gid_t gid, ib_guid_t hca_guid,
5986 ib_guid_t sysimg_guid, ib_gid_t **gids_p, uint_t *num_gids_p)
5987 {
5988 sa_node_record_t nr_req, *nr_resp;
5989 void *res_p;
5990 ibmf_saa_handle_t saa_handle;
5991 int sa_ret;
5992 ibt_status_t retval = IBT_SUCCESS;
5993 ibcm_hca_info_t *hcap;
5994 ibtl_cm_hca_port_t hport;
5995 int i, j;
5996 uint_t num_rec;
5997 ib_guid_t *guid_array = NULL;
5998 sa_path_record_t *path;
5999 size_t len;
6000 uint8_t npaths;
6001 uint32_t num_hcas = 0;
6002 boolean_t local_node = B_FALSE;
6003 boolean_t local_hca = B_FALSE;
6004 ib_guid_t h_guid = hca_guid;
6005 ib_gid_t *gidp = NULL, *t_gidp = NULL;
6006 int multi_hca_loop = 0;
6007
6008 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids(%llX:%llX, %llX, "
6009 "%llX)", gid.gid_prefix, gid.gid_guid, hca_guid, sysimg_guid);
6010
6011 if (((gid.gid_prefix == 0) || (gid.gid_guid == 0)) && (hca_guid == 0) &&
6012 (sysimg_guid == 0)) {
6013 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: "
6014 "Null Input attribute specified.");
6015 return (IBT_INVALID_PARAM);
6016 }
6017
6018 if ((num_gids_p == NULL) || (gids_p == NULL)) {
6019 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: "
6020 "num_gids_p or gids_p is NULL");
6021 return (IBT_INVALID_PARAM);
6022 }
6023
6024 *num_gids_p = 0;
6025
6026 /* Get the number of HCAs and their GUIDs */
6027 if ((num_hcas = ibt_get_hca_list(&guid_array)) == 0) {
6028 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: "
6029 "NO HCA's Found on this system");
6030 return (IBT_NO_HCAS_AVAILABLE);
6031 }
6032
6033 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: "
6034 "ibt_get_hca_list() returned %d hcas", num_hcas);
6035
6036 /*
6037 * If 'gid' is on local node, then get node lid (i.e. base lid of the
6038 * associated port) info via ibtl_cm_get_hca_port() call.
6039 */
6040 bzero(&hport, sizeof (ibtl_cm_hca_port_t));
6041 if ((gid.gid_prefix != 0) && (gid.gid_guid != 0) &&
6042 (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS)) {
6043
6044 if ((hca_guid != 0) && (hca_guid != hport.hp_hca_guid)) {
6045 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: "
6046 "Invalid GID<->HCAGUID combination specified.");
6047 retval = IBT_INVALID_PARAM;
6048 goto get_comp_pgid_exit;
6049 }
6050 h_guid = hport.hp_hca_guid;
6051 local_node = B_TRUE;
6052
6053 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: "
6054 "Local Node: HCA (0x%llX)", h_guid);
6055 } else if (h_guid) { /* Is specified HCA GUID - local? */
6056 for (i = 0; i < num_hcas; i++) {
6057 if (h_guid == guid_array[i]) {
6058 local_hca = B_TRUE;
6059 break;
6060 }
6061 }
6062 } else if (sysimg_guid) { /* Is specified SystemImage GUID - local? */
6063 for (i = 0; i < num_hcas; i++) {
6064 ibt_status_t ret;
6065 ibt_hca_attr_t hca_attr;
6066
6067 ret = ibt_query_hca_byguid(guid_array[i], &hca_attr);
6068 if (ret != IBT_SUCCESS) {
6069 IBTF_DPRINTF_L2(cmlog,
6070 "ibt_get_companion_port_gids: HCA(%llX) "
6071 "info not found", guid_array[i]);
6072 retval = IBT_NO_HCAS_AVAILABLE;
6073 continue;
6074 }
6075 if (hca_attr.hca_si_guid == sysimg_guid) {
6076 if ((hca_guid != 0) &&
6077 (hca_guid != hca_attr.hca_node_guid)) {
6078 IBTF_DPRINTF_L2(cmlog,
6079 "ibt_get_companion_port_gids: "
6080 "Invalid SysImg<->HCA GUID "
6081 "combination specified.");
6082 retval = IBT_INVALID_PARAM;
6083 goto get_comp_pgid_exit;
6084 }
6085 local_hca = B_TRUE;
6086 h_guid = hca_attr.hca_node_guid;
6087 break;
6088 }
6089 }
6090 }
6091
6092 if ((local_node == B_TRUE) || (local_hca == B_TRUE)) {
6093 retval = ibtl_cm_get_local_comp_gids(h_guid, gid, gids_p,
6094 num_gids_p);
6095 goto get_comp_pgid_exit;
6096 }
6097
6098 get_comp_for_multihca:
6099 /* We will be here, if request is for remote node */
6100 for (i = 0; i < num_hcas; i++) {
6101 int multism;
6102 uint_t count = 0;
6103 int multi_sm_loop = 0;
6104 uint_t k = 0, l;
6105
6106 hcap = ibcm_find_hca_entry(guid_array[i]);
6107 if (hcap == NULL) {
6108 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: "
6109 "HCA(%llX) info not found", guid_array[i]);
6110 retval = IBT_NO_HCAS_AVAILABLE;
6111 continue;
6112 }
6113
6114 /* 1 - MultiSM, 0 - Single SM */
6115 multism = ibtl_cm_is_multi_sm(guid_array[i]);
6116
6117 for (j = 0; j < hcap->hca_num_ports; j++) {
6118 ib_gid_t sgid;
6119 uint64_t c_mask = 0;
6120 ib_guid_t pg;
6121 uint_t port = j;
6122
6123 get_comp_for_multism:
6124 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: "
6125 "Port %d, HCA %llX, MultiSM= %d, Loop=%d",
6126 port + 1, h_guid, multism, multi_sm_loop);
6127
6128 /* Get SA Access Handle. */
6129 saa_handle = ibcm_get_saa_handle(hcap, port + 1);
6130 if (saa_handle == NULL) {
6131 IBTF_DPRINTF_L2(cmlog,
6132 "ibt_get_companion_port_gids: "
6133 "Port (%d) - NOT ACTIVE", port + 1);
6134 retval = IBT_GIDS_NOT_FOUND;
6135 continue;
6136 }
6137
6138 /*
6139 * Check whether 'gid' and this port has same subnet
6140 * prefix. If not, then there is no use in searching
6141 * from this port.
6142 */
6143 sgid = hcap->hca_port_info[port].port_sgid0;
6144 if ((h_guid == 0) && (gid.gid_prefix != 0) &&
6145 (multi_sm_loop == 0) &&
6146 (gid.gid_prefix != sgid.gid_prefix)) {
6147 IBTF_DPRINTF_L2(cmlog,
6148 "ibt_get_companion_port_gids: SnPrefix of "
6149 "GID(%llX) and Port SN_Pfx(%llX) differ",
6150 gid.gid_prefix, sgid.gid_prefix);
6151 retval = IBT_GIDS_NOT_FOUND;
6152 continue;
6153 }
6154
6155 /*
6156 * If HCA GUID or System Image GUID is specified, then
6157 * we can achieve our goal sooner!.
6158 */
6159 if ((h_guid == 0) && (sysimg_guid == 0)) {
6160 /* So only GID info is provided. */
6161
6162 /*
6163 * First Get Path Records for the specified DGID
6164 * from this port (SGID). From Path Records,
6165 * note down DLID, then use this DLID as Input
6166 * attribute to get NodeRecords.
6167 */
6168 npaths = 1;
6169 path = NULL;
6170
6171 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle,
6172 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len,
6173 &path);
6174 if (sa_ret != IBMF_SUCCESS) {
6175 IBTF_DPRINTF_L2(cmlog,
6176 "ibt_get_companion_port_gids: "
6177 "ibmf_saa_gid_to_pathrecords() "
6178 "returned error: %d ", sa_ret);
6179 retval =
6180 ibcm_ibmf_analyze_error(sa_ret);
6181 ibcm_dec_hca_acc_cnt(hcap);
6182 goto get_comp_pgid_exit;
6183 } else if ((npaths == 0) || (path == NULL)) {
6184 IBTF_DPRINTF_L2(cmlog,
6185 "ibt_get_companion_port_gids: "
6186 "failed (%d) to get path records "
6187 "for the DGID (0x%llX) from SGID "
6188 "(0x%llX)", sa_ret, gid.gid_guid,
6189 sgid.gid_guid);
6190 retval = IBT_GIDS_NOT_FOUND;
6191 continue;
6192 }
6193
6194 bzero(&nr_req, sizeof (sa_node_record_t));
6195 nr_req.LID = path->DLID; /* LID */
6196
6197 IBTF_DPRINTF_L3(cmlog,
6198 "ibt_get_companion_port_gids: "
6199 "Remote Node: LID = 0x%X", nr_req.LID);
6200
6201 /* Free SA_Access memory for path record. */
6202 kmem_free(path, len);
6203
6204 IBTF_DPRINTF_L3(cmlog,
6205 "ibt_get_companion_port_gids: SAA Call: "
6206 "based on LID ");
6207
6208 retval = ibcm_get_node_rec(saa_handle, &nr_req,
6209 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len);
6210 if (retval == IBT_NODE_RECORDS_NOT_FOUND) {
6211 IBTF_DPRINTF_L2(cmlog,
6212 "ibt_get_companion_port_gids: "
6213 "failed (%d) to get Node records",
6214 retval);
6215 continue;
6216 } else if (retval != IBT_SUCCESS) {
6217 IBTF_DPRINTF_L2(cmlog,
6218 "ibt_get_companion_port_gids: "
6219 "failed (%d) to get Node records",
6220 retval);
6221 ibcm_dec_hca_acc_cnt(hcap);
6222 goto get_comp_pgid_exit;
6223 }
6224
6225 nr_resp = (sa_node_record_t *)(uchar_t *)res_p;
6226 /* Note down HCA GUID info. */
6227 h_guid = nr_resp->NodeInfo.NodeGUID;
6228
6229 IBTF_DPRINTF_L3(cmlog,
6230 "ibt_get_companion_port_gids: "
6231 "Remote HCA GUID: 0x%llX", h_guid);
6232
6233 IBCM_DUMP_NODE_REC(nr_resp);
6234
6235 kmem_free(res_p, len);
6236 }
6237
6238 bzero(&nr_req, sizeof (sa_node_record_t));
6239 if (h_guid != 0) {
6240 nr_req.NodeInfo.NodeGUID = h_guid;
6241 c_mask = SA_NODEINFO_COMPMASK_NODEGUID;
6242 }
6243
6244 if (sysimg_guid != 0) {
6245 nr_req.NodeInfo.SystemImageGUID = sysimg_guid;
6246 c_mask |= SA_NODEINFO_COMPMASK_SYSIMAGEGUID;
6247 }
6248
6249 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: "
6250 "SAA Call: CMASK= 0x%llX", c_mask);
6251
6252 retval = ibcm_get_node_rec(saa_handle, &nr_req, c_mask,
6253 &res_p, &len);
6254 if (retval == IBT_NODE_RECORDS_NOT_FOUND) {
6255 IBTF_DPRINTF_L3(cmlog,
6256 "ibt_get_companion_port_gids: "
6257 "failed (%d) to get Node records", retval);
6258 continue;
6259 } else if (retval != IBT_SUCCESS) {
6260 IBTF_DPRINTF_L2(cmlog,
6261 "ibt_get_companion_port_gids: Error: (%d) "
6262 "while getting Node records", retval);
6263 ibcm_dec_hca_acc_cnt(hcap);
6264 goto get_comp_pgid_exit;
6265 }
6266
6267 num_rec = len/sizeof (sa_node_record_t);
6268
6269 /* We will be here, only if we found some NodeRec */
6270 if (gid.gid_prefix && gid.gid_guid) {
6271 nr_resp = (sa_node_record_t *)res_p;
6272 for (l = 0; l < num_rec; l++, nr_resp++) {
6273 pg = nr_resp->NodeInfo.PortGUID;
6274 if (gid.gid_guid != pg)
6275 count++;
6276 }
6277 } else {
6278 count = num_rec;
6279 }
6280
6281 if (count != 0) {
6282 if (multi_sm_loop == 1) {
6283 count += k;
6284 t_gidp = kmem_zalloc(count *
6285 sizeof (ib_gid_t), KM_SLEEP);
6286
6287 if ((k != 0) && (gidp != NULL)) {
6288 bcopy(gidp, t_gidp,
6289 k * sizeof (ib_gid_t));
6290 kmem_free(gidp,
6291 k * sizeof (ib_gid_t));
6292 }
6293 gidp = t_gidp;
6294 } else {
6295 gidp = kmem_zalloc(count *
6296 sizeof (ib_gid_t), KM_SLEEP);
6297 }
6298 *num_gids_p = count;
6299 *gids_p = gidp;
6300
6301 nr_resp = (sa_node_record_t *)res_p;
6302 for (l = 0; l < num_rec; l++, nr_resp++) {
6303 IBCM_DUMP_NODE_REC(nr_resp);
6304
6305 pg = nr_resp->NodeInfo.PortGUID;
6306 IBTF_DPRINTF_L4(cmlog,
6307 "ibt_get_companion_port_gids: "
6308 "PortGID %llX", pg);
6309
6310 if (pg != gid.gid_guid) {
6311 gidp[k].gid_prefix =
6312 sgid.gid_prefix;
6313 gidp[k].gid_guid = pg;
6314
6315 IBTF_DPRINTF_L3(cmlog,
6316 "ibt_get_companion_pgids: "
6317 "GID[%d] = %llX:%llX", k,
6318 gidp[k].gid_prefix,
6319 gidp[k].gid_guid);
6320
6321 k++;
6322 if (k == count)
6323 break;
6324 }
6325 }
6326 retval = IBT_SUCCESS; /* done!. */
6327 kmem_free(res_p, len);
6328 ibcm_dec_hca_acc_cnt(hcap);
6329 goto get_comp_pgid_exit;
6330 } else {
6331 IBTF_DPRINTF_L2(cmlog,
6332 "ibt_get_companion_port_gids: "
6333 "Companion PortGIDs not available");
6334 retval = IBT_GIDS_NOT_FOUND;
6335 }
6336 /* Deallocate the memory for 'res_p'. */
6337 kmem_free(res_p, len);
6338
6339 /*
6340 * If we are on MultiSM setup, then we need to lookout
6341 * from that subnet port too.
6342 */
6343 if (multism) {
6344 /* break if already searched both the subnet */
6345 if (multi_sm_loop == 1)
6346 break;
6347
6348 port = (j == 0) ? 1 : 0;
6349 multi_sm_loop = 1;
6350 goto get_comp_for_multism;
6351 } else {
6352 break;
6353 }
6354 }
6355 ibcm_dec_hca_acc_cnt(hcap);
6356
6357 /*
6358 * We may be on dual HCA with dual SM configured system. And
6359 * the input attr GID was visible from second HCA. So in order
6360 * to get the companion portgid we need to re-look from the
6361 * first HCA ports.
6362 */
6363 if ((num_hcas > 1) && (i > 0) && (h_guid != 0) &&
6364 (multi_hca_loop != 1)) {
6365 multi_hca_loop = 1;
6366 goto get_comp_for_multihca;
6367 }
6368 }
6369 if (*num_gids_p == 0)
6370 retval = IBT_GIDS_NOT_FOUND;
6371
6372 get_comp_pgid_exit:
6373 if (guid_array)
6374 ibt_free_hca_list(guid_array, num_hcas);
6375
6376 if ((retval != IBT_SUCCESS) && (*num_gids_p != 0)) {
6377 retval = IBT_SUCCESS;
6378 }
6379
6380 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: done. Status %d, "
6381 "Found %d GIDs", retval, *num_gids_p);
6382
6383 return (retval);
6384 }
6385
6386 /* RDMA IP CM Support routines */
6387 ibt_status_t
6388 ibt_get_src_ip(ibt_srcip_attr_t *sattr, ibt_srcip_info_t **src_info_p,
6389 uint_t *entries_p)
6390 {
6391 ibt_srcip_info_t *s_ip;
6392 ibcm_arp_ip_t *ipp;
6393 ibcm_arp_ibd_insts_t ibds;
6394 uint8_t i, j;
6395 uint_t count;
6396 ibt_status_t retval = IBT_SUCCESS;
6397
6398 IBTF_DPRINTF_L4(cmlog, "ibt_get_src_ip(%p, %p, %p)",
6399 sattr, src_info_p, entries_p);
6400
6401 if (sattr == NULL || entries_p == NULL) {
6402 IBTF_DPRINTF_L3(cmlog, "ibt_get_src_ip: Invalid I/P Args.");
6403 return (IBT_INVALID_PARAM);
6404 }
6405
6406 if (sattr->sip_gid.gid_prefix == 0 || sattr->sip_gid.gid_guid == 0) {
6407 IBTF_DPRINTF_L3(cmlog, "ibt_get_src_ip: Invalid GID.");
6408 return (IBT_INVALID_PARAM);
6409 }
6410
6411 /* TBD: Zoneid */
6412 retval = ibcm_arp_get_ibds(&ibds, sattr->sip_family);
6413 if (retval != IBT_SUCCESS) {
6414 IBTF_DPRINTF_L2(cmlog, "ibt_get_src_ip: ibcm_arp_get_ibds "
6415 "failed to get IBD Instances: ret 0x%x", retval);
6416 goto get_src_ip_end;
6417 }
6418
6419 count = 0;
6420 for (i = 0, ipp = ibds.ibcm_arp_ip; i < ibds.ibcm_arp_ibd_cnt;
6421 i++, ipp++) {
6422 if (ipp->ip_inet_family == AF_UNSPEC)
6423 continue;
6424 if (ipp->ip_port_gid.gid_prefix == sattr->sip_gid.gid_prefix &&
6425 ipp->ip_port_gid.gid_guid == sattr->sip_gid.gid_guid) {
6426 if ((sattr->sip_pkey) &&
6427 (ipp->ip_pkey != sattr->sip_pkey))
6428 continue;
6429
6430 if ((sattr->sip_zoneid != ALL_ZONES) &&
6431 (sattr->sip_zoneid != ipp->ip_zoneid))
6432 continue;
6433
6434 count++;
6435 break;
6436 }
6437 }
6438
6439 if (count) {
6440 /*
6441 * Allocate memory for return buffer, to be freed by
6442 * ibt_free_srcip_info().
6443 */
6444 s_ip = kmem_alloc((count * sizeof (ibt_srcip_info_t)),
6445 KM_SLEEP);
6446
6447 *src_info_p = s_ip;
6448 *entries_p = count;
6449
6450 j = 0;
6451 for (i = 0, ipp = ibds.ibcm_arp_ip; i < ibds.ibcm_arp_ibd_cnt;
6452 i++, ipp++) {
6453 if (ipp->ip_inet_family == AF_UNSPEC)
6454 continue;
6455 if ((ipp->ip_port_gid.gid_prefix ==
6456 sattr->sip_gid.gid_prefix) &&
6457 (ipp->ip_port_gid.gid_guid ==
6458 sattr->sip_gid.gid_guid)) {
6459 if ((sattr->sip_pkey) &&
6460 (ipp->ip_pkey != sattr->sip_pkey))
6461 continue;
6462
6463 if ((sattr->sip_zoneid != ALL_ZONES) &&
6464 (sattr->sip_zoneid != ipp->ip_zoneid))
6465 continue;
6466
6467 s_ip[j].ip_addr.family = ipp->ip_inet_family;
6468 if (s_ip[j].ip_addr.family == AF_INET) {
6469 bcopy(&ipp->ip_cm_sin.sin_addr,
6470 &s_ip[j].ip_addr.un.ip4addr,
6471 sizeof (in_addr_t));
6472 } else if (s_ip[j].ip_addr.family == AF_INET6) {
6473 bcopy(&ipp->ip_cm_sin6.sin6_addr,
6474 &s_ip[j].ip_addr.un.ip6addr,
6475 sizeof (in6_addr_t));
6476 /* TBD: scope_id */
6477 }
6478 IBCM_PRINT_IP("ibt_get_src_ip",
6479 &s_ip[j].ip_addr);
6480 j++;
6481 }
6482 }
6483 } else {
6484 retval = IBT_SRC_IP_NOT_FOUND;
6485 }
6486
6487 get_src_ip_end:
6488 ibcm_arp_free_ibds(&ibds);
6489 return (retval);
6490 }
6491
6492 /*
6493 * ibt_free_srcip_info()
6494 * Free the memory allocated by successful ibt_get_src_ip()
6495 *
6496 * src_info Pointer returned by ibt_get_src_ip().
6497 *
6498 * entries The number of ibt_ip_addr_t entries to free.
6499 */
6500 void
6501 ibt_free_srcip_info(ibt_srcip_info_t *src_info, uint_t entries)
6502 {
6503 IBTF_DPRINTF_L3(cmlog, "ibt_free_srcip_info: "
6504 "Free <%d> entries from 0x%p", entries, src_info);
6505
6506 if ((src_info != NULL) && (entries > 0))
6507 kmem_free(src_info, entries * sizeof (ibt_srcip_info_t));
6508 else
6509 IBTF_DPRINTF_L2(cmlog, "ibt_free_srcip_info: "
6510 "ERROR: NULL buf pointer or ZERO length specified.");
6511 }
6512
6513
6514 ib_svc_id_t
6515 ibt_get_ip_sid(uint8_t protocol_num, in_port_t dst_port)
6516 {
6517 ib_svc_id_t sid;
6518
6519 IBTF_DPRINTF_L4(cmlog, "ibt_get_ip_sid(%X, %lX)", protocol_num,
6520 dst_port);
6521
6522 /*
6523 * If protocol_num is non-zero, then formulate the SID and return it.
6524 * If protocol_num is zero, then we need to assign a locally generated
6525 * IP SID with IB_SID_IPADDR_PREFIX.
6526 */
6527 if (protocol_num) {
6528 sid = IB_SID_IPADDR_PREFIX | protocol_num << 16 | dst_port;
6529 } else {
6530 sid = ibcm_alloc_ip_sid();
6531 }
6532
6533 IBTF_DPRINTF_L3(cmlog, "ibt_get_ip_sid: SID: 0x%016llX", sid);
6534 return (sid);
6535 }
6536
6537 ibt_status_t
6538 ibt_release_ip_sid(ib_svc_id_t ip_sid)
6539 {
6540 IBTF_DPRINTF_L4(cmlog, "ibt_release_ip_sid(%llX)", ip_sid);
6541
6542 if (((ip_sid & IB_SID_IPADDR_PREFIX_MASK) != 0) ||
6543 (!(ip_sid & IB_SID_IPADDR_PREFIX))) {
6544 IBTF_DPRINTF_L2(cmlog, "ibt_release_ip_sid(0x%016llX): ERROR: "
6545 "Called for Non-RDMA IP SID", ip_sid);
6546 return (IBT_INVALID_PARAM);
6547 }
6548
6549 /*
6550 * If protocol_num in ip_sid are all ZEROs, then this SID is allocated
6551 * by IBTF. If not, then the specified ip_sid is invalid.
6552 */
6553 if (ip_sid & IB_SID_IPADDR_IPNUM_MASK) {
6554 IBTF_DPRINTF_L2(cmlog, "ibt_release_ip_sid(0x%016llX): ERROR: "
6555 "Called for Non-IBTF assigned RDMA IP SID", ip_sid);
6556 return (IBT_INVALID_PARAM);
6557 }
6558
6559 ibcm_free_ip_sid(ip_sid);
6560
6561 return (IBT_SUCCESS);
6562 }
6563
6564
6565 uint8_t
6566 ibt_get_ip_protocol_num(ib_svc_id_t sid)
6567 {
6568 return ((sid & IB_SID_IPADDR_IPNUM_MASK) >> 16);
6569 }
6570
6571 in_port_t
6572 ibt_get_ip_dst_port(ib_svc_id_t sid)
6573 {
6574 return (sid & IB_SID_IPADDR_PORTNUM_MASK);
6575 }
6576
6577 ibt_status_t
6578 ibt_format_ip_private_data(ibt_ip_cm_info_t *ip_cm_info,
6579 ibt_priv_data_len_t priv_data_len, void *priv_data_p)
6580 {
6581 ibcm_ip_pvtdata_t ip_data;
6582
6583 IBTF_DPRINTF_L4(cmlog, "ibt_format_ip_private_data(%p, %d, %p)",
6584 ip_cm_info, priv_data_len, priv_data_p);
6585
6586 if ((ip_cm_info == NULL) || (priv_data_p == NULL) ||
6587 (priv_data_len < IBT_IP_HDR_PRIV_DATA_SZ)) {
6588 IBTF_DPRINTF_L2(cmlog, "ibt_format_ip_private_data: ERROR "
6589 "Invalid Inputs.");
6590 return (IBT_INVALID_PARAM);
6591 }
6592
6593 bzero(&ip_data, sizeof (ibcm_ip_pvtdata_t));
6594 ip_data.ip_srcport = ip_cm_info->src_port; /* Source Port */
6595
6596 IBCM_PRINT_IP("format_ip_pvt: src", &ip_cm_info->src_addr);
6597 IBCM_PRINT_IP("format_ip_pvt: dst", &ip_cm_info->dst_addr);
6598 /* IPV = 0x4, if IP-Addr are IPv4 format, else 0x6 for IPv6 */
6599 if (ip_cm_info->src_addr.family == AF_INET) {
6600 ip_data.ip_ipv = IBT_CM_IP_IPV_V4;
6601 ip_data.ip_srcv4 = ip_cm_info->src_addr.un.ip4addr;
6602 ip_data.ip_dstv4 = ip_cm_info->dst_addr.un.ip4addr;
6603 } else if (ip_cm_info->src_addr.family == AF_INET6) {
6604 ip_data.ip_ipv = IBT_CM_IP_IPV_V6;
6605 bcopy(&ip_cm_info->src_addr.un.ip6addr,
6606 &ip_data.ip_srcv6, sizeof (in6_addr_t));
6607 bcopy(&ip_cm_info->dst_addr.un.ip6addr,
6608 &ip_data.ip_dstv6, sizeof (in6_addr_t));
6609 } else {
6610 IBTF_DPRINTF_L2(cmlog, "ibt_format_ip_private_data: ERROR "
6611 "IP Addr needs to be either AF_INET or AF_INET6 family.");
6612 return (IBT_INVALID_PARAM);
6613 }
6614
6615 ip_data.ip_MajV = IBT_CM_IP_MAJ_VER;
6616 ip_data.ip_MinV = IBT_CM_IP_MIN_VER;
6617
6618 bcopy(&ip_data, priv_data_p, IBT_IP_HDR_PRIV_DATA_SZ);
6619
6620 return (IBT_SUCCESS);
6621 }
6622
6623
6624 ibt_status_t
6625 ibt_get_ip_data(ibt_priv_data_len_t priv_data_len, void *priv_data,
6626 ibt_ip_cm_info_t *ip_cm_infop)
6627 {
6628 ibcm_ip_pvtdata_t ip_data;
6629
6630 IBTF_DPRINTF_L4(cmlog, "ibt_get_ip_data(%d, %p, %p)",
6631 priv_data_len, priv_data, ip_cm_infop);
6632
6633 if ((ip_cm_infop == NULL) || (priv_data == NULL) ||
6634 (priv_data_len < IBT_IP_HDR_PRIV_DATA_SZ)) {
6635 IBTF_DPRINTF_L2(cmlog, "ibt_get_ip_data: ERROR Invalid Inputs");
6636 return (IBT_INVALID_PARAM);
6637 }
6638
6639 bcopy(priv_data, &ip_data, IBT_IP_HDR_PRIV_DATA_SZ);
6640 ip_cm_infop->src_port = ip_data.ip_srcport; /* Source Port */
6641
6642 /* IPV = 0x4, if IP Address are IPv4 format, else 0x6 for IPv6 */
6643 if (ip_data.ip_ipv == IBT_CM_IP_IPV_V4) {
6644 /* Copy IPv4 Addr */
6645 ip_cm_infop->src_addr.family = ip_cm_infop->dst_addr.family =
6646 AF_INET;
6647 ip_cm_infop->src_addr.un.ip4addr = ip_data.ip_srcv4;
6648 ip_cm_infop->dst_addr.un.ip4addr = ip_data.ip_dstv4;
6649 } else if (ip_data.ip_ipv == IBT_CM_IP_IPV_V6) {
6650 /* Copy IPv6 Addr */
6651 ip_cm_infop->src_addr.family = ip_cm_infop->dst_addr.family =
6652 AF_INET6;
6653 bcopy(&ip_data.ip_srcv6, &ip_cm_infop->src_addr.un.ip6addr,
6654 sizeof (in6_addr_t));
6655 bcopy(&ip_data.ip_dstv6, &ip_cm_infop->dst_addr.un.ip6addr,
6656 sizeof (in6_addr_t));
6657 } else {
6658 IBTF_DPRINTF_L2(cmlog, "ibt_get_ip_data: ERROR: IP Addr needs"
6659 " to be either AF_INET or AF_INET6 family.");
6660 return (IBT_INVALID_PARAM);
6661 }
6662 IBCM_PRINT_IP("ibt_get_ip_data: src", &ip_cm_infop->src_addr);
6663 IBCM_PRINT_IP("ibt_get_ip_data: dst", &ip_cm_infop->dst_addr);
6664
6665 return (IBT_SUCCESS);
6666 }