1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 /*
29 * ibdm.c
30 *
31 * This file contains the InifiniBand Device Manager (IBDM) support functions.
32 * IB nexus driver will only be the client for the IBDM module.
33 *
34 * IBDM registers with IBTF for HCA arrival/removal notification.
35 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
36 * the IOU's.
37 *
38 * IB nexus driver registers with IBDM to find the information about the
39 * HCA's and IOC's (behind the IOU) present on the IB fabric.
40 */
41
42 #include <sys/sysmacros.h>
43 #include <sys/systm.h>
44 #include <sys/taskq.h>
45 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
46 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
47 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
48 #include <sys/modctl.h>
49
50 /* Function Prototype declarations */
51 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
52 static int ibdm_fini(void);
53 static int ibdm_init(void);
54 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
55 ibdm_hca_list_t *);
56 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
57 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
58 static boolean_t ibdm_is_cisco(ib_guid_t);
59 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
60 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
61 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
62 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
63 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
64 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
65 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
66 ib_guid_t *, ib_guid_t *);
67 static int ibdm_retry_command(ibdm_timeout_cb_args_t *);
68 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
69 static int ibdm_verify_mad_status(ib_mad_hdr_t *);
70 static int ibdm_handle_redirection(ibmf_msg_t *,
71 ibdm_dp_gidinfo_t *, int *);
72 static void ibdm_wait_probe_completion(void);
73 static void ibdm_sweep_fabric(int);
74 static void ibdm_probe_gid_thread(void *);
75 static void ibdm_wakeup_probe_gid_cv(void);
76 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
77 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
78 static void ibdm_update_port_attr(ibdm_port_attr_t *);
79 static void ibdm_handle_hca_attach(ib_guid_t);
80 static void ibdm_handle_srventry_mad(ibmf_msg_t *,
81 ibdm_dp_gidinfo_t *, int *);
82 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
83 static void ibdm_recv_incoming_mad(void *);
84 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
85 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
86 static void ibdm_pkt_timeout_hdlr(void *arg);
87 static void ibdm_initialize_port(ibdm_port_attr_t *);
88 static void ibdm_update_port_pkeys(ibdm_port_attr_t *port);
89 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
90 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *);
91 static void ibdm_alloc_send_buffers(ibmf_msg_t *);
92 static void ibdm_free_send_buffers(ibmf_msg_t *);
93 static void ibdm_handle_hca_detach(ib_guid_t);
94 static void ibdm_handle_port_change_event(ibt_async_event_t *);
95 static int ibdm_fini_port(ibdm_port_attr_t *);
96 static int ibdm_uninit_hca(ibdm_hca_list_t *);
97 static void ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *,
98 ibdm_dp_gidinfo_t *, int *);
99 static void ibdm_handle_iounitinfo(ibmf_handle_t,
100 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
101 static void ibdm_handle_ioc_profile(ibmf_handle_t,
102 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
103 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t,
104 ibt_async_code_t, ibt_async_event_t *);
105 static void ibdm_handle_classportinfo(ibmf_handle_t,
106 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
107 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
108 ibdm_dp_gidinfo_t *);
109
110 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *);
111 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *,
112 ibdm_dp_gidinfo_t *gid_list);
113 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
114 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t,
115 ibdm_dp_gidinfo_t *, int *);
116 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *,
117 ibdm_hca_list_t **);
118 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t,
119 size_t *, ib_guid_t);
120 static int ibdm_get_node_record_by_port(ibmf_saa_handle_t,
121 ib_guid_t, sa_node_record_t **, size_t *);
122 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
123 ib_lid_t);
124 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *,
125 ib_gid_t, ib_gid_t);
126 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t);
127 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
128 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
129 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
130 ibmf_saa_event_details_t *, void *);
131 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
132 ibdm_dp_gidinfo_t *);
133 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
134 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
135 ibdm_dp_gidinfo_t *);
136 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
137 static void ibdm_free_gid_list(ibdm_gid_t *);
138 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
139 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
140 static void ibdm_saa_event_taskq(void *);
141 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
142 static void ibdm_get_next_port(ibdm_hca_list_t **,
143 ibdm_port_attr_t **, int);
144 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
145 ibdm_dp_gidinfo_t *);
146 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
147 ibdm_hca_list_t *);
148 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
149 static void ibdm_saa_handle_new_gid(void *);
150 static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
151 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
152 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
153 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
154 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
155 static ibdm_ioc_info_t *ibdm_handle_prev_iou();
156 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *,
157 int);
158 static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t,
159 ibdm_dp_gidinfo_t **);
160
161 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT;
162 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES;
163 #ifdef DEBUG
164 int ibdm_ignore_saa_event = 0;
165 #endif
166 int ibdm_enumerate_iocs = 0;
167
168 /* Modload support */
169 static struct modlmisc ibdm_modlmisc = {
170 &mod_miscops,
171 "InfiniBand Device Manager"
172 };
173
174 struct modlinkage ibdm_modlinkage = {
175 MODREV_1,
176 (void *)&ibdm_modlmisc,
177 NULL
178 };
179
180 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
181 IBTI_V_CURR,
182 IBT_DM,
183 ibdm_event_hdlr,
184 NULL,
185 "ibdm"
186 };
187
188 /* Global variables */
189 ibdm_t ibdm;
190 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
191 char *ibdm_string = "ibdm";
192
193 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
194 ibdm.ibdm_dp_gidlist_head))
195
196 /*
197 * _init
198 * Loadable module init, called before any other module.
199 * Initialize mutex
200 * Register with IBTF
201 */
202 int
203 _init(void)
204 {
205 int err;
206
207 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);
208
209 if ((err = ibdm_init()) != IBDM_SUCCESS) {
210 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
211 (void) ibdm_fini();
212 return (DDI_FAILURE);
213 }
214
215 if ((err = mod_install(&ibdm_modlinkage)) != 0) {
216 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
217 (void) ibdm_fini();
218 }
219 return (err);
220 }
221
222
223 int
224 _fini(void)
225 {
226 int err;
227
228 if ((err = ibdm_fini()) != IBDM_SUCCESS) {
229 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
230 (void) ibdm_init();
231 return (EBUSY);
232 }
233
234 if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
235 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
236 (void) ibdm_init();
237 }
238 return (err);
239 }
240
241
242 int
243 _info(struct modinfo *modinfop)
244 {
245 return (mod_info(&ibdm_modlinkage, modinfop));
246 }
247
248
249 /*
250 * ibdm_init():
251 * Register with IBTF
252 * Allocate memory for the HCAs
253 * Allocate minor-nodes for the HCAs
254 */
255 static int
256 ibdm_init(void)
257 {
258 int i, hca_count;
259 ib_guid_t *hca_guids;
260 ibt_status_t status;
261
262 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
263 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
264 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
265 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
266 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
267 cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL);
268 mutex_enter(&ibdm.ibdm_mutex);
269 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
270 }
271
272 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
273 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
274 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
275 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
276 "failed %x", status);
277 mutex_exit(&ibdm.ibdm_mutex);
278 return (IBDM_FAILURE);
279 }
280
281 ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
282 mutex_exit(&ibdm.ibdm_mutex);
283 }
284
285
286 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
287 hca_count = ibt_get_hca_list(&hca_guids);
288 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
289 for (i = 0; i < hca_count; i++)
290 (void) ibdm_handle_hca_attach(hca_guids[i]);
291 if (hca_count)
292 ibt_free_hca_list(hca_guids, hca_count);
293
294 mutex_enter(&ibdm.ibdm_mutex);
295 ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
296 mutex_exit(&ibdm.ibdm_mutex);
297 }
298
299 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
300 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
301 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
302 mutex_enter(&ibdm.ibdm_mutex);
303 ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
304 mutex_exit(&ibdm.ibdm_mutex);
305 }
306 return (IBDM_SUCCESS);
307 }
308
309
310 static int
311 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup)
312 {
313 int ii, k, niocs;
314 size_t size;
315 ibdm_gid_t *delete, *head;
316 timeout_id_t timeout_id;
317 ibdm_ioc_info_t *ioc;
318 ibdm_iou_info_t *gl_iou = *ioup;
319
320 ASSERT(mutex_owned(&gid_info->gl_mutex));
321 if (gl_iou == NULL) {
322 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
323 return (0);
324 }
325
326 niocs = gl_iou->iou_info.iou_num_ctrl_slots;
327 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
328 gid_info, niocs);
329
330 for (ii = 0; ii < niocs; ii++) {
331 ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii];
332
333 /* handle the case where an ioc_timeout_id is scheduled */
334 if (ioc->ioc_timeout_id) {
335 timeout_id = ioc->ioc_timeout_id;
336 ioc->ioc_timeout_id = 0;
337 mutex_exit(&gid_info->gl_mutex);
338 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
339 "ioc_timeout_id = 0x%x", timeout_id);
340 if (untimeout(timeout_id) == -1) {
341 IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
342 "untimeout ioc_timeout_id failed");
343 mutex_enter(&gid_info->gl_mutex);
344 return (-1);
345 }
346 mutex_enter(&gid_info->gl_mutex);
347 }
348
349 /* handle the case where an ioc_dc_timeout_id is scheduled */
350 if (ioc->ioc_dc_timeout_id) {
351 timeout_id = ioc->ioc_dc_timeout_id;
352 ioc->ioc_dc_timeout_id = 0;
353 mutex_exit(&gid_info->gl_mutex);
354 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
355 "ioc_dc_timeout_id = 0x%x", timeout_id);
356 if (untimeout(timeout_id) == -1) {
357 IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
358 "untimeout ioc_dc_timeout_id failed");
359 mutex_enter(&gid_info->gl_mutex);
360 return (-1);
361 }
362 mutex_enter(&gid_info->gl_mutex);
363 }
364
365 /* handle the case where serv[k].se_timeout_id is scheduled */
366 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
367 if (ioc->ioc_serv[k].se_timeout_id) {
368 timeout_id = ioc->ioc_serv[k].se_timeout_id;
369 ioc->ioc_serv[k].se_timeout_id = 0;
370 mutex_exit(&gid_info->gl_mutex);
371 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
372 "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
373 k, timeout_id);
374 if (untimeout(timeout_id) == -1) {
375 IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
376 " untimeout se_timeout_id failed");
377 mutex_enter(&gid_info->gl_mutex);
378 return (-1);
379 }
380 mutex_enter(&gid_info->gl_mutex);
381 }
382 }
383
384 /* delete GID list in IOC */
385 head = ioc->ioc_gid_list;
386 while (head) {
387 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
388 "Deleting gid_list struct %p", head);
389 delete = head;
390 head = head->gid_next;
391 kmem_free(delete, sizeof (ibdm_gid_t));
392 }
393 ioc->ioc_gid_list = NULL;
394
395 /* delete ioc_serv */
396 size = ioc->ioc_profile.ioc_service_entries *
397 sizeof (ibdm_srvents_info_t);
398 if (ioc->ioc_serv && size) {
399 kmem_free(ioc->ioc_serv, size);
400 ioc->ioc_serv = NULL;
401 }
402 }
403 /*
404 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information
405 * via the switch during the probe process.
406 */
407 gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE;
408
409 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
410 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
411 kmem_free(gl_iou, size);
412 *ioup = NULL;
413 return (0);
414 }
415
416
417 /*
418 * ibdm_fini():
419 * Un-register with IBTF
420 * De allocate memory for the GID info
421 */
422 static int
423 ibdm_fini()
424 {
425 int ii;
426 ibdm_hca_list_t *hca_list, *temp;
427 ibdm_dp_gidinfo_t *gid_info, *tmp;
428 ibdm_gid_t *head, *delete;
429
430 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");
431
432 mutex_enter(&ibdm.ibdm_hl_mutex);
433 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
434 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
435 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
436 mutex_exit(&ibdm.ibdm_hl_mutex);
437 return (IBDM_FAILURE);
438 }
439 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
440 ibdm.ibdm_ibt_clnt_hdl = NULL;
441 }
442
443 hca_list = ibdm.ibdm_hca_list_head;
444 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
445 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
446 temp = hca_list;
447 hca_list = hca_list->hl_next;
448 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
449 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
450 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
451 "uninit_hca %p failed", temp);
452 mutex_exit(&ibdm.ibdm_hl_mutex);
453 return (IBDM_FAILURE);
454 }
455 }
456 mutex_exit(&ibdm.ibdm_hl_mutex);
457
458 mutex_enter(&ibdm.ibdm_mutex);
459 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
460 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;
461
462 gid_info = ibdm.ibdm_dp_gidlist_head;
463 while (gid_info) {
464 mutex_enter(&gid_info->gl_mutex);
465 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
466 mutex_exit(&gid_info->gl_mutex);
467 ibdm_delete_glhca_list(gid_info);
468
469 tmp = gid_info;
470 gid_info = gid_info->gl_next;
471 mutex_destroy(&tmp->gl_mutex);
472 head = tmp->gl_gid;
473 while (head) {
474 IBTF_DPRINTF_L4("ibdm",
475 "\tibdm_fini: Deleting gid structs");
476 delete = head;
477 head = head->gid_next;
478 kmem_free(delete, sizeof (ibdm_gid_t));
479 }
480 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
481 }
482 mutex_exit(&ibdm.ibdm_mutex);
483
484 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
485 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
486 mutex_destroy(&ibdm.ibdm_mutex);
487 mutex_destroy(&ibdm.ibdm_hl_mutex);
488 mutex_destroy(&ibdm.ibdm_ibnex_mutex);
489 cv_destroy(&ibdm.ibdm_port_settle_cv);
490 }
491 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
492 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
493 cv_destroy(&ibdm.ibdm_probe_cv);
494 cv_destroy(&ibdm.ibdm_busy_cv);
495 }
496 return (IBDM_SUCCESS);
497 }
498
499
500 /*
501 * ibdm_event_hdlr()
502 *
503 * IBDM registers this asynchronous event handler at the time of
504 * ibt_attach. IBDM support the following async events. For other
505 * event, simply returns success.
506 * IBT_HCA_ATTACH_EVENT:
507 * Retrieves the information about all the port that are
508 * present on this HCA, allocates the port attributes
509 * structure and calls IB nexus callback routine with
510 * the port attributes structure as an input argument.
511 * IBT_HCA_DETACH_EVENT:
512 * Retrieves the information about all the ports that are
513 * present on this HCA and calls IB nexus callback with
514 * port guid as an argument
515 * IBT_EVENT_PORT_UP:
516 * Register with IBMF and SA access
517 * Setup IBMF receive callback routine
518 * IBT_EVENT_PORT_DOWN:
519 * Un-Register with IBMF and SA access
520 * Teardown IBMF receive callback routine
521 */
522 /*ARGSUSED*/
523 static void
524 ibdm_event_hdlr(void *clnt_hdl,
525 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
526 {
527 ibdm_hca_list_t *hca_list;
528 ibdm_port_attr_t *port;
529 ibmf_saa_handle_t port_sa_hdl;
530
531 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);
532
533 switch (code) {
534 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */
535 ibdm_handle_hca_attach(event->ev_hca_guid);
536 break;
537
538 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */
539 ibdm_handle_hca_detach(event->ev_hca_guid);
540 mutex_enter(&ibdm.ibdm_ibnex_mutex);
541 if (ibdm.ibdm_ibnex_callback != NULL) {
542 (*ibdm.ibdm_ibnex_callback)((void *)
543 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
544 }
545 mutex_exit(&ibdm.ibdm_ibnex_mutex);
546 break;
547
548 case IBT_EVENT_PORT_UP:
549 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
550 mutex_enter(&ibdm.ibdm_hl_mutex);
551 port = ibdm_get_port_attr(event, &hca_list);
552 if (port == NULL) {
553 IBTF_DPRINTF_L2("ibdm",
554 "\tevent_hdlr: HCA not present");
555 mutex_exit(&ibdm.ibdm_hl_mutex);
556 break;
557 }
558 ibdm_initialize_port(port);
559 hca_list->hl_nports_active++;
560 cv_broadcast(&ibdm.ibdm_port_settle_cv);
561 mutex_exit(&ibdm.ibdm_hl_mutex);
562
563 /* Inform IB nexus driver */
564 mutex_enter(&ibdm.ibdm_ibnex_mutex);
565 if (ibdm.ibdm_ibnex_callback != NULL) {
566 (*ibdm.ibdm_ibnex_callback)((void *)
567 &event->ev_hca_guid, IBDM_EVENT_PORT_UP);
568 }
569 mutex_exit(&ibdm.ibdm_ibnex_mutex);
570 break;
571
572 case IBT_ERROR_PORT_DOWN:
573 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
574 mutex_enter(&ibdm.ibdm_hl_mutex);
575 port = ibdm_get_port_attr(event, &hca_list);
576 if (port == NULL) {
577 IBTF_DPRINTF_L2("ibdm",
578 "\tevent_hdlr: HCA not present");
579 mutex_exit(&ibdm.ibdm_hl_mutex);
580 break;
581 }
582 hca_list->hl_nports_active--;
583 port_sa_hdl = port->pa_sa_hdl;
584 (void) ibdm_fini_port(port);
585 port->pa_state = IBT_PORT_DOWN;
586 cv_broadcast(&ibdm.ibdm_port_settle_cv);
587 mutex_exit(&ibdm.ibdm_hl_mutex);
588 ibdm_reset_all_dgids(port_sa_hdl);
589 break;
590
591 case IBT_PORT_CHANGE_EVENT:
592 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE");
593 if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY)
594 ibdm_handle_port_change_event(event);
595 break;
596
597 default: /* Ignore all other events/errors */
598 break;
599 }
600 }
601
602 static void
603 ibdm_handle_port_change_event(ibt_async_event_t *event)
604 {
605 ibdm_port_attr_t *port;
606 ibdm_hca_list_t *hca_list;
607
608 IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:"
609 " HCA guid %llx", event->ev_hca_guid);
610 mutex_enter(&ibdm.ibdm_hl_mutex);
611 port = ibdm_get_port_attr(event, &hca_list);
612 if (port == NULL) {
613 IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present");
614 mutex_exit(&ibdm.ibdm_hl_mutex);
615 return;
616 }
617 ibdm_update_port_pkeys(port);
618 cv_broadcast(&ibdm.ibdm_port_settle_cv);
619 mutex_exit(&ibdm.ibdm_hl_mutex);
620
621 /* Inform IB nexus driver */
622 mutex_enter(&ibdm.ibdm_ibnex_mutex);
623 if (ibdm.ibdm_ibnex_callback != NULL) {
624 (*ibdm.ibdm_ibnex_callback)((void *)
625 &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE);
626 }
627 mutex_exit(&ibdm.ibdm_ibnex_mutex);
628 }
629
630 /*
631 * ibdm_update_port_pkeys()
632 * Update the pkey table
633 * Update the port attributes
634 */
635 static void
636 ibdm_update_port_pkeys(ibdm_port_attr_t *port)
637 {
638 uint_t nports, size;
639 uint_t pkey_idx, opkey_idx;
640 uint16_t npkeys;
641 ibt_hca_portinfo_t *pinfop;
642 ib_pkey_t pkey;
643 ibdm_pkey_tbl_t *pkey_tbl;
644 ibdm_port_attr_t newport;
645
646 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:");
647 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
648
649 /* Check whether the port is active */
650 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
651 NULL) != IBT_SUCCESS)
652 return;
653
654 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
655 &pinfop, &nports, &size) != IBT_SUCCESS) {
656 /* This should not occur */
657 port->pa_npkeys = 0;
658 port->pa_pkey_tbl = NULL;
659 return;
660 }
661
662 npkeys = pinfop->p_pkey_tbl_sz;
663 pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
664 newport.pa_pkey_tbl = pkey_tbl;
665 newport.pa_ibmf_hdl = port->pa_ibmf_hdl;
666
667 for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) {
668 pkey = pkey_tbl[pkey_idx].pt_pkey =
669 pinfop->p_pkey_tbl[pkey_idx];
670 /*
671 * Is this pkey present in the current table ?
672 */
673 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
674 if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) {
675 pkey_tbl[pkey_idx].pt_qp_hdl =
676 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl;
677 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL;
678 break;
679 }
680 }
681
682 if (opkey_idx == port->pa_npkeys) {
683 pkey = pkey_tbl[pkey_idx].pt_pkey;
684 if (IBDM_INVALID_PKEY(pkey)) {
685 pkey_tbl[pkey_idx].pt_qp_hdl = NULL;
686 continue;
687 }
688 ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx);
689 }
690 }
691
692 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
693 if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) {
694 if (ibdm_port_attr_ibmf_fini(port, opkey_idx) !=
695 IBDM_SUCCESS) {
696 IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: "
697 "ibdm_port_attr_ibmf_fini failed for "
698 "port pkey 0x%x",
699 port->pa_pkey_tbl[opkey_idx].pt_pkey);
700 }
701 }
702 }
703
704 if (port->pa_pkey_tbl != NULL) {
705 kmem_free(port->pa_pkey_tbl,
706 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
707 }
708
709 port->pa_npkeys = npkeys;
710 port->pa_pkey_tbl = pkey_tbl;
711 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
712 port->pa_state = pinfop->p_linkstate;
713 ibt_free_portinfo(pinfop, size);
714 }
715
716 /*
717 * ibdm_initialize_port()
718 * Register with IBMF
719 * Register with SA access
720 * Register a receive callback routine with IBMF. IBMF invokes
721 * this routine whenever a MAD arrives at this port.
722 * Update the port attributes
723 */
724 static void
725 ibdm_initialize_port(ibdm_port_attr_t *port)
726 {
727 int ii;
728 uint_t nports, size;
729 uint_t pkey_idx;
730 ib_pkey_t pkey;
731 ibt_hca_portinfo_t *pinfop;
732 ibmf_register_info_t ibmf_reg;
733 ibmf_saa_subnet_event_args_t event_args;
734
735 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
736 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
737
738 /* Check whether the port is active */
739 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
740 NULL) != IBT_SUCCESS)
741 return;
742
743 if (port->pa_sa_hdl != NULL || port->pa_pkey_tbl != NULL)
744 return;
745
746 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
747 &pinfop, &nports, &size) != IBT_SUCCESS) {
748 /* This should not occur */
749 port->pa_npkeys = 0;
750 port->pa_pkey_tbl = NULL;
751 return;
752 }
753 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
754
755 port->pa_state = pinfop->p_linkstate;
756 port->pa_npkeys = pinfop->p_pkey_tbl_sz;
757 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc(
758 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
759
760 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
761 port->pa_pkey_tbl[pkey_idx].pt_pkey =
762 pinfop->p_pkey_tbl[pkey_idx];
763
764 ibt_free_portinfo(pinfop, size);
765
766 if (ibdm_enumerate_iocs) {
767 event_args.is_event_callback = ibdm_saa_event_cb;
768 event_args.is_event_callback_arg = port;
769 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
770 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
771 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
772 "sa access registration failed");
773 (void) ibdm_fini_port(port);
774 return;
775 }
776
777 ibmf_reg.ir_ci_guid = port->pa_hca_guid;
778 ibmf_reg.ir_port_num = port->pa_port_num;
779 ibmf_reg.ir_client_class = DEV_MGT_MANAGER;
780
781 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
782 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
783 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
784 "IBMF registration failed");
785 (void) ibdm_fini_port(port);
786 return;
787 }
788
789 if (ibmf_setup_async_cb(port->pa_ibmf_hdl,
790 IBMF_QP_HANDLE_DEFAULT,
791 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
792 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
793 "IBMF setup recv cb failed");
794 (void) ibdm_fini_port(port);
795 return;
796 }
797 } else {
798 port->pa_sa_hdl = NULL;
799 port->pa_ibmf_hdl = NULL;
800 }
801
802 for (ii = 0; ii < port->pa_npkeys; ii++) {
803 pkey = port->pa_pkey_tbl[ii].pt_pkey;
804 if (IBDM_INVALID_PKEY(pkey)) {
805 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
806 continue;
807 }
808 ibdm_port_attr_ibmf_init(port, pkey, ii);
809 }
810 }
811
812
813 /*
814 * ibdm_port_attr_ibmf_init:
815 * With IBMF - Alloc QP Handle and Setup Async callback
816 */
817 static void
818 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
819 {
820 int ret;
821
822 if (ibdm_enumerate_iocs == 0) {
823 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
824 return;
825 }
826
827 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
828 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
829 IBMF_SUCCESS) {
830 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
831 "IBMF failed to alloc qp %d", ret);
832 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
833 return;
834 }
835
836 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
837 port->pa_ibmf_hdl);
838
839 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
840 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
841 IBMF_SUCCESS) {
842 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
843 "IBMF setup recv cb failed %d", ret);
844 (void) ibmf_free_qp(port->pa_ibmf_hdl,
845 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
846 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
847 }
848 }
849
850
851 /*
852 * ibdm_get_port_attr()
853 * Get port attributes from HCA guid and port number
854 * Return pointer to ibdm_port_attr_t on Success
855 * and NULL on failure
856 */
857 static ibdm_port_attr_t *
858 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
859 {
860 ibdm_hca_list_t *hca_list;
861 ibdm_port_attr_t *port_attr;
862 int ii;
863
864 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
865 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
866 hca_list = ibdm.ibdm_hca_list_head;
867 while (hca_list) {
868 if (hca_list->hl_hca_guid == event->ev_hca_guid) {
869 for (ii = 0; ii < hca_list->hl_nports; ii++) {
870 port_attr = &hca_list->hl_port_attr[ii];
871 if (port_attr->pa_port_num == event->ev_port) {
872 *retval = hca_list;
873 return (port_attr);
874 }
875 }
876 }
877 hca_list = hca_list->hl_next;
878 }
879 return (NULL);
880 }
881
882
883 /*
884 * ibdm_update_port_attr()
885 * Update the port attributes
886 */
887 static void
888 ibdm_update_port_attr(ibdm_port_attr_t *port)
889 {
890 uint_t nports, size;
891 uint_t pkey_idx;
892 ibt_hca_portinfo_t *portinfop;
893
894 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
895 if (ibt_query_hca_ports(port->pa_hca_hdl,
896 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
897 /* This should not occur */
898 port->pa_npkeys = 0;
899 port->pa_pkey_tbl = NULL;
900 return;
901 }
902 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;
903
904 port->pa_state = portinfop->p_linkstate;
905
906 /*
907 * PKey information in portinfo valid only if port is
908 * ACTIVE. Bail out if not.
909 */
910 if (port->pa_state != IBT_PORT_ACTIVE) {
911 port->pa_npkeys = 0;
912 port->pa_pkey_tbl = NULL;
913 ibt_free_portinfo(portinfop, size);
914 return;
915 }
916
917 port->pa_npkeys = portinfop->p_pkey_tbl_sz;
918 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc(
919 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
920
921 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
922 port->pa_pkey_tbl[pkey_idx].pt_pkey =
923 portinfop->p_pkey_tbl[pkey_idx];
924 }
925 ibt_free_portinfo(portinfop, size);
926 }
927
928
929 /*
930 * ibdm_handle_hca_attach()
931 */
932 static void
933 ibdm_handle_hca_attach(ib_guid_t hca_guid)
934 {
935 uint_t size;
936 uint_t ii, nports;
937 ibt_status_t status;
938 ibt_hca_hdl_t hca_hdl;
939 ibt_hca_attr_t *hca_attr;
940 ibdm_hca_list_t *hca_list, *temp;
941 ibdm_port_attr_t *port_attr;
942 ibt_hca_portinfo_t *portinfop;
943
944 IBTF_DPRINTF_L4("ibdm",
945 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);
946
947 /* open the HCA first */
948 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
949 &hca_hdl)) != IBT_SUCCESS) {
950 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
951 "open_hca failed, status 0x%x", status);
952 return;
953 }
954
955 hca_attr = (ibt_hca_attr_t *)
956 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
957 /* ibt_query_hca always returns IBT_SUCCESS */
958 (void) ibt_query_hca(hca_hdl, hca_attr);
959
960 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
961 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
962 hca_attr->hca_version_id, hca_attr->hca_nports);
963
964 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
965 &size)) != IBT_SUCCESS) {
966 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
967 "ibt_query_hca_ports failed, status 0x%x", status);
968 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
969 (void) ibt_close_hca(hca_hdl);
970 return;
971 }
972 hca_list = (ibdm_hca_list_t *)
973 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
974 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
975 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
976 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
977 hca_list->hl_nports = hca_attr->hca_nports;
978 hca_list->hl_attach_time = gethrtime();
979 hca_list->hl_hca_hdl = hca_hdl;
980
981 /*
982 * Init a dummy port attribute for the HCA node
983 * This is for Per-HCA Node. Initialize port_attr :
984 * hca_guid & port_guid -> hca_guid
985 * npkeys, pkey_tbl is NULL
986 * port_num, sn_prefix is 0
987 * vendorid, product_id, dev_version from HCA
988 * pa_state is IBT_PORT_ACTIVE
989 */
990 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
991 sizeof (ibdm_port_attr_t), KM_SLEEP);
992 port_attr = hca_list->hl_hca_port_attr;
993 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
994 port_attr->pa_productid = hca_attr->hca_device_id;
995 port_attr->pa_dev_version = hca_attr->hca_version_id;
996 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
997 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
998 port_attr->pa_port_guid = hca_attr->hca_node_guid;
999 port_attr->pa_state = IBT_PORT_ACTIVE;
1000
1001
1002 for (ii = 0; ii < nports; ii++) {
1003 port_attr = &hca_list->hl_port_attr[ii];
1004 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
1005 port_attr->pa_productid = hca_attr->hca_device_id;
1006 port_attr->pa_dev_version = hca_attr->hca_version_id;
1007 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
1008 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
1009 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid;
1010 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix;
1011 port_attr->pa_port_num = portinfop[ii].p_port_num;
1012 port_attr->pa_state = portinfop[ii].p_linkstate;
1013
1014 /*
1015 * Register with IBMF, SA access when the port is in
1016 * ACTIVE state. Also register a callback routine
1017 * with IBMF to receive incoming DM MAD's.
1018 * The IBDM event handler takes care of registration of
1019 * port which are not active.
1020 */
1021 IBTF_DPRINTF_L4("ibdm",
1022 "\thandle_hca_attach: port guid %llx Port state 0x%x",
1023 port_attr->pa_port_guid, portinfop[ii].p_linkstate);
1024
1025 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
1026 mutex_enter(&ibdm.ibdm_hl_mutex);
1027 hca_list->hl_nports_active++;
1028 ibdm_initialize_port(port_attr);
1029 cv_broadcast(&ibdm.ibdm_port_settle_cv);
1030 mutex_exit(&ibdm.ibdm_hl_mutex);
1031 }
1032 }
1033 mutex_enter(&ibdm.ibdm_hl_mutex);
1034 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
1035 if (temp->hl_hca_guid == hca_guid) {
1036 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
1037 "already seen by IBDM", hca_guid);
1038 mutex_exit(&ibdm.ibdm_hl_mutex);
1039 (void) ibdm_uninit_hca(hca_list);
1040 return;
1041 }
1042 }
1043 ibdm.ibdm_hca_count++;
1044 if (ibdm.ibdm_hca_list_head == NULL) {
1045 ibdm.ibdm_hca_list_head = hca_list;
1046 ibdm.ibdm_hca_list_tail = hca_list;
1047 } else {
1048 ibdm.ibdm_hca_list_tail->hl_next = hca_list;
1049 ibdm.ibdm_hca_list_tail = hca_list;
1050 }
1051 mutex_exit(&ibdm.ibdm_hl_mutex);
1052 mutex_enter(&ibdm.ibdm_ibnex_mutex);
1053 if (ibdm.ibdm_ibnex_callback != NULL) {
1054 (*ibdm.ibdm_ibnex_callback)((void *)
1055 &hca_guid, IBDM_EVENT_HCA_ADDED);
1056 }
1057 mutex_exit(&ibdm.ibdm_ibnex_mutex);
1058
1059 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1060 ibt_free_portinfo(portinfop, size);
1061 }
1062
1063
1064 /*
1065 * ibdm_handle_hca_detach()
1066 */
1067 static void
1068 ibdm_handle_hca_detach(ib_guid_t hca_guid)
1069 {
1070 ibdm_hca_list_t *head, *prev = NULL;
1071 size_t len;
1072 ibdm_dp_gidinfo_t *gidinfo;
1073 ibdm_port_attr_t *port_attr;
1074 int i;
1075
1076 IBTF_DPRINTF_L4("ibdm",
1077 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);
1078
1079 /* Make sure no probes are running */
1080 mutex_enter(&ibdm.ibdm_mutex);
1081 while (ibdm.ibdm_busy & IBDM_BUSY)
1082 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1083 ibdm.ibdm_busy |= IBDM_BUSY;
1084 mutex_exit(&ibdm.ibdm_mutex);
1085
1086 mutex_enter(&ibdm.ibdm_hl_mutex);
1087 head = ibdm.ibdm_hca_list_head;
1088 while (head) {
1089 if (head->hl_hca_guid == hca_guid) {
1090 if (prev == NULL)
1091 ibdm.ibdm_hca_list_head = head->hl_next;
1092 else
1093 prev->hl_next = head->hl_next;
1094 if (ibdm.ibdm_hca_list_tail == head)
1095 ibdm.ibdm_hca_list_tail = prev;
1096 ibdm.ibdm_hca_count--;
1097 break;
1098 }
1099 prev = head;
1100 head = head->hl_next;
1101 }
1102 mutex_exit(&ibdm.ibdm_hl_mutex);
1103 if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
1104 (void) ibdm_handle_hca_attach(hca_guid);
1105
1106 #ifdef DEBUG
1107 if (ibdm_enumerate_iocs == 0) {
1108 ASSERT(ibdm.ibdm_dp_gidlist_head == NULL);
1109 }
1110 #endif
1111
1112 /*
1113 * Now clean up the HCA lists in the gidlist.
1114 */
1115 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
1116 gidinfo->gl_next) {
1117 prev = NULL;
1118 head = gidinfo->gl_hca_list;
1119 while (head) {
1120 if (head->hl_hca_guid == hca_guid) {
1121 if (prev == NULL)
1122 gidinfo->gl_hca_list =
1123 head->hl_next;
1124 else
1125 prev->hl_next = head->hl_next;
1126 for (i = 0; i < head->hl_nports; i++) {
1127 port_attr = &head->hl_port_attr[i];
1128 if (port_attr->pa_pkey_tbl != NULL)
1129 kmem_free(
1130 port_attr->pa_pkey_tbl,
1131 port_attr->pa_npkeys *
1132 sizeof (ibdm_pkey_tbl_t));
1133 }
1134 len = sizeof (ibdm_hca_list_t) +
1135 (head->hl_nports *
1136 sizeof (ibdm_port_attr_t));
1137 kmem_free(head, len);
1138
1139 break;
1140 }
1141 prev = head;
1142 head = head->hl_next;
1143 }
1144 }
1145
1146 mutex_enter(&ibdm.ibdm_mutex);
1147 ibdm.ibdm_busy &= ~IBDM_BUSY;
1148 cv_broadcast(&ibdm.ibdm_busy_cv);
1149 mutex_exit(&ibdm.ibdm_mutex);
1150 }
1151
1152
1153 static int
1154 ibdm_uninit_hca(ibdm_hca_list_t *head)
1155 {
1156 int ii;
1157 ibdm_port_attr_t *port_attr;
1158
1159 for (ii = 0; ii < head->hl_nports; ii++) {
1160 port_attr = &head->hl_port_attr[ii];
1161 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
1162 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
1163 "ibdm_fini_port() failed", head, ii);
1164 return (IBDM_FAILURE);
1165 }
1166 }
1167 if (head->hl_hca_hdl)
1168 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) {
1169 IBTF_DPRINTF_L2("ibdm", "uninit_hca: "
1170 "ibt_close_hca() failed");
1171 return (IBDM_FAILURE);
1172 }
1173 kmem_free(head->hl_port_attr,
1174 head->hl_nports * sizeof (ibdm_port_attr_t));
1175 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
1176 kmem_free(head, sizeof (ibdm_hca_list_t));
1177 return (IBDM_SUCCESS);
1178 }
1179
1180
1181 /*
1182 * For each port on the HCA,
1183 * 1) Teardown IBMF receive callback function
1184 * 2) Unregister with IBMF
1185 * 3) Unregister with SA access
1186 */
1187 static int
1188 ibdm_fini_port(ibdm_port_attr_t *port_attr)
1189 {
1190 int ii, ibmf_status;
1191
1192 for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
1193 if (port_attr->pa_pkey_tbl == NULL)
1194 break;
1195 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
1196 continue;
1197 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
1198 IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1199 "ibdm_port_attr_ibmf_fini failed for "
1200 "port pkey 0x%x", ii);
1201 return (IBDM_FAILURE);
1202 }
1203 }
1204
1205 if (port_attr->pa_ibmf_hdl) {
1206 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1207 IBMF_QP_HANDLE_DEFAULT, 0);
1208 if (ibmf_status != IBMF_SUCCESS) {
1209 IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1210 "ibmf_tear_down_async_cb failed %d", ibmf_status);
1211 return (IBDM_FAILURE);
1212 }
1213
1214 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
1215 if (ibmf_status != IBMF_SUCCESS) {
1216 IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1217 "ibmf_unregister failed %d", ibmf_status);
1218 return (IBDM_FAILURE);
1219 }
1220
1221 port_attr->pa_ibmf_hdl = NULL;
1222 }
1223
1224 if (port_attr->pa_sa_hdl) {
1225 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
1226 if (ibmf_status != IBMF_SUCCESS) {
1227 IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1228 "ibmf_sa_session_close failed %d", ibmf_status);
1229 return (IBDM_FAILURE);
1230 }
1231 port_attr->pa_sa_hdl = NULL;
1232 }
1233
1234 if (port_attr->pa_pkey_tbl != NULL) {
1235 kmem_free(port_attr->pa_pkey_tbl,
1236 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
1237 port_attr->pa_pkey_tbl = NULL;
1238 port_attr->pa_npkeys = 0;
1239 }
1240
1241 return (IBDM_SUCCESS);
1242 }
1243
1244
1245 /*
1246 * ibdm_port_attr_ibmf_fini:
1247 * With IBMF - Tear down Async callback and free QP Handle
1248 */
1249 static int
1250 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
1251 {
1252 int ibmf_status;
1253
1254 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");
1255
1256 if (ibdm_enumerate_iocs == 0) {
1257 ASSERT(port_attr->pa_pkey_tbl[ii].pt_qp_hdl == NULL);
1258 return (IBDM_SUCCESS);
1259 }
1260
1261 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
1262 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1263 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1264 if (ibmf_status != IBMF_SUCCESS) {
1265 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1266 "ibmf_tear_down_async_cb failed %d", ibmf_status);
1267 return (IBDM_FAILURE);
1268 }
1269 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
1270 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1271 if (ibmf_status != IBMF_SUCCESS) {
1272 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1273 "ibmf_free_qp failed %d", ibmf_status);
1274 return (IBDM_FAILURE);
1275 }
1276 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
1277 }
1278 return (IBDM_SUCCESS);
1279 }
1280
1281
1282 /*
1283 * ibdm_gid_decr_pending:
1284 * decrement gl_pending_cmds. If zero wakeup sleeping threads
1285 */
1286 static void
1287 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
1288 {
1289 mutex_enter(&ibdm.ibdm_mutex);
1290 mutex_enter(&gidinfo->gl_mutex);
1291 if (--gidinfo->gl_pending_cmds == 0) {
1292 /*
1293 * Handle DGID getting removed.
1294 */
1295 if (gidinfo->gl_disconnected) {
1296 mutex_exit(&gidinfo->gl_mutex);
1297 mutex_exit(&ibdm.ibdm_mutex);
1298
1299 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
1300 "gidinfo %p hot removal", gidinfo);
1301 ibdm_delete_gidinfo(gidinfo);
1302
1303 mutex_enter(&ibdm.ibdm_mutex);
1304 ibdm.ibdm_ngid_probes_in_progress--;
1305 ibdm_wait_probe_completion();
1306 mutex_exit(&ibdm.ibdm_mutex);
1307 return;
1308 }
1309 mutex_exit(&gidinfo->gl_mutex);
1310 mutex_exit(&ibdm.ibdm_mutex);
1311 ibdm_notify_newgid_iocs(gidinfo);
1312 mutex_enter(&ibdm.ibdm_mutex);
1313 mutex_enter(&gidinfo->gl_mutex);
1314
1315 ibdm.ibdm_ngid_probes_in_progress--;
1316 ibdm_wait_probe_completion();
1317 }
1318 mutex_exit(&gidinfo->gl_mutex);
1319 mutex_exit(&ibdm.ibdm_mutex);
1320 }
1321
1322
1323 /*
1324 * ibdm_wait_probe_completion:
1325 * wait for probing to complete
1326 */
1327 static void
1328 ibdm_wait_probe_completion(void)
1329 {
1330 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1331 if (ibdm.ibdm_ngid_probes_in_progress) {
1332 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete");
1333 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
1334 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
1335 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
1336 }
1337 }
1338
1339
1340 /*
1341 * ibdm_wait_cisco_probe_completion:
1342 * wait for the reply from the Cisco FC GW switch after a setclassportinfo
1343 * request is sent. This wait can be achieved on each gid.
1344 */
1345 static void
1346 ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo)
1347 {
1348 ASSERT(MUTEX_HELD(&gidinfo->gl_mutex));
1349 IBTF_DPRINTF_L4("ibdm", "\twait for cisco probe complete");
1350 gidinfo->gl_flag |= IBDM_CISCO_PROBE;
1351 while (gidinfo->gl_flag & IBDM_CISCO_PROBE)
1352 cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex);
1353 }
1354
1355
1356 /*
1357 * ibdm_wakeup_probe_gid_cv:
1358 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
1359 */
1360 static void
1361 ibdm_wakeup_probe_gid_cv(void)
1362 {
1363 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1364 if (!ibdm.ibdm_ngid_probes_in_progress) {
1365 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
1366 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
1367 cv_broadcast(&ibdm.ibdm_probe_cv);
1368 }
1369
1370 }
1371
1372
1373 /*
1374 * ibdm_sweep_fabric(reprobe_flag)
1375 * Find all possible Managed IOU's and their IOC's that are visible
1376 * to the host. The algorithm used is as follows
1377 *
1378 * Send a "bus walk" request for each port on the host HCA to SA access
1379 * SA returns complete set of GID's that are reachable from
1380 * source port. This is done in parallel.
1381 *
1382 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE
1383 *
1384 * Sort the GID list and eliminate duplicate GID's
1385 * 1) Use DGID for sorting
1386 * 2) use PortGuid for sorting
1387 * Send SA query to retrieve NodeRecord and
1388 * extract PortGuid from that.
1389 *
1390 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
1391 * support DM MAD's
1392 * Send a "Portinfo" query to get the port capabilities and
1393 * then check for DM MAD's support
1394 *
1395 * Send "ClassPortInfo" request for all the GID's in parallel,
1396 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
1397 * cv_signal to complete.
1398 *
1399 * When DM agent on the remote GID sends back the response, IBMF
1400 * invokes DM callback routine.
1401 *
1402 * If the response is proper, send "IOUnitInfo" request and set
1403 * GID state to IBDM_GET_IOUNITINFO.
1404 *
1405 * If the response is proper, send "IocProfileInfo" request to
1406 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
1407 *
1408 * Send request to get Service entries simultaneously
1409 *
1410 * Signal the waiting thread when received response for all the commands.
1411 *
1412 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error
1413 * response during the probing period.
1414 *
1415 * Note:
1416 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
1417 * keep track of number commands in progress at any point of time.
1418 * MAD transaction ID is used to identify a particular GID
1419 * TBD: Consider registering the IBMF receive callback on demand
1420 *
1421 * Note: This routine must be called with ibdm.ibdm_mutex held
1422 * TBD: Re probe the failure GID (for certain failures) when requested
1423 * for fabric sweep next time
1424 *
1425 * Parameters : If reprobe_flag is set, All IOCs will be reprobed.
1426 */
1427 static void
1428 ibdm_sweep_fabric(int reprobe_flag)
1429 {
1430 int ii;
1431 int new_paths = 0;
1432 uint8_t niocs;
1433 taskqid_t tid;
1434 ibdm_ioc_info_t *ioc;
1435 ibdm_hca_list_t *hca_list = NULL;
1436 ibdm_port_attr_t *port = NULL;
1437 ibdm_dp_gidinfo_t *gid_info;
1438
1439 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
1440 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1441
1442 /*
1443 * Check whether a sweep already in progress. If so, just
1444 * wait for the fabric sweep to complete
1445 */
1446 while (ibdm.ibdm_busy & IBDM_BUSY)
1447 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1448 ibdm.ibdm_busy |= IBDM_BUSY;
1449 mutex_exit(&ibdm.ibdm_mutex);
1450
1451 ibdm_dump_sweep_fabric_timestamp(0);
1452
1453 /* Rescan the GID list for any removed GIDs for reprobe */
1454 if (reprobe_flag)
1455 ibdm_rescan_gidlist(NULL);
1456
1457 /*
1458 * Get list of all the ports reachable from the local known HCA
1459 * ports which are active
1460 */
1461 mutex_enter(&ibdm.ibdm_hl_mutex);
1462 for (ibdm_get_next_port(&hca_list, &port, 1); port;
1463 ibdm_get_next_port(&hca_list, &port, 1)) {
1464 /*
1465 * Get PATHS to all the reachable ports from
1466 * SGID and update the global ibdm structure.
1467 */
1468 new_paths = ibdm_get_reachable_ports(port, hca_list);
1469 ibdm.ibdm_ngids += new_paths;
1470 }
1471 mutex_exit(&ibdm.ibdm_hl_mutex);
1472
1473 mutex_enter(&ibdm.ibdm_mutex);
1474 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
1475 mutex_exit(&ibdm.ibdm_mutex);
1476
1477 /* Send a request to probe GIDs asynchronously. */
1478 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1479 gid_info = gid_info->gl_next) {
1480 mutex_enter(&gid_info->gl_mutex);
1481 gid_info->gl_reprobe_flag = reprobe_flag;
1482 mutex_exit(&gid_info->gl_mutex);
1483
1484 /* process newly encountered GIDs */
1485 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
1486 (void *)gid_info, TQ_NOSLEEP);
1487 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
1488 " taskq_id = %x", gid_info, tid);
1489 /* taskq failed to dispatch call it directly */
1490 if (tid == NULL)
1491 ibdm_probe_gid_thread((void *)gid_info);
1492 }
1493
1494 mutex_enter(&ibdm.ibdm_mutex);
1495 ibdm_wait_probe_completion();
1496
1497 /*
1498 * Update the properties, if reprobe_flag is set
1499 * Skip if gl_reprobe_flag is set, this will be
1500 * a re-inserted / new GID, for which notifications
1501 * have already been send.
1502 */
1503 if (reprobe_flag) {
1504 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1505 gid_info = gid_info->gl_next) {
1506 if (gid_info->gl_iou == NULL)
1507 continue;
1508 if (gid_info->gl_reprobe_flag) {
1509 gid_info->gl_reprobe_flag = 0;
1510 continue;
1511 }
1512
1513 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1514 for (ii = 0; ii < niocs; ii++) {
1515 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1516 if (ioc)
1517 ibdm_reprobe_update_port_srv(ioc,
1518 gid_info);
1519 }
1520 }
1521 } else if (ibdm.ibdm_prev_iou) {
1522 ibdm_ioc_info_t *ioc_list;
1523
1524 /*
1525 * Get the list of IOCs which have changed.
1526 * If any IOCs have changed, Notify IBNexus
1527 */
1528 ibdm.ibdm_prev_iou = 0;
1529 ioc_list = ibdm_handle_prev_iou();
1530 if (ioc_list) {
1531 if (ibdm.ibdm_ibnex_callback != NULL) {
1532 (*ibdm.ibdm_ibnex_callback)(
1533 (void *)ioc_list,
1534 IBDM_EVENT_IOC_PROP_UPDATE);
1535 }
1536 }
1537 }
1538
1539 ibdm_dump_sweep_fabric_timestamp(1);
1540
1541 ibdm.ibdm_busy &= ~IBDM_BUSY;
1542 cv_broadcast(&ibdm.ibdm_busy_cv);
1543 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
1544 }
1545
1546
1547 /*
1548 * ibdm_is_cisco:
1549 * Check if this is a Cisco device or not.
1550 */
1551 static boolean_t
1552 ibdm_is_cisco(ib_guid_t guid)
1553 {
1554 if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID)
1555 return (B_TRUE);
1556 return (B_FALSE);
1557 }
1558
1559
1560 /*
1561 * ibdm_is_cisco_switch:
1562 * Check if this switch is a CISCO switch or not.
1563 * Note that if this switch is already activated, ibdm_is_cisco_switch()
1564 * returns B_FALSE not to re-activate it again.
1565 */
1566 static boolean_t
1567 ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info)
1568 {
1569 int company_id, device_id;
1570 ASSERT(gid_info != 0);
1571 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
1572
1573 /*
1574 * If this switch is already activated, don't re-activate it.
1575 */
1576 if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE)
1577 return (B_FALSE);
1578
1579 /*
1580 * Check if this switch is a Cisco FC GW or not.
1581 * Use the node guid (the OUI part) instead of the vendor id
1582 * since the vendor id is zero in practice.
1583 */
1584 company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT;
1585 device_id = gid_info->gl_devid;
1586
1587 if (company_id == IBDM_CISCO_COMPANY_ID &&
1588 device_id == IBDM_CISCO_DEVICE_ID)
1589 return (B_TRUE);
1590 return (B_FALSE);
1591 }
1592
1593
1594 /*
1595 * ibdm_probe_gid_thread:
1596 * thread that does the actual work for sweeping the fabric
1597 * for a given GID
1598 */
1599 static void
1600 ibdm_probe_gid_thread(void *args)
1601 {
1602 int reprobe_flag;
1603 ib_guid_t node_guid;
1604 ib_guid_t port_guid;
1605 ibdm_dp_gidinfo_t *gid_info;
1606
1607 gid_info = (ibdm_dp_gidinfo_t *)args;
1608 reprobe_flag = gid_info->gl_reprobe_flag;
1609 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
1610 gid_info, reprobe_flag);
1611 ASSERT(gid_info != NULL);
1612 ASSERT(gid_info->gl_pending_cmds == 0);
1613
1614 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
1615 reprobe_flag == 0) {
1616 /*
1617 * This GID may have been already probed. Send
1618 * in a CLP to check if IOUnitInfo changed?
1619 * Explicitly set gl_reprobe_flag to 0 so that
1620 * IBnex is not notified on completion
1621 */
1622 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
1623 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
1624 "get new IOCs information");
1625 mutex_enter(&gid_info->gl_mutex);
1626 gid_info->gl_pending_cmds++;
1627 gid_info->gl_state = IBDM_GET_IOUNITINFO;
1628 gid_info->gl_reprobe_flag = 0;
1629 mutex_exit(&gid_info->gl_mutex);
1630 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
1631 mutex_enter(&gid_info->gl_mutex);
1632 --gid_info->gl_pending_cmds;
1633 mutex_exit(&gid_info->gl_mutex);
1634 mutex_enter(&ibdm.ibdm_mutex);
1635 --ibdm.ibdm_ngid_probes_in_progress;
1636 ibdm_wakeup_probe_gid_cv();
1637 mutex_exit(&ibdm.ibdm_mutex);
1638 }
1639 } else {
1640 mutex_enter(&ibdm.ibdm_mutex);
1641 --ibdm.ibdm_ngid_probes_in_progress;
1642 ibdm_wakeup_probe_gid_cv();
1643 mutex_exit(&ibdm.ibdm_mutex);
1644 }
1645 return;
1646 } else if (reprobe_flag && gid_info->gl_state ==
1647 IBDM_GID_PROBING_COMPLETE) {
1648 /*
1649 * Reprobe all IOCs for the GID which has completed
1650 * probe. Skip other port GIDs to same IOU.
1651 * Explicitly set gl_reprobe_flag to 0 so that
1652 * IBnex is not notified on completion
1653 */
1654 ibdm_ioc_info_t *ioc_info;
1655 uint8_t niocs, ii;
1656
1657 ASSERT(gid_info->gl_iou);
1658 mutex_enter(&gid_info->gl_mutex);
1659 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1660 gid_info->gl_state = IBDM_GET_IOC_DETAILS;
1661 gid_info->gl_pending_cmds += niocs;
1662 gid_info->gl_reprobe_flag = 0;
1663 mutex_exit(&gid_info->gl_mutex);
1664 for (ii = 0; ii < niocs; ii++) {
1665 uchar_t slot_info;
1666 ib_dm_io_unitinfo_t *giou_info;
1667
1668 /*
1669 * Check whether IOC is present in the slot
1670 * Series of nibbles (in the field
1671 * iou_ctrl_list) represents a slot in the
1672 * IOU.
1673 * Byte format: 76543210
1674 * Bits 0-3 of first byte represent Slot 2
1675 * bits 4-7 of first byte represent slot 1,
1676 * bits 0-3 of second byte represent slot 4
1677 * and so on
1678 * Each 4-bit nibble has the following meaning
1679 * 0x0 : IOC not installed
1680 * 0x1 : IOC is present
1681 * 0xf : Slot does not exist
1682 * and all other values are reserved.
1683 */
1684 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1685 giou_info = &gid_info->gl_iou->iou_info;
1686 slot_info = giou_info->iou_ctrl_list[(ii/2)];
1687 if ((ii % 2) == 0)
1688 slot_info = (slot_info >> 4);
1689
1690 if ((slot_info & 0xf) != 1) {
1691 ioc_info->ioc_state =
1692 IBDM_IOC_STATE_PROBE_FAILED;
1693 ibdm_gid_decr_pending(gid_info);
1694 continue;
1695 }
1696
1697 if (ibdm_send_ioc_profile(gid_info, ii) !=
1698 IBDM_SUCCESS) {
1699 ibdm_gid_decr_pending(gid_info);
1700 }
1701 }
1702
1703 return;
1704 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
1705 mutex_enter(&ibdm.ibdm_mutex);
1706 --ibdm.ibdm_ngid_probes_in_progress;
1707 ibdm_wakeup_probe_gid_cv();
1708 mutex_exit(&ibdm.ibdm_mutex);
1709 return;
1710 }
1711
1712 /*
1713 * Check whether the destination GID supports DM agents. If
1714 * not, stop probing the GID and continue with the next GID
1715 * in the list.
1716 */
1717 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
1718 mutex_enter(&gid_info->gl_mutex);
1719 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1720 gid_info->gl_is_dm_capable = B_FALSE;
1721 mutex_exit(&gid_info->gl_mutex);
1722 ibdm_delete_glhca_list(gid_info);
1723 mutex_enter(&ibdm.ibdm_mutex);
1724 --ibdm.ibdm_ngid_probes_in_progress;
1725 ibdm_wakeup_probe_gid_cv();
1726 mutex_exit(&ibdm.ibdm_mutex);
1727 return;
1728 }
1729
1730 /*
1731 * This GID is Device management capable
1732 */
1733 mutex_enter(&gid_info->gl_mutex);
1734 gid_info->gl_is_dm_capable = B_TRUE;
1735 mutex_exit(&gid_info->gl_mutex);
1736
1737 /* Get the nodeguid and portguid of the port */
1738 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
1739 &node_guid, &port_guid) != IBDM_SUCCESS) {
1740 mutex_enter(&gid_info->gl_mutex);
1741 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1742 mutex_exit(&gid_info->gl_mutex);
1743 ibdm_delete_glhca_list(gid_info);
1744 mutex_enter(&ibdm.ibdm_mutex);
1745 --ibdm.ibdm_ngid_probes_in_progress;
1746 ibdm_wakeup_probe_gid_cv();
1747 mutex_exit(&ibdm.ibdm_mutex);
1748 return;
1749 }
1750
1751 /*
1752 * Check whether we already knew about this NodeGuid
1753 * If so, do not probe the GID and continue with the
1754 * next GID in the gid list. Set the GID state to
1755 * probing done.
1756 */
1757 mutex_enter(&ibdm.ibdm_mutex);
1758 gid_info->gl_nodeguid = node_guid;
1759 gid_info->gl_portguid = port_guid;
1760 if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
1761 mutex_exit(&ibdm.ibdm_mutex);
1762 mutex_enter(&gid_info->gl_mutex);
1763 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
1764 mutex_exit(&gid_info->gl_mutex);
1765 ibdm_delete_glhca_list(gid_info);
1766 mutex_enter(&ibdm.ibdm_mutex);
1767 --ibdm.ibdm_ngid_probes_in_progress;
1768 ibdm_wakeup_probe_gid_cv();
1769 mutex_exit(&ibdm.ibdm_mutex);
1770 return;
1771 }
1772 ibdm_add_to_gl_gid(gid_info, gid_info);
1773 mutex_exit(&ibdm.ibdm_mutex);
1774
1775 /*
1776 * New or reinserted GID : Enable notification to IBnex
1777 */
1778 mutex_enter(&gid_info->gl_mutex);
1779 gid_info->gl_reprobe_flag = 1;
1780
1781 /*
1782 * A Cisco FC GW needs the special handling to get IOUnitInfo.
1783 */
1784 if (ibdm_is_cisco_switch(gid_info)) {
1785 gid_info->gl_pending_cmds++;
1786 gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
1787 mutex_exit(&gid_info->gl_mutex);
1788
1789 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
1790 mutex_enter(&gid_info->gl_mutex);
1791 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1792 --gid_info->gl_pending_cmds;
1793 mutex_exit(&gid_info->gl_mutex);
1794
1795 /* free the hca_list on this gid_info */
1796 ibdm_delete_glhca_list(gid_info);
1797
1798 mutex_enter(&ibdm.ibdm_mutex);
1799 --ibdm.ibdm_ngid_probes_in_progress;
1800 ibdm_wakeup_probe_gid_cv();
1801 mutex_exit(&ibdm.ibdm_mutex);
1802
1803 return;
1804 }
1805
1806 mutex_enter(&gid_info->gl_mutex);
1807 ibdm_wait_cisco_probe_completion(gid_info);
1808
1809 IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: "
1810 "CISCO Wakeup signal received");
1811 }
1812
1813 /* move on to the 'GET_CLASSPORTINFO' stage */
1814 gid_info->gl_pending_cmds++;
1815 gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
1816 mutex_exit(&gid_info->gl_mutex);
1817
1818 IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: "
1819 "%d: gid_info %p gl_state %d pending_cmds %d",
1820 __LINE__, gid_info, gid_info->gl_state,
1821 gid_info->gl_pending_cmds);
1822
1823 /*
1824 * Send ClassPortInfo request to the GID asynchronously.
1825 */
1826 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
1827
1828 mutex_enter(&gid_info->gl_mutex);
1829 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1830 --gid_info->gl_pending_cmds;
1831 mutex_exit(&gid_info->gl_mutex);
1832
1833 /* free the hca_list on this gid_info */
1834 ibdm_delete_glhca_list(gid_info);
1835
1836 mutex_enter(&ibdm.ibdm_mutex);
1837 --ibdm.ibdm_ngid_probes_in_progress;
1838 ibdm_wakeup_probe_gid_cv();
1839 mutex_exit(&ibdm.ibdm_mutex);
1840
1841 return;
1842 }
1843 }
1844
1845
1846 /*
1847 * ibdm_check_dest_nodeguid
1848 * Searches for the NodeGuid in the GID list
1849 * Returns matching gid_info if found and otherwise NULL
1850 *
1851 * This function is called to handle new GIDs discovered
1852 * during device sweep / probe or for GID_AVAILABLE event.
1853 *
1854 * Parameter :
1855 * gid_info GID to check
1856 */
1857 static ibdm_dp_gidinfo_t *
1858 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
1859 {
1860 ibdm_dp_gidinfo_t *gid_list;
1861 ibdm_gid_t *tmp;
1862
1863 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");
1864
1865 gid_list = ibdm.ibdm_dp_gidlist_head;
1866 while (gid_list) {
1867 if ((gid_list != gid_info) &&
1868 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
1869 IBTF_DPRINTF_L4("ibdm",
1870 "\tcheck_dest_nodeguid: NodeGuid is present");
1871
1872 /* Add to gid_list */
1873 tmp = kmem_zalloc(sizeof (ibdm_gid_t),
1874 KM_SLEEP);
1875 tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
1876 tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
1877 tmp->gid_next = gid_list->gl_gid;
1878 gid_list->gl_gid = tmp;
1879 gid_list->gl_ngids++;
1880 return (gid_list);
1881 }
1882
1883 gid_list = gid_list->gl_next;
1884 }
1885
1886 return (NULL);
1887 }
1888
1889
1890 /*
1891 * ibdm_is_dev_mgt_supported
1892 * Get the PortInfo attribute (SA Query)
1893 * Check "CompatabilityMask" field in the Portinfo.
1894 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
1895 * by the port, otherwise IBDM_FAILURE
1896 */
1897 static int
1898 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
1899 {
1900 int ret;
1901 size_t length = 0;
1902 sa_portinfo_record_t req, *resp = NULL;
1903 ibmf_saa_access_args_t qargs;
1904
1905 bzero(&req, sizeof (sa_portinfo_record_t));
1906 req.EndportLID = gid_info->gl_dlid;
1907
1908 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID;
1909 qargs.sq_access_type = IBMF_SAA_RETRIEVE;
1910 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
1911 qargs.sq_template = &req;
1912 qargs.sq_callback = NULL;
1913 qargs.sq_callback_arg = NULL;
1914
1915 ret = ibmf_sa_access(gid_info->gl_sa_hdl,
1916 &qargs, 0, &length, (void **)&resp);
1917
1918 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1919 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
1920 "failed to get PORTINFO attribute %d", ret);
1921 return (IBDM_FAILURE);
1922 }
1923
1924 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
1925 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
1926 ret = IBDM_SUCCESS;
1927 } else {
1928 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
1929 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
1930 ret = IBDM_FAILURE;
1931 }
1932 kmem_free(resp, length);
1933 return (ret);
1934 }
1935
1936
1937 /*
1938 * ibdm_get_node_port_guids()
1939 * Get the NodeInfoRecord of the port
1940 * Save NodeGuid and PortGUID values in the GID list structure.
1941 * Return IBDM_SUCCESS/IBDM_FAILURE
1942 */
1943 static int
1944 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
1945 ib_guid_t *node_guid, ib_guid_t *port_guid)
1946 {
1947 int ret;
1948 size_t length = 0;
1949 sa_node_record_t req, *resp = NULL;
1950 ibmf_saa_access_args_t qargs;
1951
1952 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");
1953
1954 bzero(&req, sizeof (sa_node_record_t));
1955 req.LID = dlid;
1956
1957 qargs.sq_attr_id = SA_NODERECORD_ATTRID;
1958 qargs.sq_access_type = IBMF_SAA_RETRIEVE;
1959 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
1960 qargs.sq_template = &req;
1961 qargs.sq_callback = NULL;
1962 qargs.sq_callback_arg = NULL;
1963
1964 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
1965 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1966 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
1967 " SA Retrieve Failed: %d", ret);
1968 return (IBDM_FAILURE);
1969 }
1970 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
1971 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);
1972
1973 *node_guid = resp->NodeInfo.NodeGUID;
1974 *port_guid = resp->NodeInfo.PortGUID;
1975 kmem_free(resp, length);
1976 return (IBDM_SUCCESS);
1977 }
1978
1979
1980 /*
1981 * ibdm_get_reachable_ports()
1982 * Get list of the destination GID (and its path records) by
1983 * querying the SA access.
1984 *
1985 * Returns Number paths
1986 */
1987 static int
1988 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
1989 {
1990 uint_t ii, jj, nrecs;
1991 uint_t npaths = 0;
1992 size_t length;
1993 ib_gid_t sgid;
1994 ibdm_pkey_tbl_t *pkey_tbl;
1995 sa_path_record_t *result;
1996 sa_path_record_t *precp;
1997 ibdm_dp_gidinfo_t *gid_info;
1998
1999 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
2000 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);
2001
2002 sgid.gid_prefix = portinfo->pa_sn_prefix;
2003 sgid.gid_guid = portinfo->pa_port_guid;
2004
2005 /* get reversible paths */
2006 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
2007 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
2008 != IBMF_SUCCESS) {
2009 IBTF_DPRINTF_L2("ibdm",
2010 "\tget_reachable_ports: Getting path records failed");
2011 return (0);
2012 }
2013
2014 for (ii = 0; ii < nrecs; ii++) {
2015 sa_node_record_t *nrec;
2016 size_t length;
2017
2018 precp = &result[ii];
2019 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
2020 precp->DGID.gid_prefix)) != NULL) {
2021 IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: "
2022 "Already exists nrecs %d, ii %d", nrecs, ii);
2023 ibdm_addto_glhcalist(gid_info, hca);
2024 continue;
2025 }
2026 /*
2027 * This is a new GID. Allocate a GID structure and
2028 * initialize the structure
2029 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
2030 * by kmem_zalloc call
2031 */
2032 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
2033 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
2034 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
2035 gid_info->gl_dgid_hi = precp->DGID.gid_prefix;
2036 gid_info->gl_dgid_lo = precp->DGID.gid_guid;
2037 gid_info->gl_sgid_hi = precp->SGID.gid_prefix;
2038 gid_info->gl_sgid_lo = precp->SGID.gid_guid;
2039 gid_info->gl_p_key = precp->P_Key;
2040 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl;
2041 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl;
2042 gid_info->gl_slid = precp->SLID;
2043 gid_info->gl_dlid = precp->DLID;
2044 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID)
2045 << IBDM_GID_TRANSACTIONID_SHIFT;
2046 gid_info->gl_min_transactionID = gid_info->gl_transactionID;
2047 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1)
2048 << IBDM_GID_TRANSACTIONID_SHIFT;
2049 gid_info->gl_SL = precp->SL;
2050
2051 /*
2052 * get the node record with this guid if the destination
2053 * device is a Cisco one.
2054 */
2055 if (ibdm_is_cisco(precp->DGID.gid_guid) &&
2056 (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) &&
2057 ibdm_get_node_record_by_port(portinfo->pa_sa_hdl,
2058 precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) {
2059 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
2060 gid_info->gl_devid = nrec->NodeInfo.DeviceID;
2061 kmem_free(nrec, length);
2062 }
2063
2064 ibdm_addto_glhcalist(gid_info, hca);
2065
2066 ibdm_dump_path_info(precp);
2067
2068 gid_info->gl_qp_hdl = NULL;
2069 ASSERT(portinfo->pa_pkey_tbl != NULL &&
2070 portinfo->pa_npkeys != 0);
2071
2072 for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
2073 pkey_tbl = &portinfo->pa_pkey_tbl[jj];
2074 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
2075 (pkey_tbl->pt_qp_hdl != NULL)) {
2076 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
2077 break;
2078 }
2079 }
2080
2081 /*
2082 * QP handle for GID not initialized. No matching Pkey
2083 * was found!! ibdm should *not* hit this case. Flag an
2084 * error and drop the GID if ibdm does encounter this.
2085 */
2086 if (gid_info->gl_qp_hdl == NULL) {
2087 IBTF_DPRINTF_L2(ibdm_string,
2088 "\tget_reachable_ports: No matching Pkey");
2089 ibdm_delete_gidinfo(gid_info);
2090 continue;
2091 }
2092 if (ibdm.ibdm_dp_gidlist_head == NULL) {
2093 ibdm.ibdm_dp_gidlist_head = gid_info;
2094 ibdm.ibdm_dp_gidlist_tail = gid_info;
2095 } else {
2096 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
2097 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
2098 ibdm.ibdm_dp_gidlist_tail = gid_info;
2099 }
2100 npaths++;
2101 }
2102 kmem_free(result, length);
2103 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
2104 return (npaths);
2105 }
2106
2107
2108 /*
2109 * ibdm_check_dgid()
2110 * Look in the global list to check whether we know this DGID already
2111 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
2112 */
2113 static ibdm_dp_gidinfo_t *
2114 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
2115 {
2116 ibdm_dp_gidinfo_t *gid_list;
2117
2118 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2119 gid_list = gid_list->gl_next) {
2120 if ((guid == gid_list->gl_dgid_lo) &&
2121 (prefix == gid_list->gl_dgid_hi)) {
2122 break;
2123 }
2124 }
2125 return (gid_list);
2126 }
2127
2128
2129 /*
2130 * ibdm_find_gid()
2131 * Look in the global list to find a GID entry with matching
2132 * port & node GUID.
2133 * Return pointer to gidinfo if found, else return NULL
2134 */
2135 static ibdm_dp_gidinfo_t *
2136 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
2137 {
2138 ibdm_dp_gidinfo_t *gid_list;
2139
2140 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
2141 nodeguid, portguid);
2142
2143 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2144 gid_list = gid_list->gl_next) {
2145 if ((portguid == gid_list->gl_portguid) &&
2146 (nodeguid == gid_list->gl_nodeguid)) {
2147 break;
2148 }
2149 }
2150
2151 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
2152 gid_list);
2153 return (gid_list);
2154 }
2155
2156
2157 /*
2158 * ibdm_set_classportinfo()
2159 * ibdm_set_classportinfo() is a function to activate a Cisco FC GW
2160 * by sending the setClassPortInfo request with the trapLID, trapGID
2161 * and etc. to the gateway since the gateway doesn't provide the IO
2162 * Unit Information othewise. This behavior is the Cisco specific one,
2163 * and this function is called to a Cisco FC GW only.
2164 * Returns IBDM_SUCCESS/IBDM_FAILURE
2165 */
2166 static int
2167 ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2168 {
2169 ibmf_msg_t *msg;
2170 ib_mad_hdr_t *hdr;
2171 ibdm_timeout_cb_args_t *cb_args;
2172 void *data;
2173 ib_mad_classportinfo_t *cpi;
2174
2175 IBTF_DPRINTF_L4("ibdm",
2176 "\tset_classportinfo: gid info 0x%p", gid_info);
2177
2178 /*
2179 * Send command to set classportinfo attribute. Allocate a IBMF
2180 * packet and initialize the packet.
2181 */
2182 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2183 &msg) != IBMF_SUCCESS) {
2184 IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail");
2185 return (IBDM_FAILURE);
2186 }
2187
2188 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2189 ibdm_alloc_send_buffers(msg);
2190 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2191
2192 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2193 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2194 msg->im_local_addr.ia_remote_qno = 1;
2195 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2196 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2197 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2198
2199 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2200 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2201 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2202 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2203 hdr->R_Method = IB_DM_DEVMGT_METHOD_SET;
2204 hdr->Status = 0;
2205 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2206 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
2207 hdr->AttributeModifier = 0;
2208
2209 data = msg->im_msgbufs_send.im_bufs_cl_data;
2210 cpi = (ib_mad_classportinfo_t *)data;
2211
2212 /*
2213 * Set the classportinfo values to activate this Cisco FC GW.
2214 */
2215 cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi);
2216 cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo);
2217 cpi->TrapLID = h2b16(gid_info->gl_slid);
2218 cpi->TrapSL = gid_info->gl_SL;
2219 cpi->TrapP_Key = h2b16(gid_info->gl_p_key);
2220 cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn));
2221 cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *)
2222 gid_info->gl_qp_hdl)->isq_qkey));
2223
2224 cb_args = &gid_info->gl_cpi_cb_args;
2225 cb_args->cb_gid_info = gid_info;
2226 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2227 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2228
2229 mutex_enter(&gid_info->gl_mutex);
2230 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2231 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2232 mutex_exit(&gid_info->gl_mutex);
2233
2234 IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: "
2235 "timeout id %x", gid_info->gl_timeout_id);
2236
2237 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2238 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2239 IBTF_DPRINTF_L2("ibdm",
2240 "\tset_classportinfo: ibmf send failed");
2241 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2242 }
2243
2244 return (IBDM_SUCCESS);
2245 }
2246
2247
2248 /*
2249 * ibdm_send_classportinfo()
2250 * Send classportinfo request. When the request is completed
2251 * IBMF calls ibdm_classportinfo_cb routine to inform about
2252 * the completion.
2253 * Returns IBDM_SUCCESS/IBDM_FAILURE
2254 */
2255 static int
2256 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2257 {
2258 ibmf_msg_t *msg;
2259 ib_mad_hdr_t *hdr;
2260 ibdm_timeout_cb_args_t *cb_args;
2261
2262 IBTF_DPRINTF_L4("ibdm",
2263 "\tsend_classportinfo: gid info 0x%p", gid_info);
2264
2265 /*
2266 * Send command to get classportinfo attribute. Allocate a IBMF
2267 * packet and initialize the packet.
2268 */
2269 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2270 &msg) != IBMF_SUCCESS) {
2271 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
2272 return (IBDM_FAILURE);
2273 }
2274
2275 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2276 ibdm_alloc_send_buffers(msg);
2277 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2278
2279 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2280 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2281 msg->im_local_addr.ia_remote_qno = 1;
2282 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2283 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2284 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2285
2286 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2287 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2288 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2289 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2290 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2291 hdr->Status = 0;
2292 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2293 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
2294 hdr->AttributeModifier = 0;
2295
2296 cb_args = &gid_info->gl_cpi_cb_args;
2297 cb_args->cb_gid_info = gid_info;
2298 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2299 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2300
2301 mutex_enter(&gid_info->gl_mutex);
2302 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2303 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2304 mutex_exit(&gid_info->gl_mutex);
2305
2306 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
2307 "timeout id %x", gid_info->gl_timeout_id);
2308
2309 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2310 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2311 IBTF_DPRINTF_L2("ibdm",
2312 "\tsend_classportinfo: ibmf send failed");
2313 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2314 }
2315
2316 return (IBDM_SUCCESS);
2317 }
2318
2319
2320 /*
2321 * ibdm_handle_setclassportinfo()
2322 * Invoked by the IBMF when setClassPortInfo request is completed.
2323 */
2324 static void
2325 ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl,
2326 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2327 {
2328 void *data;
2329 timeout_id_t timeout_id;
2330 ib_mad_classportinfo_t *cpi;
2331
2332 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl "
2333 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2334
2335 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2336 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: "
2337 "Not a ClassPortInfo resp");
2338 *flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2339 return;
2340 }
2341
2342 /*
2343 * Verify whether timeout handler is created/active.
2344 * If created/ active, cancel the timeout handler
2345 */
2346 mutex_enter(&gid_info->gl_mutex);
2347 if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) {
2348 IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp");
2349 *flag |= IBDM_IBMF_PKT_DUP_RESP;
2350 mutex_exit(&gid_info->gl_mutex);
2351 return;
2352 }
2353 ibdm_bump_transactionID(gid_info);
2354
2355 gid_info->gl_iou_cb_args.cb_req_type = 0;
2356 if (gid_info->gl_timeout_id) {
2357 timeout_id = gid_info->gl_timeout_id;
2358 mutex_exit(&gid_info->gl_mutex);
2359 IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: "
2360 "gl_timeout_id = 0x%x", timeout_id);
2361 if (untimeout(timeout_id) == -1) {
2362 IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: "
2363 "untimeout gl_timeout_id failed");
2364 }
2365 mutex_enter(&gid_info->gl_mutex);
2366 gid_info->gl_timeout_id = 0;
2367 }
2368 mutex_exit(&gid_info->gl_mutex);
2369
2370 data = msg->im_msgbufs_recv.im_bufs_cl_data;
2371 cpi = (ib_mad_classportinfo_t *)data;
2372
2373 ibdm_dump_classportinfo(cpi);
2374 }
2375
2376
2377 /*
2378 * ibdm_handle_classportinfo()
2379 * Invoked by the IBMF when the classportinfo request is completed.
2380 */
2381 static void
2382 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
2383 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2384 {
2385 void *data;
2386 timeout_id_t timeout_id;
2387 ib_mad_hdr_t *hdr;
2388 ib_mad_classportinfo_t *cpi;
2389
2390 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
2391 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2392
2393 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2394 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
2395 "Not a ClassPortInfo resp");
2396 *flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2397 return;
2398 }
2399
2400 /*
2401 * Verify whether timeout handler is created/active.
2402 * If created/ active, cancel the timeout handler
2403 */
2404 mutex_enter(&gid_info->gl_mutex);
2405 ibdm_bump_transactionID(gid_info);
2406 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
2407 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
2408 *flag |= IBDM_IBMF_PKT_DUP_RESP;
2409 mutex_exit(&gid_info->gl_mutex);
2410 return;
2411 }
2412 gid_info->gl_iou_cb_args.cb_req_type = 0;
2413 if (gid_info->gl_timeout_id) {
2414 timeout_id = gid_info->gl_timeout_id;
2415 mutex_exit(&gid_info->gl_mutex);
2416 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
2417 "gl_timeout_id = 0x%x", timeout_id);
2418 if (untimeout(timeout_id) == -1) {
2419 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
2420 "untimeout gl_timeout_id failed");
2421 }
2422 mutex_enter(&gid_info->gl_mutex);
2423 gid_info->gl_timeout_id = 0;
2424 }
2425 gid_info->gl_state = IBDM_GET_IOUNITINFO;
2426 gid_info->gl_pending_cmds++;
2427 mutex_exit(&gid_info->gl_mutex);
2428
2429 data = msg->im_msgbufs_recv.im_bufs_cl_data;
2430 cpi = (ib_mad_classportinfo_t *)data;
2431
2432 /*
2433 * Cache the "RespTimeValue" and redirection information in the
2434 * global gid list data structure. This cached information will
2435 * be used to send any further requests to the GID.
2436 */
2437 gid_info->gl_resp_timeout =
2438 (b2h32(cpi->RespTimeValue) & 0x1F);
2439
2440 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) &
2441 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
2442 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID);
2443 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff);
2444 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key);
2445 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key);
2446 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi);
2447 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo);
2448 gid_info->gl_redirectSL = cpi->RedirectSL;
2449
2450 ibdm_dump_classportinfo(cpi);
2451
2452 /*
2453 * Send IOUnitInfo request
2454 * Reuse previously allocated IBMF packet for sending ClassPortInfo
2455 * Check whether DM agent on the remote node requested redirection
2456 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
2457 */
2458 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2459 ibdm_alloc_send_buffers(msg);
2460 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2461 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2462 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2463
2464 if (gid_info->gl_redirected == B_TRUE) {
2465 if (gid_info->gl_redirect_dlid != 0) {
2466 msg->im_local_addr.ia_remote_lid =
2467 gid_info->gl_redirect_dlid;
2468 }
2469 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
2470 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
2471 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
2472 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
2473 } else {
2474 msg->im_local_addr.ia_remote_qno = 1;
2475 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2476 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2477 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2478 }
2479
2480 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2481 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2482 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2483 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2484 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2485 hdr->Status = 0;
2486 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2487 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
2488 hdr->AttributeModifier = 0;
2489
2490 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2491 gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2492 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2493
2494 mutex_enter(&gid_info->gl_mutex);
2495 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2496 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2497 mutex_exit(&gid_info->gl_mutex);
2498
2499 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
2500 "timeout %x", gid_info->gl_timeout_id);
2501
2502 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
2503 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
2504 IBTF_DPRINTF_L2("ibdm",
2505 "\thandle_classportinfo: msg transport failed");
2506 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
2507 }
2508 (*flag) |= IBDM_IBMF_PKT_REUSED;
2509 }
2510
2511
2512 /*
2513 * ibdm_send_iounitinfo:
2514 * Sends a DM request to get IOU unitinfo.
2515 */
2516 static int
2517 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
2518 {
2519 ibmf_msg_t *msg;
2520 ib_mad_hdr_t *hdr;
2521
2522 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);
2523
2524 /*
2525 * Send command to get iounitinfo attribute. Allocate a IBMF
2526 * packet and initialize the packet.
2527 */
2528 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
2529 IBMF_SUCCESS) {
2530 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
2531 return (IBDM_FAILURE);
2532 }
2533
2534 mutex_enter(&gid_info->gl_mutex);
2535 ibdm_bump_transactionID(gid_info);
2536 mutex_exit(&gid_info->gl_mutex);
2537
2538
2539 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2540 ibdm_alloc_send_buffers(msg);
2541 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2542 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2543 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2544 msg->im_local_addr.ia_remote_qno = 1;
2545 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2546 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2547 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2548
2549 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2550 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2551 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2552 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2553 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2554 hdr->Status = 0;
2555 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2556 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
2557 hdr->AttributeModifier = 0;
2558
2559 gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2560 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2561 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2562
2563 mutex_enter(&gid_info->gl_mutex);
2564 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2565 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2566 mutex_exit(&gid_info->gl_mutex);
2567
2568 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
2569 "timeout %x", gid_info->gl_timeout_id);
2570
2571 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
2572 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
2573 IBMF_SUCCESS) {
2574 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
2575 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
2576 msg, &gid_info->gl_iou_cb_args);
2577 }
2578 return (IBDM_SUCCESS);
2579 }
2580
2581 /*
2582 * ibdm_handle_iounitinfo()
2583 * Invoked by the IBMF when IO Unitinfo request is completed.
2584 */
2585 static void
2586 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
2587 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2588 {
2589 int ii, first = B_TRUE;
2590 int num_iocs;
2591 size_t size;
2592 uchar_t slot_info;
2593 timeout_id_t timeout_id;
2594 ib_mad_hdr_t *hdr;
2595 ibdm_ioc_info_t *ioc_info;
2596 ib_dm_io_unitinfo_t *iou_info;
2597 ib_dm_io_unitinfo_t *giou_info;
2598 ibdm_timeout_cb_args_t *cb_args;
2599
2600 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
2601 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);
2602
2603 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
2604 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
2605 "Unexpected response");
2606 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2607 return;
2608 }
2609
2610 mutex_enter(&gid_info->gl_mutex);
2611 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
2612 IBTF_DPRINTF_L4("ibdm",
2613 "\thandle_iounitinfo: DUP resp");
2614 mutex_exit(&gid_info->gl_mutex);
2615 (*flag) = IBDM_IBMF_PKT_DUP_RESP;
2616 return;
2617 }
2618 gid_info->gl_iou_cb_args.cb_req_type = 0;
2619 if (gid_info->gl_timeout_id) {
2620 timeout_id = gid_info->gl_timeout_id;
2621 mutex_exit(&gid_info->gl_mutex);
2622 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
2623 "gl_timeout_id = 0x%x", timeout_id);
2624 if (untimeout(timeout_id) == -1) {
2625 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
2626 "untimeout gl_timeout_id failed");
2627 }
2628 mutex_enter(&gid_info->gl_mutex);
2629 gid_info->gl_timeout_id = 0;
2630 }
2631 gid_info->gl_state = IBDM_GET_IOC_DETAILS;
2632
2633 iou_info = IBDM_IN_IBMFMSG2IOU(msg);
2634 ibdm_dump_iounitinfo(iou_info);
2635 num_iocs = iou_info->iou_num_ctrl_slots;
2636 /*
2637 * check if number of IOCs reported is zero? if yes, return.
2638 * when num_iocs are reported zero internal IOC database needs
2639 * to be updated. To ensure that save the number of IOCs in
2640 * the new field "gl_num_iocs". Use a new field instead of
2641 * "giou_info->iou_num_ctrl_slots" as that would prevent
2642 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
2643 */
2644 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
2645 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
2646 mutex_exit(&gid_info->gl_mutex);
2647 return;
2648 }
2649 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);
2650
2651 /*
2652 * if there is an existing gl_iou (IOU has been probed before)
2653 * check if the "iou_changeid" is same as saved entry in
2654 * "giou_info->iou_changeid".
2655 * (note: this logic can prevent IOC enumeration if a given
2656 * vendor doesn't support setting iou_changeid field for its IOU)
2657 *
2658 * if there is an existing gl_iou and iou_changeid has changed :
2659 * free up existing gl_iou info and its related structures.
2660 * reallocate gl_iou info all over again.
2661 * if we donot free this up; then this leads to memory leaks
2662 */
2663 if (gid_info->gl_iou) {
2664 giou_info = &gid_info->gl_iou->iou_info;
2665 if (b2h16(iou_info->iou_changeid) ==
2666 giou_info->iou_changeid) {
2667 IBTF_DPRINTF_L3("ibdm",
2668 "\thandle_iounitinfo: no IOCs changed");
2669 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2670 mutex_exit(&gid_info->gl_mutex);
2671 return;
2672 }
2673
2674 /*
2675 * Store the iou info as prev_iou to be used after
2676 * sweep is done.
2677 */
2678 ASSERT(gid_info->gl_prev_iou == NULL);
2679 IBTF_DPRINTF_L4(ibdm_string,
2680 "\thandle_iounitinfo: setting gl_prev_iou %p",
2681 gid_info->gl_prev_iou);
2682 gid_info->gl_prev_iou = gid_info->gl_iou;
2683 ibdm.ibdm_prev_iou = 1;
2684 gid_info->gl_iou = NULL;
2685 }
2686
2687 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
2688 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
2689 giou_info = &gid_info->gl_iou->iou_info;
2690 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
2691 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));
2692
2693 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs;
2694 giou_info->iou_flag = iou_info->iou_flag;
2695 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
2696 giou_info->iou_changeid = b2h16(iou_info->iou_changeid);
2697 gid_info->gl_pending_cmds++; /* for diag code */
2698 mutex_exit(&gid_info->gl_mutex);
2699
2700 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
2701 mutex_enter(&gid_info->gl_mutex);
2702 gid_info->gl_pending_cmds--;
2703 mutex_exit(&gid_info->gl_mutex);
2704 }
2705 /*
2706 * Parallelize getting IOC controller profiles from here.
2707 * Allocate IBMF packets and send commands to get IOC profile for
2708 * each IOC present on the IOU.
2709 */
2710 for (ii = 0; ii < num_iocs; ii++) {
2711 /*
2712 * Check whether IOC is present in the slot
2713 * Series of nibbles (in the field iou_ctrl_list) represents
2714 * a slot in the IOU.
2715 * Byte format: 76543210
2716 * Bits 0-3 of first byte represent Slot 2
2717 * bits 4-7 of first byte represent slot 1,
2718 * bits 0-3 of second byte represent slot 4 and so on
2719 * Each 4-bit nibble has the following meaning
2720 * 0x0 : IOC not installed
2721 * 0x1 : IOC is present
2722 * 0xf : Slot does not exist
2723 * and all other values are reserved.
2724 */
2725 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
2726 slot_info = giou_info->iou_ctrl_list[(ii/2)];
2727 if ((ii % 2) == 0)
2728 slot_info = (slot_info >> 4);
2729
2730 if ((slot_info & 0xf) != 1) {
2731 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2732 "No IOC is present in the slot = %d", ii);
2733 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
2734 continue;
2735 }
2736
2737 mutex_enter(&gid_info->gl_mutex);
2738 ibdm_bump_transactionID(gid_info);
2739 mutex_exit(&gid_info->gl_mutex);
2740
2741 /*
2742 * Re use the already allocated packet (for IOUnitinfo) to
2743 * send the first IOC controller attribute. Allocate new
2744 * IBMF packets for the rest of the IOC's
2745 */
2746 if (first != B_TRUE) {
2747 msg = NULL;
2748 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2749 &msg) != IBMF_SUCCESS) {
2750 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2751 "IBMF packet allocation failed");
2752 continue;
2753 }
2754
2755 }
2756
2757 /* allocate send buffers for all messages */
2758 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2759 ibdm_alloc_send_buffers(msg);
2760 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2761
2762 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2763 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2764 if (gid_info->gl_redirected == B_TRUE) {
2765 if (gid_info->gl_redirect_dlid != 0) {
2766 msg->im_local_addr.ia_remote_lid =
2767 gid_info->gl_redirect_dlid;
2768 }
2769 msg->im_local_addr.ia_remote_qno =
2770 gid_info->gl_redirect_QP;
2771 msg->im_local_addr.ia_p_key =
2772 gid_info->gl_redirect_pkey;
2773 msg->im_local_addr.ia_q_key =
2774 gid_info->gl_redirect_qkey;
2775 msg->im_local_addr.ia_service_level =
2776 gid_info->gl_redirectSL;
2777 } else {
2778 msg->im_local_addr.ia_remote_qno = 1;
2779 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2780 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2781 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2782 }
2783
2784 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2785 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2786 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2787 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2788 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2789 hdr->Status = 0;
2790 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2791 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
2792 hdr->AttributeModifier = h2b32(ii + 1);
2793
2794 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID;
2795 cb_args = &ioc_info->ioc_cb_args;
2796 cb_args->cb_gid_info = gid_info;
2797 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2798 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO;
2799 cb_args->cb_ioc_num = ii;
2800
2801 mutex_enter(&gid_info->gl_mutex);
2802 gid_info->gl_pending_cmds++; /* for diag code */
2803
2804 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2805 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2806 mutex_exit(&gid_info->gl_mutex);
2807
2808 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
2809 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);
2810
2811 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2812 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2813 IBTF_DPRINTF_L2("ibdm",
2814 "\thandle_iounitinfo: msg transport failed");
2815 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2816 }
2817 (*flag) |= IBDM_IBMF_PKT_REUSED;
2818 first = B_FALSE;
2819 gid_info->gl_iou->iou_niocs_probe_in_progress++;
2820 }
2821 }
2822
2823
2824 /*
2825 * ibdm_handle_ioc_profile()
2826 * Invoked by the IBMF when the IOCControllerProfile request
2827 * gets completed
2828 */
2829 static void
2830 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
2831 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2832 {
2833 int first = B_TRUE, reprobe = 0;
2834 uint_t ii, ioc_no, srv_start;
2835 uint_t nserv_entries;
2836 timeout_id_t timeout_id;
2837 ib_mad_hdr_t *hdr;
2838 ibdm_ioc_info_t *ioc_info;
2839 ibdm_timeout_cb_args_t *cb_args;
2840 ib_dm_ioc_ctrl_profile_t *ioc, *gioc;
2841
2842 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2843 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2844
2845 ioc = IBDM_IN_IBMFMSG2IOC(msg);
2846 /*
2847 * Check whether we know this IOC already
2848 * This will return NULL if reprobe is in progress
2849 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
2850 * Do not hold mutexes here.
2851 */
2852 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
2853 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2854 "IOC guid %llx is present", ioc->ioc_guid);
2855 return;
2856 }
2857 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2858 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);
2859
2860 /* Make sure that IOC index is with the valid range */
2861 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
2862 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
2863 "IOC index Out of range, index %d", ioc);
2864 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2865 return;
2866 }
2867 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
2868 ioc_info->ioc_iou_info = gid_info->gl_iou;
2869
2870 mutex_enter(&gid_info->gl_mutex);
2871 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
2872 reprobe = 1;
2873 ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
2874 ioc_info->ioc_serv = NULL;
2875 ioc_info->ioc_prev_serv_cnt =
2876 ioc_info->ioc_profile.ioc_service_entries;
2877 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
2878 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
2879 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
2880 mutex_exit(&gid_info->gl_mutex);
2881 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2882 return;
2883 }
2884 ioc_info->ioc_cb_args.cb_req_type = 0;
2885 if (ioc_info->ioc_timeout_id) {
2886 timeout_id = ioc_info->ioc_timeout_id;
2887 ioc_info->ioc_timeout_id = 0;
2888 mutex_exit(&gid_info->gl_mutex);
2889 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
2890 "ioc_timeout_id = 0x%x", timeout_id);
2891 if (untimeout(timeout_id) == -1) {
2892 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
2893 "untimeout ioc_timeout_id failed");
2894 }
2895 mutex_enter(&gid_info->gl_mutex);
2896 }
2897
2898 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
2899 if (reprobe == 0) {
2900 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
2901 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
2902 }
2903
2904 /*
2905 * Save all the IOC information in the global structures.
2906 * Note the wire format is Big Endian and the Sparc process also
2907 * big endian. So, there is no need to convert the data fields
2908 * The conversion routines used below are ineffective on Sparc
2909 * machines where as they will be effective on little endian
2910 * machines such as Intel processors.
2911 */
2912 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;
2913
2914 /*
2915 * Restrict updates to onlyport GIDs and service entries during reprobe
2916 */
2917 if (reprobe == 0) {
2918 gioc->ioc_guid = b2h64(ioc->ioc_guid);
2919 gioc->ioc_vendorid =
2920 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
2921 >> IB_DM_VENDORID_SHIFT);
2922 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid);
2923 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver);
2924 gioc->ioc_subsys_vendorid =
2925 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
2926 >> IB_DM_VENDORID_SHIFT);
2927 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id);
2928 gioc->ioc_io_class = b2h16(ioc->ioc_io_class);
2929 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass);
2930 gioc->ioc_protocol = b2h16(ioc->ioc_protocol);
2931 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver);
2932 gioc->ioc_send_msg_qdepth =
2933 b2h16(ioc->ioc_send_msg_qdepth);
2934 gioc->ioc_rdma_read_qdepth =
2935 b2h16(ioc->ioc_rdma_read_qdepth);
2936 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz);
2937 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz);
2938 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask;
2939 bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
2940 IB_DM_IOC_ID_STRING_LEN);
2941
2942 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
2943 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
2944 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
2945 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;
2946
2947 if (ioc_info->ioc_diagdeviceid == B_TRUE) {
2948 gid_info->gl_pending_cmds++;
2949 IBTF_DPRINTF_L3(ibdm_string,
2950 "\tibdm_handle_ioc_profile: "
2951 "%d: gid_info %p gl_state %d pending_cmds %d",
2952 __LINE__, gid_info, gid_info->gl_state,
2953 gid_info->gl_pending_cmds);
2954 }
2955 }
2956 gioc->ioc_service_entries = ioc->ioc_service_entries;
2957 mutex_exit(&gid_info->gl_mutex);
2958
2959 ibdm_dump_ioc_profile(gioc);
2960
2961 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
2962 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
2963 mutex_enter(&gid_info->gl_mutex);
2964 gid_info->gl_pending_cmds--;
2965 mutex_exit(&gid_info->gl_mutex);
2966 }
2967 }
2968 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
2969 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
2970 KM_SLEEP);
2971
2972 /*
2973 * In one single request, maximum number of requests that can be
2974 * obtained is 4. If number of service entries are more than four,
2975 * calculate number requests needed and send them parallelly.
2976 */
2977 nserv_entries = ioc->ioc_service_entries;
2978 ii = 0;
2979 while (nserv_entries) {
2980 mutex_enter(&gid_info->gl_mutex);
2981 gid_info->gl_pending_cmds++;
2982 ibdm_bump_transactionID(gid_info);
2983 mutex_exit(&gid_info->gl_mutex);
2984
2985 if (first != B_TRUE) {
2986 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2987 &msg) != IBMF_SUCCESS) {
2988 continue;
2989 }
2990
2991 }
2992 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2993 ibdm_alloc_send_buffers(msg);
2994 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2995 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2996 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2997 if (gid_info->gl_redirected == B_TRUE) {
2998 if (gid_info->gl_redirect_dlid != 0) {
2999 msg->im_local_addr.ia_remote_lid =
3000 gid_info->gl_redirect_dlid;
3001 }
3002 msg->im_local_addr.ia_remote_qno =
3003 gid_info->gl_redirect_QP;
3004 msg->im_local_addr.ia_p_key =
3005 gid_info->gl_redirect_pkey;
3006 msg->im_local_addr.ia_q_key =
3007 gid_info->gl_redirect_qkey;
3008 msg->im_local_addr.ia_service_level =
3009 gid_info->gl_redirectSL;
3010 } else {
3011 msg->im_local_addr.ia_remote_qno = 1;
3012 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3013 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3014 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3015 }
3016
3017 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3018 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3019 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3020 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3021 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3022 hdr->Status = 0;
3023 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3024 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
3025
3026 srv_start = ii * 4;
3027 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
3028 cb_args->cb_gid_info = gid_info;
3029 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
3030 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS;
3031 cb_args->cb_srvents_start = srv_start;
3032 cb_args->cb_ioc_num = ioc_no - 1;
3033
3034 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
3035 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
3036 cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
3037 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
3038 } else {
3039 cb_args->cb_srvents_end =
3040 (cb_args->cb_srvents_start + nserv_entries - 1);
3041 nserv_entries = 0;
3042 }
3043 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3044 ibdm_fill_srv_attr_mod(hdr, cb_args);
3045 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3046
3047 mutex_enter(&gid_info->gl_mutex);
3048 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
3049 ibdm_pkt_timeout_hdlr, cb_args,
3050 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3051 mutex_exit(&gid_info->gl_mutex);
3052
3053 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
3054 "timeout %x, ioc %d srv %d",
3055 ioc_info->ioc_serv[srv_start].se_timeout_id,
3056 ioc_no - 1, srv_start);
3057
3058 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
3059 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3060 IBTF_DPRINTF_L2("ibdm",
3061 "\thandle_ioc_profile: msg send failed");
3062 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
3063 }
3064 (*flag) |= IBDM_IBMF_PKT_REUSED;
3065 first = B_FALSE;
3066 ii++;
3067 }
3068 }
3069
3070
3071 /*
3072 * ibdm_handle_srventry_mad()
3073 */
3074 static void
3075 ibdm_handle_srventry_mad(ibmf_msg_t *msg,
3076 ibdm_dp_gidinfo_t *gid_info, int *flag)
3077 {
3078 uint_t ii, ioc_no, attrmod;
3079 uint_t nentries, start, end;
3080 timeout_id_t timeout_id;
3081 ib_dm_srv_t *srv_ents;
3082 ibdm_ioc_info_t *ioc_info;
3083 ibdm_srvents_info_t *gsrv_ents;
3084
3085 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
3086 " IBMF msg %p gid info %p", msg, gid_info);
3087
3088 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
3089 /*
3090 * Get the start and end index of the service entries
3091 * Upper 16 bits identify the IOC
3092 * Lower 16 bits specify the range of service entries
3093 * LSB specifies (Big endian) end of the range
3094 * MSB specifies (Big endian) start of the range
3095 */
3096 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3097 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK);
3098 end = ((attrmod >> 8) & IBDM_8_BIT_MASK);
3099 start = (attrmod & IBDM_8_BIT_MASK);
3100
3101 /* Make sure that IOC index is with the valid range */
3102 if ((ioc_no < 1) |
3103 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
3104 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3105 "IOC index Out of range, index %d", ioc_no);
3106 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3107 return;
3108 }
3109 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3110
3111 /*
3112 * Make sure that the "start" and "end" service indexes are
3113 * with in the valid range
3114 */
3115 nentries = ioc_info->ioc_profile.ioc_service_entries;
3116 if ((start > end) | (start >= nentries) | (end >= nentries)) {
3117 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3118 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
3119 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3120 return;
3121 }
3122 gsrv_ents = &ioc_info->ioc_serv[start];
3123 mutex_enter(&gid_info->gl_mutex);
3124 if (gsrv_ents->se_state != IBDM_SE_INVALID) {
3125 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3126 "already known, ioc %d, srv %d, se_state %x",
3127 ioc_no - 1, start, gsrv_ents->se_state);
3128 mutex_exit(&gid_info->gl_mutex);
3129 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3130 return;
3131 }
3132 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
3133 if (ioc_info->ioc_serv[start].se_timeout_id) {
3134 IBTF_DPRINTF_L2("ibdm",
3135 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
3136 timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
3137 ioc_info->ioc_serv[start].se_timeout_id = 0;
3138 mutex_exit(&gid_info->gl_mutex);
3139 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
3140 "se_timeout_id = 0x%x", timeout_id);
3141 if (untimeout(timeout_id) == -1) {
3142 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
3143 "untimeout se_timeout_id failed");
3144 }
3145 mutex_enter(&gid_info->gl_mutex);
3146 }
3147
3148 gsrv_ents->se_state = IBDM_SE_VALID;
3149 mutex_exit(&gid_info->gl_mutex);
3150 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
3151 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
3152 bcopy(srv_ents->srv_name,
3153 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
3154 ibdm_dump_service_entries(&gsrv_ents->se_attr);
3155 }
3156 }
3157
3158
3159 /*
3160 * ibdm_get_diagcode:
3161 * Send request to get IOU/IOC diag code
3162 * Returns IBDM_SUCCESS/IBDM_FAILURE
3163 */
3164 static int
3165 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
3166 {
3167 ibmf_msg_t *msg;
3168 ib_mad_hdr_t *hdr;
3169 ibdm_ioc_info_t *ioc;
3170 ibdm_timeout_cb_args_t *cb_args;
3171 timeout_id_t *timeout_id;
3172
3173 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
3174 gid_info, attr);
3175
3176 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
3177 &msg) != IBMF_SUCCESS) {
3178 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
3179 return (IBDM_FAILURE);
3180 }
3181
3182 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
3183 ibdm_alloc_send_buffers(msg);
3184 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
3185
3186 mutex_enter(&gid_info->gl_mutex);
3187 ibdm_bump_transactionID(gid_info);
3188 mutex_exit(&gid_info->gl_mutex);
3189
3190 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
3191 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
3192 if (gid_info->gl_redirected == B_TRUE) {
3193 if (gid_info->gl_redirect_dlid != 0) {
3194 msg->im_local_addr.ia_remote_lid =
3195 gid_info->gl_redirect_dlid;
3196 }
3197
3198 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3199 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3200 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3201 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3202 } else {
3203 msg->im_local_addr.ia_remote_qno = 1;
3204 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3205 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3206 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3207 }
3208
3209 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3210 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3211 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3212 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3213 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3214 hdr->Status = 0;
3215 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3216
3217 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3218 hdr->AttributeModifier = h2b32(attr);
3219
3220 if (attr == 0) {
3221 cb_args = &gid_info->gl_iou_cb_args;
3222 gid_info->gl_iou->iou_dc_valid = B_FALSE;
3223 cb_args->cb_ioc_num = 0;
3224 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE;
3225 timeout_id = &gid_info->gl_timeout_id;
3226 } else {
3227 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
3228 ioc->ioc_dc_valid = B_FALSE;
3229 cb_args = &ioc->ioc_dc_cb_args;
3230 cb_args->cb_ioc_num = attr - 1;
3231 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE;
3232 timeout_id = &ioc->ioc_dc_timeout_id;
3233 }
3234 cb_args->cb_gid_info = gid_info;
3235 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
3236 cb_args->cb_srvents_start = 0;
3237
3238 mutex_enter(&gid_info->gl_mutex);
3239 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3240 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3241 mutex_exit(&gid_info->gl_mutex);
3242
3243 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
3244 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);
3245
3246 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3247 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3248 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
3249 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3250 }
3251 return (IBDM_SUCCESS);
3252 }
3253
3254 /*
3255 * ibdm_handle_diagcode:
3256 * Process the DiagCode MAD response and update local DM
3257 * data structure.
3258 */
3259 static void
3260 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
3261 ibdm_dp_gidinfo_t *gid_info, int *flag)
3262 {
3263 uint16_t attrmod, *diagcode;
3264 ibdm_iou_info_t *iou;
3265 ibdm_ioc_info_t *ioc;
3266 timeout_id_t timeout_id;
3267 ibdm_timeout_cb_args_t *cb_args;
3268
3269 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;
3270
3271 mutex_enter(&gid_info->gl_mutex);
3272 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
3273 iou = gid_info->gl_iou;
3274 if (attrmod == 0) {
3275 if (iou->iou_dc_valid != B_FALSE) {
3276 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3277 IBTF_DPRINTF_L4("ibdm",
3278 "\thandle_diagcode: Duplicate IOU DiagCode");
3279 mutex_exit(&gid_info->gl_mutex);
3280 return;
3281 }
3282 cb_args = &gid_info->gl_iou_cb_args;
3283 cb_args->cb_req_type = 0;
3284 iou->iou_diagcode = b2h16(*diagcode);
3285 iou->iou_dc_valid = B_TRUE;
3286 if (gid_info->gl_timeout_id) {
3287 timeout_id = gid_info->gl_timeout_id;
3288 mutex_exit(&gid_info->gl_mutex);
3289 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
3290 "gl_timeout_id = 0x%x", timeout_id);
3291 if (untimeout(timeout_id) == -1) {
3292 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
3293 "untimeout gl_timeout_id failed");
3294 }
3295 mutex_enter(&gid_info->gl_mutex);
3296 gid_info->gl_timeout_id = 0;
3297 }
3298 } else {
3299 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
3300 if (ioc->ioc_dc_valid != B_FALSE) {
3301 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3302 IBTF_DPRINTF_L4("ibdm",
3303 "\thandle_diagcode: Duplicate IOC DiagCode");
3304 mutex_exit(&gid_info->gl_mutex);
3305 return;
3306 }
3307 cb_args = &ioc->ioc_dc_cb_args;
3308 cb_args->cb_req_type = 0;
3309 ioc->ioc_diagcode = b2h16(*diagcode);
3310 ioc->ioc_dc_valid = B_TRUE;
3311 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
3312 if (timeout_id) {
3313 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
3314 mutex_exit(&gid_info->gl_mutex);
3315 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
3316 "timeout_id = 0x%x", timeout_id);
3317 if (untimeout(timeout_id) == -1) {
3318 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
3319 "untimeout ioc_dc_timeout_id failed");
3320 }
3321 mutex_enter(&gid_info->gl_mutex);
3322 }
3323 }
3324 mutex_exit(&gid_info->gl_mutex);
3325
3326 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
3327 "attrmod : 0x%x", b2h16(*diagcode), attrmod);
3328 }
3329
3330
3331 /*
3332 * ibdm_is_ioc_present()
3333 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list
3334 */
3335 static ibdm_ioc_info_t *
3336 ibdm_is_ioc_present(ib_guid_t ioc_guid,
3337 ibdm_dp_gidinfo_t *gid_info, int *flag)
3338 {
3339 int ii;
3340 ibdm_ioc_info_t *ioc;
3341 ibdm_dp_gidinfo_t *head;
3342 ib_dm_io_unitinfo_t *iou;
3343
3344 mutex_enter(&ibdm.ibdm_mutex);
3345 head = ibdm.ibdm_dp_gidlist_head;
3346 while (head) {
3347 mutex_enter(&head->gl_mutex);
3348 if (head->gl_iou == NULL) {
3349 mutex_exit(&head->gl_mutex);
3350 head = head->gl_next;
3351 continue;
3352 }
3353 iou = &head->gl_iou->iou_info;
3354 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
3355 ioc = IBDM_GIDINFO2IOCINFO(head, ii);
3356 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
3357 (ioc->ioc_profile.ioc_guid == ioc_guid)) {
3358 if (gid_info == head) {
3359 *flag |= IBDM_IBMF_PKT_DUP_RESP;
3360 } else if (ibdm_check_dgid(head->gl_dgid_lo,
3361 head->gl_dgid_hi) != NULL) {
3362 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
3363 "present: gid not present");
3364 ibdm_add_to_gl_gid(gid_info, head);
3365 }
3366 mutex_exit(&head->gl_mutex);
3367 mutex_exit(&ibdm.ibdm_mutex);
3368 return (ioc);
3369 }
3370 }
3371 mutex_exit(&head->gl_mutex);
3372 head = head->gl_next;
3373 }
3374 mutex_exit(&ibdm.ibdm_mutex);
3375 return (NULL);
3376 }
3377
3378
3379 /*
3380 * ibdm_ibmf_send_cb()
3381 * IBMF invokes this callback routine after posting the DM MAD to
3382 * the HCA.
3383 */
3384 /*ARGSUSED*/
3385 static void
3386 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
3387 {
3388 ibdm_dump_ibmf_msg(ibmf_msg, 1);
3389 ibdm_free_send_buffers(ibmf_msg);
3390 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
3391 IBTF_DPRINTF_L4("ibdm",
3392 "\tibmf_send_cb: IBMF free msg failed");
3393 }
3394 }
3395
3396
3397 /*
3398 * ibdm_ibmf_recv_cb()
3399 * Invoked by the IBMF when a response to the one of the DM requests
3400 * is received.
3401 */
3402 /*ARGSUSED*/
3403 static void
3404 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3405 {
3406 ibdm_taskq_args_t *taskq_args;
3407
3408 /*
3409 * If the taskq enable is set then dispatch a taskq to process
3410 * the MAD, otherwise just process it on this thread
3411 */
3412 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
3413 ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
3414 return;
3415 }
3416
3417 /*
3418 * create a taskq and dispatch it to process the incoming MAD
3419 */
3420 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
3421 if (taskq_args == NULL) {
3422 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
3423 "taskq_args");
3424 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3425 IBTF_DPRINTF_L4("ibmf_recv_cb",
3426 "\tibmf_recv_cb: IBMF free msg failed");
3427 }
3428 return;
3429 }
3430 taskq_args->tq_ibmf_handle = ibmf_hdl;
3431 taskq_args->tq_ibmf_msg = msg;
3432 taskq_args->tq_args = arg;
3433
3434 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
3435 TQ_NOSLEEP) == 0) {
3436 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
3437 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3438 IBTF_DPRINTF_L4("ibmf_recv_cb",
3439 "\tibmf_recv_cb: IBMF free msg failed");
3440 }
3441 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3442 return;
3443 }
3444
3445 /* taskq_args are deleted in ibdm_recv_incoming_mad() */
3446 }
3447
3448
3449 void
3450 ibdm_recv_incoming_mad(void *args)
3451 {
3452 ibdm_taskq_args_t *taskq_args;
3453
3454 taskq_args = (ibdm_taskq_args_t *)args;
3455
3456 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
3457 "Processing incoming MAD via taskq");
3458
3459 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
3460 taskq_args->tq_ibmf_msg, taskq_args->tq_args);
3461
3462 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3463 }
3464
3465
3466 /*
3467 * Calls ibdm_process_incoming_mad with all function arguments extracted
3468 * from args
3469 */
3470 /*ARGSUSED*/
3471 static void
3472 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3473 {
3474 int flag = 0;
3475 int ret;
3476 uint64_t transaction_id;
3477 ib_mad_hdr_t *hdr;
3478 ibdm_dp_gidinfo_t *gid_info = NULL;
3479
3480 IBTF_DPRINTF_L4("ibdm",
3481 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
3482 ibdm_dump_ibmf_msg(msg, 0);
3483
3484 /*
3485 * IBMF calls this routine for every DM MAD that arrives at this port.
3486 * But we handle only the responses for requests we sent. We drop all
3487 * the DM packets that does not have response bit set in the MAD
3488 * header(this eliminates all the requests sent to this port).
3489 * We handle only DM class version 1 MAD's
3490 */
3491 hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
3492 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
3493 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3494 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3495 "IBMF free msg failed DM request drop it");
3496 }
3497 return;
3498 }
3499
3500 transaction_id = b2h64(hdr->TransactionID);
3501
3502 mutex_enter(&ibdm.ibdm_mutex);
3503 gid_info = ibdm.ibdm_dp_gidlist_head;
3504 while (gid_info) {
3505 if ((gid_info->gl_transactionID &
3506 IBDM_GID_TRANSACTIONID_MASK) ==
3507 (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
3508 break;
3509 gid_info = gid_info->gl_next;
3510 }
3511 mutex_exit(&ibdm.ibdm_mutex);
3512
3513 if (gid_info == NULL) {
3514 /* Drop the packet */
3515 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
3516 " does not match: 0x%llx", transaction_id);
3517 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3518 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3519 "IBMF free msg failed DM request drop it");
3520 }
3521 return;
3522 }
3523
3524 /* Handle redirection for all the MAD's, except ClassPortInfo */
3525 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
3526 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
3527 ret = ibdm_handle_redirection(msg, gid_info, &flag);
3528 if (ret == IBDM_SUCCESS) {
3529 return;
3530 }
3531 } else {
3532 uint_t gl_state;
3533
3534 mutex_enter(&gid_info->gl_mutex);
3535 gl_state = gid_info->gl_state;
3536 mutex_exit(&gid_info->gl_mutex);
3537
3538 switch (gl_state) {
3539
3540 case IBDM_SET_CLASSPORTINFO:
3541 ibdm_handle_setclassportinfo(
3542 ibmf_hdl, msg, gid_info, &flag);
3543 break;
3544
3545 case IBDM_GET_CLASSPORTINFO:
3546 ibdm_handle_classportinfo(
3547 ibmf_hdl, msg, gid_info, &flag);
3548 break;
3549
3550 case IBDM_GET_IOUNITINFO:
3551 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
3552 break;
3553
3554 case IBDM_GET_IOC_DETAILS:
3555 switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3556
3557 case IB_DM_ATTR_SERVICE_ENTRIES:
3558 ibdm_handle_srventry_mad(msg, gid_info, &flag);
3559 break;
3560
3561 case IB_DM_ATTR_IOC_CTRL_PROFILE:
3562 ibdm_handle_ioc_profile(
3563 ibmf_hdl, msg, gid_info, &flag);
3564 break;
3565
3566 case IB_DM_ATTR_DIAG_CODE:
3567 ibdm_handle_diagcode(msg, gid_info, &flag);
3568 break;
3569
3570 default:
3571 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3572 "Error state, wrong attribute :-(");
3573 (void) ibmf_free_msg(ibmf_hdl, &msg);
3574 return;
3575 }
3576 break;
3577 default:
3578 IBTF_DPRINTF_L2("ibdm",
3579 "process_incoming_mad: Dropping the packet"
3580 " gl_state %x", gl_state);
3581 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3582 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3583 "IBMF free msg failed DM request drop it");
3584 }
3585 return;
3586 }
3587 }
3588
3589 if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
3590 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
3591 IBTF_DPRINTF_L2("ibdm",
3592 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
3593 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3594 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3595 "IBMF free msg failed DM request drop it");
3596 }
3597 return;
3598 }
3599
3600 mutex_enter(&gid_info->gl_mutex);
3601 if (gid_info->gl_pending_cmds < 1) {
3602 IBTF_DPRINTF_L2("ibdm",
3603 "\tprocess_incoming_mad: pending commands negative");
3604 }
3605 if (--gid_info->gl_pending_cmds) {
3606 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
3607 "gid_info %p pending cmds %d",
3608 gid_info, gid_info->gl_pending_cmds);
3609 mutex_exit(&gid_info->gl_mutex);
3610 } else {
3611 uint_t prev_state;
3612 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
3613 prev_state = gid_info->gl_state;
3614 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
3615 if (prev_state == IBDM_SET_CLASSPORTINFO) {
3616 IBTF_DPRINTF_L4("ibdm",
3617 "\tprocess_incoming_mad: "
3618 "Setclassportinfo for Cisco FC GW is done.");
3619 gid_info->gl_flag &= ~IBDM_CISCO_PROBE;
3620 gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE;
3621 mutex_exit(&gid_info->gl_mutex);
3622 cv_broadcast(&gid_info->gl_probe_cv);
3623 } else {
3624 mutex_exit(&gid_info->gl_mutex);
3625 ibdm_notify_newgid_iocs(gid_info);
3626 mutex_enter(&ibdm.ibdm_mutex);
3627 if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3628 IBTF_DPRINTF_L4("ibdm",
3629 "\tprocess_incoming_mad: Wakeup");
3630 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3631 cv_broadcast(&ibdm.ibdm_probe_cv);
3632 }
3633 mutex_exit(&ibdm.ibdm_mutex);
3634 }
3635 }
3636
3637 /*
3638 * Do not deallocate the IBMF packet if atleast one request
3639 * is posted. IBMF packet is reused.
3640 */
3641 if (!(flag & IBDM_IBMF_PKT_REUSED)) {
3642 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3643 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3644 "IBMF free msg failed DM request drop it");
3645 }
3646 }
3647 }
3648
3649
3650 /*
3651 * ibdm_verify_mad_status()
3652 * Verifies the MAD status
3653 * Returns IBDM_SUCCESS if status is correct
3654 * Returns IBDM_FAILURE for bogus MAD status
3655 */
3656 static int
3657 ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
3658 {
3659 int ret = 0;
3660
3661 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
3662 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
3663 return (IBDM_FAILURE);
3664 }
3665
3666 if (b2h16(hdr->Status) == 0)
3667 ret = IBDM_SUCCESS;
3668 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
3669 ret = IBDM_SUCCESS;
3670 else {
3671 IBTF_DPRINTF_L2("ibdm",
3672 "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status));
3673 ret = IBDM_FAILURE;
3674 }
3675 return (ret);
3676 }
3677
3678
3679
3680 /*
3681 * ibdm_handle_redirection()
3682 * Returns IBDM_SUCCESS/IBDM_FAILURE
3683 */
3684 static int
3685 ibdm_handle_redirection(ibmf_msg_t *msg,
3686 ibdm_dp_gidinfo_t *gid_info, int *flag)
3687 {
3688 int attrmod, ioc_no, start;
3689 void *data;
3690 timeout_id_t *timeout_id;
3691 ib_mad_hdr_t *hdr;
3692 ibdm_ioc_info_t *ioc = NULL;
3693 ibdm_timeout_cb_args_t *cb_args;
3694 ib_mad_classportinfo_t *cpi;
3695
3696 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
3697 mutex_enter(&gid_info->gl_mutex);
3698 switch (gid_info->gl_state) {
3699 case IBDM_GET_IOUNITINFO:
3700 cb_args = &gid_info->gl_iou_cb_args;
3701 timeout_id = &gid_info->gl_timeout_id;
3702 break;
3703
3704 case IBDM_GET_IOC_DETAILS:
3705 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3706 switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3707
3708 case IB_DM_ATTR_DIAG_CODE:
3709 if (attrmod == 0) {
3710 cb_args = &gid_info->gl_iou_cb_args;
3711 timeout_id = &gid_info->gl_timeout_id;
3712 break;
3713 }
3714 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3715 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3716 "IOC# Out of range %d", attrmod);
3717 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3718 mutex_exit(&gid_info->gl_mutex);
3719 return (IBDM_FAILURE);
3720 }
3721 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3722 cb_args = &ioc->ioc_dc_cb_args;
3723 timeout_id = &ioc->ioc_dc_timeout_id;
3724 break;
3725
3726 case IB_DM_ATTR_IOC_CTRL_PROFILE:
3727 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3728 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3729 "IOC# Out of range %d", attrmod);
3730 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3731 mutex_exit(&gid_info->gl_mutex);
3732 return (IBDM_FAILURE);
3733 }
3734 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3735 cb_args = &ioc->ioc_cb_args;
3736 timeout_id = &ioc->ioc_timeout_id;
3737 break;
3738
3739 case IB_DM_ATTR_SERVICE_ENTRIES:
3740 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK);
3741 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
3742 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3743 "IOC# Out of range %d", ioc_no);
3744 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3745 mutex_exit(&gid_info->gl_mutex);
3746 return (IBDM_FAILURE);
3747 }
3748 start = (attrmod & IBDM_8_BIT_MASK);
3749 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3750 if (start > ioc->ioc_profile.ioc_service_entries) {
3751 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3752 " SE index Out of range %d", start);
3753 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3754 mutex_exit(&gid_info->gl_mutex);
3755 return (IBDM_FAILURE);
3756 }
3757 cb_args = &ioc->ioc_serv[start].se_cb_args;
3758 timeout_id = &ioc->ioc_serv[start].se_timeout_id;
3759 break;
3760
3761 default:
3762 /* ERROR State */
3763 IBTF_DPRINTF_L2("ibdm",
3764 "\thandle_redirection: wrong attribute :-(");
3765 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3766 mutex_exit(&gid_info->gl_mutex);
3767 return (IBDM_FAILURE);
3768 }
3769 break;
3770 default:
3771 /* ERROR State */
3772 IBTF_DPRINTF_L2("ibdm",
3773 "\thandle_redirection: Error state :-(");
3774 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3775 mutex_exit(&gid_info->gl_mutex);
3776 return (IBDM_FAILURE);
3777 }
3778 if ((*timeout_id) != 0) {
3779 mutex_exit(&gid_info->gl_mutex);
3780 if (untimeout(*timeout_id) == -1) {
3781 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
3782 "untimeout failed %x", *timeout_id);
3783 } else {
3784 IBTF_DPRINTF_L5("ibdm",
3785 "\thandle_redirection: timeout %x", *timeout_id);
3786 }
3787 mutex_enter(&gid_info->gl_mutex);
3788 *timeout_id = 0;
3789 }
3790
3791 data = msg->im_msgbufs_recv.im_bufs_cl_data;
3792 cpi = (ib_mad_classportinfo_t *)data;
3793
3794 gid_info->gl_resp_timeout =
3795 (b2h32(cpi->RespTimeValue) & 0x1F);
3796
3797 gid_info->gl_redirected = B_TRUE;
3798 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID);
3799 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff);
3800 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key);
3801 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key);
3802 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi);
3803 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo);
3804 gid_info->gl_redirectSL = cpi->RedirectSL;
3805
3806 if (gid_info->gl_redirect_dlid != 0) {
3807 msg->im_local_addr.ia_remote_lid =
3808 gid_info->gl_redirect_dlid;
3809 }
3810 ibdm_bump_transactionID(gid_info);
3811 mutex_exit(&gid_info->gl_mutex);
3812
3813 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3814 ibdm_alloc_send_buffers(msg);
3815
3816 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3817 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3818 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3819 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3820 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3821 hdr->Status = 0;
3822 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3823 hdr->AttributeID =
3824 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
3825 hdr->AttributeModifier =
3826 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
3827 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3828
3829 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3830 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3831 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3832 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3833
3834 mutex_enter(&gid_info->gl_mutex);
3835 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3836 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3837 mutex_exit(&gid_info->gl_mutex);
3838
3839 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
3840 "timeout %x", *timeout_id);
3841
3842 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3843 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3844 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
3845 "message transport failed");
3846 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3847 }
3848 (*flag) |= IBDM_IBMF_PKT_REUSED;
3849 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
3850 return (IBDM_SUCCESS);
3851 }
3852
3853
3854 /*
3855 * ibdm_pkt_timeout_hdlr
3856 * This timeout handler is registed for every IBMF packet that is
3857 * sent through the IBMF. It gets called when no response is received
3858 * within the specified time for the packet. No retries for the failed
3859 * commands currently. Drops the failed IBMF packet and update the
3860 * pending list commands.
3861 */
3862 static void
3863 ibdm_pkt_timeout_hdlr(void *arg)
3864 {
3865 ibdm_iou_info_t *iou;
3866 ibdm_ioc_info_t *ioc;
3867 ibdm_timeout_cb_args_t *cb_args = arg;
3868 ibdm_dp_gidinfo_t *gid_info;
3869 int srv_ent;
3870 uint_t new_gl_state;
3871
3872 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
3873 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3874 cb_args->cb_req_type, cb_args->cb_ioc_num,
3875 cb_args->cb_srvents_start);
3876
3877 gid_info = cb_args->cb_gid_info;
3878 mutex_enter(&gid_info->gl_mutex);
3879
3880 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
3881 (cb_args->cb_req_type == 0)) {
3882
3883 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
3884 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
3885 cb_args->cb_ioc_num, cb_args->cb_srvents_start);
3886
3887 if (gid_info->gl_timeout_id)
3888 gid_info->gl_timeout_id = 0;
3889 mutex_exit(&gid_info->gl_mutex);
3890 return;
3891 }
3892 if (cb_args->cb_retry_count) {
3893 cb_args->cb_retry_count--;
3894 /*
3895 * A new timeout_id is set inside ibdm_retry_command().
3896 * When the function returns an error, the timeout_id
3897 * is reset (to zero) in the switch statement below.
3898 */
3899 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
3900 mutex_exit(&gid_info->gl_mutex);
3901 return;
3902 }
3903 cb_args->cb_retry_count = 0;
3904 }
3905
3906 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
3907 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3908 cb_args->cb_req_type, cb_args->cb_ioc_num,
3909 cb_args->cb_srvents_start);
3910
3911 switch (cb_args->cb_req_type) {
3912
3913 case IBDM_REQ_TYPE_CLASSPORTINFO:
3914 case IBDM_REQ_TYPE_IOUINFO:
3915 new_gl_state = IBDM_GID_PROBING_FAILED;
3916 if (gid_info->gl_timeout_id)
3917 gid_info->gl_timeout_id = 0;
3918 break;
3919
3920 case IBDM_REQ_TYPE_IOCINFO:
3921 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3922 iou = gid_info->gl_iou;
3923 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3924 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3925 if (ioc->ioc_timeout_id)
3926 ioc->ioc_timeout_id = 0;
3927 break;
3928
3929 case IBDM_REQ_TYPE_SRVENTS:
3930 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3931 iou = gid_info->gl_iou;
3932 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3933 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3934 srv_ent = cb_args->cb_srvents_start;
3935 if (ioc->ioc_serv[srv_ent].se_timeout_id)
3936 ioc->ioc_serv[srv_ent].se_timeout_id = 0;
3937 break;
3938
3939 case IBDM_REQ_TYPE_IOU_DIAGCODE:
3940 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3941 iou = gid_info->gl_iou;
3942 iou->iou_dc_valid = B_FALSE;
3943 if (gid_info->gl_timeout_id)
3944 gid_info->gl_timeout_id = 0;
3945 break;
3946
3947 case IBDM_REQ_TYPE_IOC_DIAGCODE:
3948 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3949 iou = gid_info->gl_iou;
3950 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3951 ioc->ioc_dc_valid = B_FALSE;
3952 if (ioc->ioc_dc_timeout_id)
3953 ioc->ioc_dc_timeout_id = 0;
3954 break;
3955
3956 default: /* ERROR State */
3957 new_gl_state = IBDM_GID_PROBING_FAILED;
3958 if (gid_info->gl_timeout_id)
3959 gid_info->gl_timeout_id = 0;
3960 IBTF_DPRINTF_L2("ibdm",
3961 "\tpkt_timeout_hdlr: wrong request type.");
3962 break;
3963 }
3964
3965 --gid_info->gl_pending_cmds; /* decrease the counter */
3966
3967 if (gid_info->gl_pending_cmds == 0) {
3968 gid_info->gl_state = new_gl_state;
3969 mutex_exit(&gid_info->gl_mutex);
3970 /*
3971 * Delete this gid_info if the gid probe fails.
3972 */
3973 if (new_gl_state == IBDM_GID_PROBING_FAILED) {
3974 ibdm_delete_glhca_list(gid_info);
3975 }
3976 ibdm_notify_newgid_iocs(gid_info);
3977 mutex_enter(&ibdm.ibdm_mutex);
3978 if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3979 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
3980 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3981 cv_broadcast(&ibdm.ibdm_probe_cv);
3982 }
3983 mutex_exit(&ibdm.ibdm_mutex);
3984 } else {
3985 /*
3986 * Reset gl_pending_cmd if the extra timeout happens since
3987 * gl_pending_cmd becomes negative as a result.
3988 */
3989 if (gid_info->gl_pending_cmds < 0) {
3990 gid_info->gl_pending_cmds = 0;
3991 IBTF_DPRINTF_L2("ibdm",
3992 "\tpkt_timeout_hdlr: extra timeout request."
3993 " reset gl_pending_cmds");
3994 }
3995 mutex_exit(&gid_info->gl_mutex);
3996 /*
3997 * Delete this gid_info if the gid probe fails.
3998 */
3999 if (new_gl_state == IBDM_GID_PROBING_FAILED) {
4000 ibdm_delete_glhca_list(gid_info);
4001 }
4002 }
4003 }
4004
4005
4006 /*
4007 * ibdm_retry_command()
4008 * Retries the failed command.
4009 * Returns IBDM_FAILURE/IBDM_SUCCESS
4010 */
4011 static int
4012 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
4013 {
4014 int ret;
4015 ibmf_msg_t *msg;
4016 ib_mad_hdr_t *hdr;
4017 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info;
4018 timeout_id_t *timeout_id;
4019 ibdm_ioc_info_t *ioc;
4020 int ioc_no;
4021 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4022
4023 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
4024 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4025 cb_args->cb_req_type, cb_args->cb_ioc_num,
4026 cb_args->cb_srvents_start);
4027
4028 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);
4029
4030
4031 /*
4032 * Reset the gid if alloc_msg failed with BAD_HANDLE
4033 * ibdm_reset_gidinfo reinits the gid_info
4034 */
4035 if (ret == IBMF_BAD_HANDLE) {
4036 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
4037 gid_info);
4038
4039 mutex_exit(&gid_info->gl_mutex);
4040 ibdm_reset_gidinfo(gid_info);
4041 mutex_enter(&gid_info->gl_mutex);
4042
4043 /* Retry alloc */
4044 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
4045 &msg);
4046 }
4047
4048 if (ret != IBDM_SUCCESS) {
4049 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
4050 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4051 cb_args->cb_req_type, cb_args->cb_ioc_num,
4052 cb_args->cb_srvents_start);
4053 return (IBDM_FAILURE);
4054 }
4055
4056 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
4057 ibdm_alloc_send_buffers(msg);
4058 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
4059
4060 ibdm_bump_transactionID(gid_info);
4061
4062 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
4063 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
4064 if (gid_info->gl_redirected == B_TRUE) {
4065 if (gid_info->gl_redirect_dlid != 0) {
4066 msg->im_local_addr.ia_remote_lid =
4067 gid_info->gl_redirect_dlid;
4068 }
4069 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
4070 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
4071 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
4072 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
4073 } else {
4074 msg->im_local_addr.ia_remote_qno = 1;
4075 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
4076 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
4077 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
4078 }
4079 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
4080 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
4081 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
4082 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
4083 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
4084 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
4085 hdr->Status = 0;
4086 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
4087
4088 switch (cb_args->cb_req_type) {
4089 case IBDM_REQ_TYPE_CLASSPORTINFO:
4090 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
4091 hdr->AttributeModifier = 0;
4092 timeout_id = &gid_info->gl_timeout_id;
4093 break;
4094 case IBDM_REQ_TYPE_IOUINFO:
4095 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
4096 hdr->AttributeModifier = 0;
4097 timeout_id = &gid_info->gl_timeout_id;
4098 break;
4099 case IBDM_REQ_TYPE_IOCINFO:
4100 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
4101 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4102 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4103 timeout_id = &ioc->ioc_timeout_id;
4104 break;
4105 case IBDM_REQ_TYPE_SRVENTS:
4106 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
4107 ibdm_fill_srv_attr_mod(hdr, cb_args);
4108 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4109 timeout_id =
4110 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
4111 break;
4112 case IBDM_REQ_TYPE_IOU_DIAGCODE:
4113 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4114 hdr->AttributeModifier = 0;
4115 timeout_id = &gid_info->gl_timeout_id;
4116 break;
4117 case IBDM_REQ_TYPE_IOC_DIAGCODE:
4118 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4119 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4120 ioc_no = cb_args->cb_ioc_num;
4121 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
4122 timeout_id = &ioc->ioc_dc_timeout_id;
4123 break;
4124 }
4125 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr))
4126
4127 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
4128 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
4129
4130 mutex_exit(&gid_info->gl_mutex);
4131
4132 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
4133 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
4134 cb_args->cb_srvents_start, *timeout_id);
4135
4136 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl,
4137 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
4138 cb_args, 0) != IBMF_SUCCESS) {
4139 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
4140 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4141 cb_args->cb_req_type, cb_args->cb_ioc_num,
4142 cb_args->cb_srvents_start);
4143 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
4144 }
4145 mutex_enter(&gid_info->gl_mutex);
4146 return (IBDM_SUCCESS);
4147 }
4148
4149
4150 /*
4151 * ibdm_update_ioc_port_gidlist()
4152 */
4153 static void
4154 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
4155 ibdm_dp_gidinfo_t *gid_info)
4156 {
4157 int ii, ngid_ents;
4158 ibdm_gid_t *tmp;
4159 ibdm_hca_list_t *gid_hca_head, *temp;
4160 ibdm_hca_list_t *ioc_head = NULL;
4161 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4162
4163 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");
4164
4165 ngid_ents = gid_info->gl_ngids;
4166 dest->ioc_nportgids = ngid_ents;
4167 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
4168 ngid_ents, KM_SLEEP);
4169 tmp = gid_info->gl_gid;
4170 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
4171 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
4172 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
4173 tmp = tmp->gid_next;
4174 }
4175
4176 gid_hca_head = gid_info->gl_hca_list;
4177 while (gid_hca_head) {
4178 temp = ibdm_dup_hca_attr(gid_hca_head);
4179 temp->hl_next = ioc_head;
4180 ioc_head = temp;
4181 gid_hca_head = gid_hca_head->hl_next;
4182 }
4183 dest->ioc_hca_list = ioc_head;
4184 }
4185
4186
4187 /*
4188 * ibdm_alloc_send_buffers()
4189 * Allocates memory for the IBMF send buffer to send and/or receive
4190 * the Device Management MAD packet.
4191 */
4192 static void
4193 ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
4194 {
4195 msgp->im_msgbufs_send.im_bufs_mad_hdr =
4196 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);
4197
4198 msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *)
4199 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
4200 msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ;
4201
4202 msgp->im_msgbufs_send.im_bufs_cl_data =
4203 ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ);
4204 msgp->im_msgbufs_send.im_bufs_cl_data_len =
4205 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ;
4206 }
4207
4208
4209 /*
4210 * ibdm_alloc_send_buffers()
4211 * De-allocates memory for the IBMF send buffer
4212 */
4213 static void
4214 ibdm_free_send_buffers(ibmf_msg_t *msgp)
4215 {
4216 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
4217 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
4218 }
4219
4220 /*
4221 * ibdm_probe_ioc()
4222 * 1. Gets the node records for the port GUID. This detects all the port
4223 * to the IOU.
4224 * 2. Selectively probes all the IOC, given it's node GUID
4225 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC
4226 * Controller Profile asynchronously
4227 */
4228 /*ARGSUSED*/
4229 static void
4230 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
4231 {
4232 int ii, nrecords;
4233 size_t nr_len = 0, pi_len = 0;
4234 ib_gid_t sgid, dgid;
4235 ibdm_hca_list_t *hca_list = NULL;
4236 sa_node_record_t *nr, *tmp;
4237 ibdm_port_attr_t *port = NULL;
4238 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid;
4239 ibdm_dp_gidinfo_t *temp_gidinfo;
4240 ibdm_gid_t *temp_gid;
4241 sa_portinfo_record_t *pi;
4242
4243 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin",
4244 nodeguid, ioc_guid, reprobe_flag);
4245
4246 /* Rescan the GID list for any removed GIDs for reprobe */
4247 if (reprobe_flag)
4248 ibdm_rescan_gidlist(&ioc_guid);
4249
4250 mutex_enter(&ibdm.ibdm_hl_mutex);
4251 for (ibdm_get_next_port(&hca_list, &port, 1); port;
4252 ibdm_get_next_port(&hca_list, &port, 1)) {
4253 reprobe_gid = new_gid = node_gid = NULL;
4254
4255 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
4256 if (nr == NULL) {
4257 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
4258 continue;
4259 }
4260 nrecords = (nr_len / sizeof (sa_node_record_t));
4261 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) {
4262 if ((pi = ibdm_get_portinfo(
4263 port->pa_sa_hdl, &pi_len, tmp->LID)) == NULL) {
4264 IBTF_DPRINTF_L4("ibdm",
4265 "\tibdm_get_portinfo: no portinfo recs");
4266 continue;
4267 }
4268
4269 /*
4270 * If Device Management is not supported on
4271 * this port, skip the rest.
4272 */
4273 if (!(pi->PortInfo.CapabilityMask &
4274 SM_CAP_MASK_IS_DM_SUPPD)) {
4275 kmem_free(pi, pi_len);
4276 continue;
4277 }
4278
4279 /*
4280 * For reprobes: Check if GID, already in
4281 * the list. If so, set the state to SKIPPED
4282 */
4283 if (((temp_gidinfo = ibdm_find_gid(nodeguid,
4284 tmp->NodeInfo.PortGUID)) != NULL) &&
4285 temp_gidinfo->gl_state ==
4286 IBDM_GID_PROBING_COMPLETE) {
4287 ASSERT(reprobe_gid == NULL);
4288 ibdm_addto_glhcalist(temp_gidinfo,
4289 hca_list);
4290 reprobe_gid = temp_gidinfo;
4291 kmem_free(pi, pi_len);
4292 continue;
4293 } else if (temp_gidinfo != NULL) {
4294 kmem_free(pi, pi_len);
4295 ibdm_addto_glhcalist(temp_gidinfo,
4296 hca_list);
4297 continue;
4298 }
4299
4300 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
4301 "create_gid : prefix %llx, guid %llx\n",
4302 pi->PortInfo.GidPrefix,
4303 tmp->NodeInfo.PortGUID);
4304
4305 sgid.gid_prefix = port->pa_sn_prefix;
4306 sgid.gid_guid = port->pa_port_guid;
4307 dgid.gid_prefix = pi->PortInfo.GidPrefix;
4308 dgid.gid_guid = tmp->NodeInfo.PortGUID;
4309 new_gid = ibdm_create_gid_info(port, sgid,
4310 dgid);
4311 if (new_gid == NULL) {
4312 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4313 "create_gid_info failed\n");
4314 kmem_free(pi, pi_len);
4315 continue;
4316 }
4317 if (node_gid == NULL) {
4318 node_gid = new_gid;
4319 ibdm_add_to_gl_gid(node_gid, node_gid);
4320 } else {
4321 IBTF_DPRINTF_L4("ibdm",
4322 "\tprobe_ioc: new gid");
4323 temp_gid = kmem_zalloc(
4324 sizeof (ibdm_gid_t), KM_SLEEP);
4325 temp_gid->gid_dgid_hi =
4326 new_gid->gl_dgid_hi;
4327 temp_gid->gid_dgid_lo =
4328 new_gid->gl_dgid_lo;
4329 temp_gid->gid_next = node_gid->gl_gid;
4330 node_gid->gl_gid = temp_gid;
4331 node_gid->gl_ngids++;
4332 }
4333 new_gid->gl_is_dm_capable = B_TRUE;
4334 new_gid->gl_nodeguid = nodeguid;
4335 new_gid->gl_portguid = dgid.gid_guid;
4336 ibdm_addto_glhcalist(new_gid, hca_list);
4337
4338 /*
4339 * Set the state to skipped as all these
4340 * gids point to the same node.
4341 * We (re)probe only one GID below and reset
4342 * state appropriately
4343 */
4344 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
4345 new_gid->gl_devid = (*tmp).NodeInfo.DeviceID;
4346 kmem_free(pi, pi_len);
4347 }
4348 kmem_free(nr, nr_len);
4349
4350 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
4351 "reprobe_gid %p new_gid %p node_gid %p",
4352 reprobe_flag, reprobe_gid, new_gid, node_gid);
4353
4354 if (reprobe_flag != 0 && reprobe_gid != NULL) {
4355 int niocs, jj;
4356 ibdm_ioc_info_t *tmp_ioc;
4357 int ioc_matched = 0;
4358
4359 mutex_exit(&ibdm.ibdm_hl_mutex);
4360 mutex_enter(&reprobe_gid->gl_mutex);
4361 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
4362 niocs =
4363 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
4364 reprobe_gid->gl_pending_cmds++;
4365 mutex_exit(&reprobe_gid->gl_mutex);
4366
4367 for (jj = 0; jj < niocs; jj++) {
4368 tmp_ioc =
4369 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
4370 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
4371 continue;
4372
4373 ioc_matched = 1;
4374
4375 /*
4376 * Explicitly set gl_reprobe_flag to 0 so that
4377 * IBnex is not notified on completion
4378 */
4379 mutex_enter(&reprobe_gid->gl_mutex);
4380 reprobe_gid->gl_reprobe_flag = 0;
4381 mutex_exit(&reprobe_gid->gl_mutex);
4382
4383 mutex_enter(&ibdm.ibdm_mutex);
4384 ibdm.ibdm_ngid_probes_in_progress++;
4385 mutex_exit(&ibdm.ibdm_mutex);
4386 if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
4387 IBDM_SUCCESS) {
4388 IBTF_DPRINTF_L4("ibdm",
4389 "\tprobe_ioc: "
4390 "send_ioc_profile failed "
4391 "for ioc %d", jj);
4392 ibdm_gid_decr_pending(reprobe_gid);
4393 break;
4394 }
4395 mutex_enter(&ibdm.ibdm_mutex);
4396 ibdm_wait_probe_completion();
4397 mutex_exit(&ibdm.ibdm_mutex);
4398 break;
4399 }
4400 if (ioc_matched == 0)
4401 ibdm_gid_decr_pending(reprobe_gid);
4402 else {
4403 mutex_enter(&ibdm.ibdm_hl_mutex);
4404 break;
4405 }
4406 } else if (new_gid != NULL) {
4407 mutex_exit(&ibdm.ibdm_hl_mutex);
4408 node_gid = node_gid ? node_gid : new_gid;
4409
4410 /*
4411 * New or reinserted GID : Enable notification
4412 * to IBnex
4413 */
4414 mutex_enter(&node_gid->gl_mutex);
4415 node_gid->gl_reprobe_flag = 1;
4416 mutex_exit(&node_gid->gl_mutex);
4417
4418 ibdm_probe_gid(node_gid);
4419
4420 mutex_enter(&ibdm.ibdm_hl_mutex);
4421 }
4422 }
4423 mutex_exit(&ibdm.ibdm_hl_mutex);
4424 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
4425 }
4426
4427
4428 /*
4429 * ibdm_probe_gid()
4430 * Selectively probes the GID
4431 */
4432 static void
4433 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
4434 {
4435 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");
4436
4437 /*
4438 * A Cisco FC GW needs the special handling to get IOUnitInfo.
4439 */
4440 mutex_enter(&gid_info->gl_mutex);
4441 if (ibdm_is_cisco_switch(gid_info)) {
4442 gid_info->gl_pending_cmds++;
4443 gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
4444 mutex_exit(&gid_info->gl_mutex);
4445
4446 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
4447
4448 mutex_enter(&gid_info->gl_mutex);
4449 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4450 --gid_info->gl_pending_cmds;
4451 mutex_exit(&gid_info->gl_mutex);
4452
4453 /* free the hca_list on this gid_info */
4454 ibdm_delete_glhca_list(gid_info);
4455 gid_info = gid_info->gl_next;
4456 return;
4457 }
4458
4459 mutex_enter(&gid_info->gl_mutex);
4460 ibdm_wait_cisco_probe_completion(gid_info);
4461
4462 IBTF_DPRINTF_L4("ibdm",
4463 "\tprobe_gid: CISCO Wakeup signal received");
4464 }
4465
4466 /* move on to the 'GET_CLASSPORTINFO' stage */
4467 gid_info->gl_pending_cmds++;
4468 gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
4469 mutex_exit(&gid_info->gl_mutex);
4470
4471 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
4472
4473 mutex_enter(&gid_info->gl_mutex);
4474 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4475 --gid_info->gl_pending_cmds;
4476 mutex_exit(&gid_info->gl_mutex);
4477
4478 /* free the hca_list on this gid_info */
4479 ibdm_delete_glhca_list(gid_info);
4480 gid_info = gid_info->gl_next;
4481 return;
4482 }
4483
4484 mutex_enter(&ibdm.ibdm_mutex);
4485 ibdm.ibdm_ngid_probes_in_progress++;
4486 gid_info = gid_info->gl_next;
4487 ibdm_wait_probe_completion();
4488 mutex_exit(&ibdm.ibdm_mutex);
4489
4490 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
4491 }
4492
4493
4494 /*
4495 * ibdm_create_gid_info()
4496 * Allocates a gid_info structure and initializes
4497 * Returns pointer to the structure on success
4498 * and NULL on failure
4499 */
4500 static ibdm_dp_gidinfo_t *
4501 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
4502 {
4503 uint8_t ii, npaths;
4504 sa_path_record_t *path;
4505 size_t len;
4506 ibdm_pkey_tbl_t *pkey_tbl;
4507 ibdm_dp_gidinfo_t *gid_info = NULL;
4508 int ret;
4509
4510 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
4511 npaths = 1;
4512
4513 /* query for reversible paths */
4514 if (port->pa_sa_hdl)
4515 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
4516 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
4517 &len, &path);
4518 else
4519 return (NULL);
4520
4521 if (ret == IBMF_SUCCESS && path) {
4522 ibdm_dump_path_info(path);
4523
4524 gid_info = kmem_zalloc(
4525 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
4526 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
4527 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
4528 gid_info->gl_dgid_hi = path->DGID.gid_prefix;
4529 gid_info->gl_dgid_lo = path->DGID.gid_guid;
4530 gid_info->gl_sgid_hi = path->SGID.gid_prefix;
4531 gid_info->gl_sgid_lo = path->SGID.gid_guid;
4532 gid_info->gl_p_key = path->P_Key;
4533 gid_info->gl_sa_hdl = port->pa_sa_hdl;
4534 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl;
4535 gid_info->gl_slid = path->SLID;
4536 gid_info->gl_dlid = path->DLID;
4537 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID)
4538 << IBDM_GID_TRANSACTIONID_SHIFT;
4539 gid_info->gl_min_transactionID = gid_info->gl_transactionID;
4540 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1)
4541 << IBDM_GID_TRANSACTIONID_SHIFT;
4542 gid_info->gl_SL = path->SL;
4543
4544 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
4545 for (ii = 0; ii < port->pa_npkeys; ii++) {
4546 if (port->pa_pkey_tbl == NULL)
4547 break;
4548
4549 pkey_tbl = &port->pa_pkey_tbl[ii];
4550 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
4551 (pkey_tbl->pt_qp_hdl != NULL)) {
4552 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
4553 break;
4554 }
4555 }
4556 kmem_free(path, len);
4557
4558 /*
4559 * QP handle for GID not initialized. No matching Pkey
4560 * was found!! ibdm should *not* hit this case. Flag an
4561 * error and drop the GID if ibdm does encounter this.
4562 */
4563 if (gid_info->gl_qp_hdl == NULL) {
4564 IBTF_DPRINTF_L2(ibdm_string,
4565 "\tcreate_gid_info: No matching Pkey");
4566 ibdm_delete_gidinfo(gid_info);
4567 return (NULL);
4568 }
4569
4570 ibdm.ibdm_ngids++;
4571 if (ibdm.ibdm_dp_gidlist_head == NULL) {
4572 ibdm.ibdm_dp_gidlist_head = gid_info;
4573 ibdm.ibdm_dp_gidlist_tail = gid_info;
4574 } else {
4575 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
4576 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
4577 ibdm.ibdm_dp_gidlist_tail = gid_info;
4578 }
4579 }
4580
4581 return (gid_info);
4582 }
4583
4584
4585 /*
4586 * ibdm_get_node_records
4587 * Sends a SA query to get the NODE record
4588 * Returns pointer to the sa_node_record_t on success
4589 * and NULL on failure
4590 */
4591 static sa_node_record_t *
4592 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
4593 {
4594 sa_node_record_t req, *resp = NULL;
4595 ibmf_saa_access_args_t args;
4596 int ret;
4597
4598 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");
4599
4600 bzero(&req, sizeof (sa_node_record_t));
4601 req.NodeInfo.NodeGUID = guid;
4602
4603 args.sq_attr_id = SA_NODERECORD_ATTRID;
4604 args.sq_access_type = IBMF_SAA_RETRIEVE;
4605 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID;
4606 args.sq_template = &req;
4607 args.sq_callback = NULL;
4608 args.sq_callback_arg = NULL;
4609
4610 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4611 if (ret != IBMF_SUCCESS) {
4612 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
4613 " SA Retrieve Failed: %d", ret);
4614 return (NULL);
4615 }
4616 if ((resp == NULL) || (*length == 0)) {
4617 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
4618 return (NULL);
4619 }
4620
4621 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
4622 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);
4623
4624 return (resp);
4625 }
4626
4627
4628 /*
4629 * ibdm_get_portinfo()
4630 * Sends a SA query to get the PortInfo record
4631 * Returns pointer to the sa_portinfo_record_t on success
4632 * and NULL on failure
4633 */
4634 static sa_portinfo_record_t *
4635 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
4636 {
4637 sa_portinfo_record_t req, *resp = NULL;
4638 ibmf_saa_access_args_t args;
4639 int ret;
4640
4641 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");
4642
4643 bzero(&req, sizeof (sa_portinfo_record_t));
4644 req.EndportLID = lid;
4645
4646 args.sq_attr_id = SA_PORTINFORECORD_ATTRID;
4647 args.sq_access_type = IBMF_SAA_RETRIEVE;
4648 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
4649 args.sq_template = &req;
4650 args.sq_callback = NULL;
4651 args.sq_callback_arg = NULL;
4652
4653 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4654 if (ret != IBMF_SUCCESS) {
4655 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
4656 " SA Retrieve Failed: 0x%X", ret);
4657 return (NULL);
4658 }
4659 if ((*length == 0) || (resp == NULL))
4660 return (NULL);
4661
4662 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
4663 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
4664 return (resp);
4665 }
4666
4667
4668 /*
4669 * ibdm_ibnex_register_callback
4670 * IB nexus callback routine for HCA attach and detach notification
4671 */
4672 void
4673 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
4674 {
4675 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
4676 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4677 ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
4678 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4679 }
4680
4681
4682 /*
4683 * ibdm_ibnex_unregister_callbacks
4684 */
4685 void
4686 ibdm_ibnex_unregister_callback()
4687 {
4688 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4689 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4690 ibdm.ibdm_ibnex_callback = NULL;
4691 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4692 }
4693
4694 /*
4695 * ibdm_get_waittime()
4696 * Calculates the wait time based on the last HCA attach time
4697 */
4698 static clock_t
4699 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait_sec)
4700 {
4701 const hrtime_t dft_wait = dft_wait_sec * NANOSEC;
4702 hrtime_t temp, wait_time = 0;
4703 clock_t usecs;
4704 int i;
4705 ibdm_hca_list_t *hca;
4706
4707 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4708 "\tport settling time %d", hca_guid, dft_wait);
4709
4710 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4711
4712 hca = ibdm.ibdm_hca_list_head;
4713
4714 for (i = 0; i < ibdm.ibdm_hca_count; i++, hca = hca->hl_next) {
4715 if (hca->hl_nports == hca->hl_nports_active)
4716 continue;
4717
4718 if (hca_guid && (hca_guid != hca->hl_hca_guid))
4719 continue;
4720
4721 temp = gethrtime() - hca->hl_attach_time;
4722 temp = MAX(0, (dft_wait - temp));
4723
4724 if (hca_guid) {
4725 wait_time = temp;
4726 break;
4727 }
4728
4729 wait_time = MAX(temp, wait_time);
4730 }
4731
4732 /* convert to microseconds */
4733 usecs = MIN(wait_time, dft_wait) / (NANOSEC / MICROSEC);
4734
4735 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld usecs",
4736 (long)usecs);
4737
4738 return (drv_usectohz(usecs));
4739 }
4740
4741 void
4742 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4743 {
4744 clock_t wait_time;
4745
4746 mutex_enter(&ibdm.ibdm_hl_mutex);
4747
4748 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0)
4749 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4750 &ibdm.ibdm_hl_mutex, wait_time, TR_CLOCK_TICK);
4751
4752 mutex_exit(&ibdm.ibdm_hl_mutex);
4753 }
4754
4755
4756 /*
4757 * ibdm_ibnex_probe_hcaport
4758 * Probes the presence of HCA port (with HCA dip and port number)
4759 * Returns port attributes structure on SUCCESS
4760 */
4761 ibdm_port_attr_t *
4762 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4763 {
4764 int ii, jj;
4765 ibdm_hca_list_t *hca_list;
4766 ibdm_port_attr_t *port_attr;
4767
4768 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4769
4770 mutex_enter(&ibdm.ibdm_hl_mutex);
4771 hca_list = ibdm.ibdm_hca_list_head;
4772 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4773 if (hca_list->hl_hca_guid == hca_guid) {
4774 for (jj = 0; jj < hca_list->hl_nports; jj++) {
4775 if (hca_list->hl_port_attr[jj].pa_port_num ==
4776 port_num) {
4777 break;
4778 }
4779 }
4780 if (jj != hca_list->hl_nports)
4781 break;
4782 }
4783 hca_list = hca_list->hl_next;
4784 }
4785 if (ii == ibdm.ibdm_hca_count) {
4786 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
4787 mutex_exit(&ibdm.ibdm_hl_mutex);
4788 return (NULL);
4789 }
4790 port_attr = (ibdm_port_attr_t *)kmem_zalloc(
4791 sizeof (ibdm_port_attr_t), KM_SLEEP);
4792 bcopy((char *)&hca_list->hl_port_attr[jj],
4793 port_attr, sizeof (ibdm_port_attr_t));
4794 ibdm_update_port_attr(port_attr);
4795
4796 mutex_exit(&ibdm.ibdm_hl_mutex);
4797 return (port_attr);
4798 }
4799
4800
4801 /*
4802 * ibdm_ibnex_get_port_attrs
4803 * Scan all HCAs for a matching port_guid.
4804 * Returns "port attributes" structure on success.
4805 */
4806 ibdm_port_attr_t *
4807 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
4808 {
4809 int ii, jj;
4810 ibdm_hca_list_t *hca_list;
4811 ibdm_port_attr_t *port_attr;
4812
4813 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");
4814
4815 mutex_enter(&ibdm.ibdm_hl_mutex);
4816 hca_list = ibdm.ibdm_hca_list_head;
4817
4818 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4819 for (jj = 0; jj < hca_list->hl_nports; jj++) {
4820 if (hca_list->hl_port_attr[jj].pa_port_guid ==
4821 port_guid) {
4822 break;
4823 }
4824 }
4825 if (jj != hca_list->hl_nports)
4826 break;
4827 hca_list = hca_list->hl_next;
4828 }
4829
4830 if (ii == ibdm.ibdm_hca_count) {
4831 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
4832 mutex_exit(&ibdm.ibdm_hl_mutex);
4833 return (NULL);
4834 }
4835
4836 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
4837 KM_SLEEP);
4838 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
4839 sizeof (ibdm_port_attr_t));
4840 ibdm_update_port_attr(port_attr);
4841
4842 mutex_exit(&ibdm.ibdm_hl_mutex);
4843 return (port_attr);
4844 }
4845
4846
4847 /*
4848 * ibdm_ibnex_free_port_attr()
4849 */
4850 void
4851 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
4852 {
4853 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
4854 if (port_attr) {
4855 if (port_attr->pa_pkey_tbl != NULL) {
4856 kmem_free(port_attr->pa_pkey_tbl,
4857 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
4858 }
4859 kmem_free(port_attr, sizeof (ibdm_port_attr_t));
4860 }
4861 }
4862
4863
4864 /*
4865 * ibdm_ibnex_get_hca_list()
4866 * Returns portinfo for all the port for all the HCA's
4867 */
4868 void
4869 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
4870 {
4871 ibdm_hca_list_t *head = NULL, *temp, *temp1;
4872 int ii;
4873
4874 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");
4875
4876 mutex_enter(&ibdm.ibdm_hl_mutex);
4877 temp = ibdm.ibdm_hca_list_head;
4878 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4879 temp1 = ibdm_dup_hca_attr(temp);
4880 temp1->hl_next = head;
4881 head = temp1;
4882 temp = temp->hl_next;
4883 }
4884 *count = ibdm.ibdm_hca_count;
4885 *hca = head;
4886 mutex_exit(&ibdm.ibdm_hl_mutex);
4887 }
4888
4889
4890 /*
4891 * ibdm_ibnex_get_hca_info_by_guid()
4892 */
4893 ibdm_hca_list_t *
4894 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
4895 {
4896 ibdm_hca_list_t *head = NULL, *hca = NULL;
4897
4898 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");
4899
4900 mutex_enter(&ibdm.ibdm_hl_mutex);
4901 head = ibdm.ibdm_hca_list_head;
4902 while (head) {
4903 if (head->hl_hca_guid == hca_guid) {
4904 hca = ibdm_dup_hca_attr(head);
4905 hca->hl_next = NULL;
4906 break;
4907 }
4908 head = head->hl_next;
4909 }
4910 mutex_exit(&ibdm.ibdm_hl_mutex);
4911 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
4912 return (hca);
4913 }
4914
4915
4916 /*
4917 * ibdm_dup_hca_attr()
4918 * Allocate a new HCA attribute strucuture and initialize
4919 * hca attribute structure with the incoming HCA attributes
4920 * returned the allocated hca attributes.
4921 */
4922 static ibdm_hca_list_t *
4923 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
4924 {
4925 int len;
4926 ibdm_hca_list_t *out_hca;
4927
4928 len = sizeof (ibdm_hca_list_t) +
4929 (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
4930 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
4931 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
4932 bcopy((char *)in_hca,
4933 (char *)out_hca, sizeof (ibdm_hca_list_t));
4934 if (in_hca->hl_nports) {
4935 out_hca->hl_port_attr = (ibdm_port_attr_t *)
4936 ((char *)out_hca + sizeof (ibdm_hca_list_t));
4937 bcopy((char *)in_hca->hl_port_attr,
4938 (char *)out_hca->hl_port_attr,
4939 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
4940 for (len = 0; len < out_hca->hl_nports; len++)
4941 ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
4942 }
4943 return (out_hca);
4944 }
4945
4946
4947 /*
4948 * ibdm_ibnex_free_hca_list()
4949 * Free one/more HCA lists
4950 */
4951 void
4952 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
4953 {
4954 int ii;
4955 size_t len;
4956 ibdm_hca_list_t *temp;
4957 ibdm_port_attr_t *port;
4958
4959 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
4960 ASSERT(hca_list);
4961 while (hca_list) {
4962 temp = hca_list;
4963 hca_list = hca_list->hl_next;
4964 for (ii = 0; ii < temp->hl_nports; ii++) {
4965 port = &temp->hl_port_attr[ii];
4966 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
4967 if (len != 0)
4968 kmem_free(port->pa_pkey_tbl, len);
4969 }
4970 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
4971 sizeof (ibdm_port_attr_t));
4972 kmem_free(temp, len);
4973 }
4974 }
4975
4976
4977 /*
4978 * ibdm_ibnex_probe_iocguid()
4979 * Probes the IOC on the fabric and returns the IOC information
4980 * if present. Otherwise, NULL is returned
4981 */
4982 /* ARGSUSED */
4983 ibdm_ioc_info_t *
4984 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
4985 {
4986 int k;
4987 ibdm_ioc_info_t *ioc_info;
4988 ibdm_dp_gidinfo_t *gid_info; /* used as index and arg */
4989 timeout_id_t *timeout_id;
4990
4991 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
4992 iou, ioc_guid, reprobe_flag);
4993
4994 if (ibdm_enumerate_iocs == 0)
4995 return (NULL);
4996
4997 /* Check whether we know this already */
4998 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
4999 if (ioc_info == NULL) {
5000 mutex_enter(&ibdm.ibdm_mutex);
5001 while (ibdm.ibdm_busy & IBDM_BUSY)
5002 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5003 ibdm.ibdm_busy |= IBDM_BUSY;
5004 mutex_exit(&ibdm.ibdm_mutex);
5005 ibdm_probe_ioc(iou, ioc_guid, 0);
5006 mutex_enter(&ibdm.ibdm_mutex);
5007 ibdm.ibdm_busy &= ~IBDM_BUSY;
5008 cv_broadcast(&ibdm.ibdm_busy_cv);
5009 mutex_exit(&ibdm.ibdm_mutex);
5010 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5011 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */
5012 ASSERT(gid_info != NULL);
5013 /* Free the ioc_list before reprobe; and cancel any timers */
5014 mutex_enter(&ibdm.ibdm_mutex);
5015 mutex_enter(&gid_info->gl_mutex);
5016 if (ioc_info->ioc_timeout_id) {
5017 timeout_id = ioc_info->ioc_timeout_id;
5018 ioc_info->ioc_timeout_id = 0;
5019 mutex_exit(&gid_info->gl_mutex);
5020 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5021 "ioc_timeout_id = 0x%x", timeout_id);
5022 if (untimeout(timeout_id) == -1) {
5023 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5024 "untimeout ioc_timeout_id failed");
5025 }
5026 mutex_enter(&gid_info->gl_mutex);
5027 }
5028 if (ioc_info->ioc_dc_timeout_id) {
5029 timeout_id = ioc_info->ioc_dc_timeout_id;
5030 ioc_info->ioc_dc_timeout_id = 0;
5031 mutex_exit(&gid_info->gl_mutex);
5032 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5033 "ioc_dc_timeout_id = 0x%x", timeout_id);
5034 if (untimeout(timeout_id) == -1) {
5035 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5036 "untimeout ioc_dc_timeout_id failed");
5037 }
5038 mutex_enter(&gid_info->gl_mutex);
5039 }
5040 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
5041 if (ioc_info->ioc_serv[k].se_timeout_id) {
5042 timeout_id = ioc_info->ioc_serv[k].
5043 se_timeout_id;
5044 ioc_info->ioc_serv[k].se_timeout_id = 0;
5045 mutex_exit(&gid_info->gl_mutex);
5046 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5047 "ioc_info->ioc_serv[k].se_timeout_id = %x",
5048 k, timeout_id);
5049 if (untimeout(timeout_id) == -1) {
5050 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5051 "untimeout se_timeout_id %d "
5052 "failed", k);
5053 }
5054 mutex_enter(&gid_info->gl_mutex);
5055 }
5056 mutex_exit(&gid_info->gl_mutex);
5057 mutex_exit(&ibdm.ibdm_mutex);
5058 ibdm_ibnex_free_ioc_list(ioc_info);
5059
5060 mutex_enter(&ibdm.ibdm_mutex);
5061 while (ibdm.ibdm_busy & IBDM_BUSY)
5062 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5063 ibdm.ibdm_busy |= IBDM_BUSY;
5064 mutex_exit(&ibdm.ibdm_mutex);
5065
5066 ibdm_probe_ioc(iou, ioc_guid, 1);
5067
5068 /*
5069 * Skip if gl_reprobe_flag is set, this will be
5070 * a re-inserted / new GID, for which notifications
5071 * have already been send.
5072 */
5073 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
5074 gid_info = gid_info->gl_next) {
5075 uint8_t ii, niocs;
5076 ibdm_ioc_info_t *ioc;
5077
5078 if (gid_info->gl_iou == NULL)
5079 continue;
5080
5081 if (gid_info->gl_reprobe_flag) {
5082 gid_info->gl_reprobe_flag = 0;
5083 continue;
5084 }
5085
5086 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
5087 for (ii = 0; ii < niocs; ii++) {
5088 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
5089 if (ioc->ioc_profile.ioc_guid == ioc_guid) {
5090 mutex_enter(&ibdm.ibdm_mutex);
5091 ibdm_reprobe_update_port_srv(ioc,
5092 gid_info);
5093 mutex_exit(&ibdm.ibdm_mutex);
5094 }
5095 }
5096 }
5097 mutex_enter(&ibdm.ibdm_mutex);
5098 ibdm.ibdm_busy &= ~IBDM_BUSY;
5099 cv_broadcast(&ibdm.ibdm_busy_cv);
5100 mutex_exit(&ibdm.ibdm_mutex);
5101
5102 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5103 }
5104 return (ioc_info);
5105 }
5106
5107
5108 /*
5109 * ibdm_get_ioc_info_with_gid()
5110 * Returns pointer to ibdm_ioc_info_t if it finds
5111 * matching record for the ioc_guid. Otherwise NULL is returned.
5112 * The pointer to gid_info is set to the second argument in case that
5113 * the non-NULL value returns (and the second argument is not NULL).
5114 *
5115 * Note. use the same strings as "ibnex_get_ioc_info" in
5116 * IBTF_DPRINTF() to keep compatibility.
5117 */
5118 static ibdm_ioc_info_t *
5119 ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid,
5120 ibdm_dp_gidinfo_t **gid_info)
5121 {
5122 int ii;
5123 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL;
5124 ibdm_dp_gidinfo_t *gid_list;
5125 ib_dm_io_unitinfo_t *iou;
5126
5127 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);
5128
5129 mutex_enter(&ibdm.ibdm_mutex);
5130 while (ibdm.ibdm_busy & IBDM_BUSY)
5131 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5132 ibdm.ibdm_busy |= IBDM_BUSY;
5133
5134 if (gid_info)
5135 *gid_info = NULL; /* clear the value of gid_info */
5136
5137 gid_list = ibdm.ibdm_dp_gidlist_head;
5138 while (gid_list) {
5139 mutex_enter(&gid_list->gl_mutex);
5140 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5141 mutex_exit(&gid_list->gl_mutex);
5142 gid_list = gid_list->gl_next;
5143 continue;
5144 }
5145 if (gid_list->gl_iou == NULL) {
5146 IBTF_DPRINTF_L2("ibdm",
5147 "\tget_ioc_info: No IOU info");
5148 mutex_exit(&gid_list->gl_mutex);
5149 gid_list = gid_list->gl_next;
5150 continue;
5151 }
5152 iou = &gid_list->gl_iou->iou_info;
5153 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5154 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5155 if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
5156 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
5157 ioc = ibdm_dup_ioc_info(tmp, gid_list);
5158 if (gid_info)
5159 *gid_info = gid_list; /* set this ptr */
5160 mutex_exit(&gid_list->gl_mutex);
5161 ibdm.ibdm_busy &= ~IBDM_BUSY;
5162 cv_broadcast(&ibdm.ibdm_busy_cv);
5163 mutex_exit(&ibdm.ibdm_mutex);
5164 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
5165 return (ioc);
5166 }
5167 }
5168 if (ii == iou->iou_num_ctrl_slots)
5169 ioc = NULL;
5170
5171 mutex_exit(&gid_list->gl_mutex);
5172 gid_list = gid_list->gl_next;
5173 }
5174
5175 ibdm.ibdm_busy &= ~IBDM_BUSY;
5176 cv_broadcast(&ibdm.ibdm_busy_cv);
5177 mutex_exit(&ibdm.ibdm_mutex);
5178 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
5179 return (ioc);
5180 }
5181
5182 /*
5183 * ibdm_ibnex_get_ioc_info()
5184 * Returns pointer to ibdm_ioc_info_t if it finds
5185 * matching record for the ioc_guid, otherwise NULL
5186 * is returned
5187 *
5188 * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now.
5189 */
5190 ibdm_ioc_info_t *
5191 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
5192 {
5193 if (ibdm_enumerate_iocs == 0)
5194 return (NULL);
5195
5196 /* will not use the gid_info pointer, so the second arg is NULL */
5197 return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL));
5198 }
5199
5200 /*
5201 * ibdm_ibnex_get_ioc_count()
5202 * Returns number of ibdm_ioc_info_t it finds
5203 */
5204 int
5205 ibdm_ibnex_get_ioc_count(void)
5206 {
5207 int count = 0, k;
5208 ibdm_ioc_info_t *ioc;
5209 ibdm_dp_gidinfo_t *gid_list;
5210
5211 if (ibdm_enumerate_iocs == 0)
5212 return (0);
5213
5214 mutex_enter(&ibdm.ibdm_mutex);
5215 ibdm_sweep_fabric(0);
5216
5217 while (ibdm.ibdm_busy & IBDM_BUSY)
5218 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5219 ibdm.ibdm_busy |= IBDM_BUSY;
5220
5221 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5222 gid_list = gid_list->gl_next) {
5223 mutex_enter(&gid_list->gl_mutex);
5224 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
5225 (gid_list->gl_iou == NULL)) {
5226 mutex_exit(&gid_list->gl_mutex);
5227 continue;
5228 }
5229 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
5230 k++) {
5231 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
5232 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
5233 ++count;
5234 }
5235 mutex_exit(&gid_list->gl_mutex);
5236 }
5237 ibdm.ibdm_busy &= ~IBDM_BUSY;
5238 cv_broadcast(&ibdm.ibdm_busy_cv);
5239 mutex_exit(&ibdm.ibdm_mutex);
5240
5241 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
5242 return (count);
5243 }
5244
5245
5246 /*
5247 * ibdm_ibnex_get_ioc_list()
5248 * Returns information about all the IOCs present on the fabric.
5249 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
5250 * Does not sweep fabric if DONOT_PROBE is set
5251 */
5252 ibdm_ioc_info_t *
5253 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
5254 {
5255 int ii;
5256 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc;
5257 ibdm_dp_gidinfo_t *gid_list;
5258 ib_dm_io_unitinfo_t *iou;
5259
5260 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");
5261
5262 if (ibdm_enumerate_iocs == 0)
5263 return (NULL);
5264
5265 mutex_enter(&ibdm.ibdm_mutex);
5266 if (list_flag != IBDM_IBNEX_DONOT_PROBE)
5267 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);
5268
5269 while (ibdm.ibdm_busy & IBDM_BUSY)
5270 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5271 ibdm.ibdm_busy |= IBDM_BUSY;
5272
5273 gid_list = ibdm.ibdm_dp_gidlist_head;
5274 while (gid_list) {
5275 mutex_enter(&gid_list->gl_mutex);
5276 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5277 mutex_exit(&gid_list->gl_mutex);
5278 gid_list = gid_list->gl_next;
5279 continue;
5280 }
5281 if (gid_list->gl_iou == NULL) {
5282 IBTF_DPRINTF_L2("ibdm",
5283 "\tget_ioc_list: No IOU info");
5284 mutex_exit(&gid_list->gl_mutex);
5285 gid_list = gid_list->gl_next;
5286 continue;
5287 }
5288 iou = &gid_list->gl_iou->iou_info;
5289 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5290 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5291 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5292 tmp = ibdm_dup_ioc_info(ioc, gid_list);
5293 tmp->ioc_next = ioc_list;
5294 ioc_list = tmp;
5295 }
5296 }
5297 mutex_exit(&gid_list->gl_mutex);
5298 gid_list = gid_list->gl_next;
5299 }
5300 ibdm.ibdm_busy &= ~IBDM_BUSY;
5301 cv_broadcast(&ibdm.ibdm_busy_cv);
5302 mutex_exit(&ibdm.ibdm_mutex);
5303
5304 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
5305 return (ioc_list);
5306 }
5307
5308 /*
5309 * ibdm_dup_ioc_info()
5310 * Duplicate the IOC information and return the IOC
5311 * information.
5312 */
5313 static ibdm_ioc_info_t *
5314 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
5315 {
5316 ibdm_ioc_info_t *out_ioc;
5317 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
5318 ASSERT(MUTEX_HELD(&gid_list->gl_mutex));
5319
5320 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
5321 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
5322 ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
5323 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
5324 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;
5325
5326 return (out_ioc);
5327 }
5328
5329
5330 /*
5331 * ibdm_free_ioc_list()
5332 * Deallocate memory for IOC list structure
5333 */
5334 void
5335 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
5336 {
5337 ibdm_ioc_info_t *temp;
5338
5339 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
5340 while (ioc) {
5341 temp = ioc;
5342 ioc = ioc->ioc_next;
5343 kmem_free(temp->ioc_gid_list,
5344 (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
5345 if (temp->ioc_hca_list)
5346 ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
5347 kmem_free(temp, sizeof (ibdm_ioc_info_t));
5348 }
5349 }
5350
5351
5352 /*
5353 * ibdm_ibnex_update_pkey_tbls
5354 * Updates the DM P_Key database.
5355 * NOTE: Two cases are handled here: P_Key being added or removed.
5356 *
5357 * Arguments : NONE
5358 * Return Values : NONE
5359 */
5360 void
5361 ibdm_ibnex_update_pkey_tbls(void)
5362 {
5363 int h, pp, pidx;
5364 uint_t nports;
5365 uint_t size;
5366 ib_pkey_t new_pkey;
5367 ib_pkey_t *orig_pkey;
5368 ibdm_hca_list_t *hca_list;
5369 ibdm_port_attr_t *port;
5370 ibt_hca_portinfo_t *pinfop;
5371
5372 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");
5373
5374 mutex_enter(&ibdm.ibdm_hl_mutex);
5375 hca_list = ibdm.ibdm_hca_list_head;
5376
5377 for (h = 0; h < ibdm.ibdm_hca_count; h++) {
5378
5379 /* This updates P_Key Tables for all ports of this HCA */
5380 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
5381 &nports, &size);
5382
5383 /* number of ports shouldn't have changed */
5384 ASSERT(nports == hca_list->hl_nports);
5385
5386 for (pp = 0; pp < hca_list->hl_nports; pp++) {
5387 port = &hca_list->hl_port_attr[pp];
5388
5389 /*
5390 * First figure out the P_Keys from IBTL.
5391 * Three things could have happened:
5392 * New P_Keys added
5393 * Existing P_Keys removed
5394 * Both of the above two
5395 *
5396 * Loop through the P_Key Indices and check if a
5397 * give P_Key_Ix matches that of the one seen by
5398 * IBDM. If they match no action is needed.
5399 *
5400 * If they don't match:
5401 * 1. if orig_pkey is invalid and new_pkey is valid
5402 * ---> add new_pkey to DM database
5403 * 2. if orig_pkey is valid and new_pkey is invalid
5404 * ---> remove orig_pkey from DM database
5405 * 3. if orig_pkey and new_pkey are both valid:
5406 * ---> remov orig_pkey from DM database
5407 * ---> add new_pkey to DM database
5408 * 4. if orig_pkey and new_pkey are both invalid:
5409 * ---> do nothing. Updated DM database.
5410 */
5411
5412 for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
5413 new_pkey = pinfop[pp].p_pkey_tbl[pidx];
5414 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;
5415
5416 /* keys match - do nothing */
5417 if (*orig_pkey == new_pkey)
5418 continue;
5419
5420 if (IBDM_INVALID_PKEY(*orig_pkey) &&
5421 !IBDM_INVALID_PKEY(new_pkey)) {
5422 /* P_Key was added */
5423 IBTF_DPRINTF_L5("ibdm",
5424 "\tibnex_update_pkey_tbls: new "
5425 "P_Key added = 0x%x", new_pkey);
5426 *orig_pkey = new_pkey;
5427 ibdm_port_attr_ibmf_init(port,
5428 new_pkey, pp);
5429 } else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5430 IBDM_INVALID_PKEY(new_pkey)) {
5431 /* P_Key was removed */
5432 IBTF_DPRINTF_L5("ibdm",
5433 "\tibnex_update_pkey_tbls: P_Key "
5434 "removed = 0x%x", *orig_pkey);
5435 *orig_pkey = new_pkey;
5436 (void) ibdm_port_attr_ibmf_fini(port,
5437 pidx);
5438 } else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5439 !IBDM_INVALID_PKEY(new_pkey)) {
5440 /* P_Key were replaced */
5441 IBTF_DPRINTF_L5("ibdm",
5442 "\tibnex_update_pkey_tbls: P_Key "
5443 "replaced 0x%x with 0x%x",
5444 *orig_pkey, new_pkey);
5445 (void) ibdm_port_attr_ibmf_fini(port,
5446 pidx);
5447 *orig_pkey = new_pkey;
5448 ibdm_port_attr_ibmf_init(port,
5449 new_pkey, pp);
5450 } else {
5451 /*
5452 * P_Keys are invalid
5453 * set anyway to reflect if
5454 * INVALID_FULL was changed to
5455 * INVALID_LIMITED or vice-versa.
5456 */
5457 *orig_pkey = new_pkey;
5458 } /* end of else */
5459
5460 } /* loop of p_key index */
5461
5462 } /* loop of #ports of HCA */
5463
5464 ibt_free_portinfo(pinfop, size);
5465 hca_list = hca_list->hl_next;
5466
5467 } /* loop for all HCAs in the system */
5468
5469 mutex_exit(&ibdm.ibdm_hl_mutex);
5470 }
5471
5472
5473 /*
5474 * ibdm_send_ioc_profile()
5475 * Send IOC Controller Profile request. When the request is completed
5476 * IBMF calls ibdm_process_incoming_mad routine to inform about
5477 * the completion.
5478 */
5479 static int
5480 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
5481 {
5482 ibmf_msg_t *msg;
5483 ib_mad_hdr_t *hdr;
5484 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
5485 ibdm_timeout_cb_args_t *cb_args;
5486
5487 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
5488 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);
5489
5490 /*
5491 * Send command to get IOC profile.
5492 * Allocate a IBMF packet and initialize the packet.
5493 */
5494 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
5495 &msg) != IBMF_SUCCESS) {
5496 IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail");
5497 return (IBDM_FAILURE);
5498 }
5499
5500 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
5501 ibdm_alloc_send_buffers(msg);
5502 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
5503
5504 mutex_enter(&gid_info->gl_mutex);
5505 ibdm_bump_transactionID(gid_info);
5506 mutex_exit(&gid_info->gl_mutex);
5507
5508 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
5509 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
5510 if (gid_info->gl_redirected == B_TRUE) {
5511 if (gid_info->gl_redirect_dlid != 0) {
5512 msg->im_local_addr.ia_remote_lid =
5513 gid_info->gl_redirect_dlid;
5514 }
5515 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
5516 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
5517 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
5518 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
5519 } else {
5520 msg->im_local_addr.ia_remote_qno = 1;
5521 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
5522 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
5523 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
5524 }
5525
5526 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
5527 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
5528 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
5529 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
5530 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
5531 hdr->Status = 0;
5532 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
5533 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
5534 hdr->AttributeModifier = h2b32(ioc_no + 1);
5535
5536 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5537 cb_args = &ioc_info->ioc_cb_args;
5538 cb_args->cb_gid_info = gid_info;
5539 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
5540 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO;
5541 cb_args->cb_ioc_num = ioc_no;
5542
5543 mutex_enter(&gid_info->gl_mutex);
5544 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
5545 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
5546 mutex_exit(&gid_info->gl_mutex);
5547
5548 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
5549 "timeout %x", ioc_info->ioc_timeout_id);
5550
5551 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
5552 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
5553 IBTF_DPRINTF_L2("ibdm",
5554 "\tsend_ioc_profile: msg transport failed");
5555 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
5556 }
5557 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5558 return (IBDM_SUCCESS);
5559 }
5560
5561
5562 /*
5563 * ibdm_port_reachable
5564 * Returns B_TRUE if the port GID is reachable by sending
5565 * a SA query to get the NODE record for this port GUID.
5566 */
5567 static boolean_t
5568 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid)
5569 {
5570 sa_node_record_t *resp;
5571 size_t length;
5572
5573 /*
5574 * Verify if it's reachable by getting the node record.
5575 */
5576 if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) ==
5577 IBDM_SUCCESS) {
5578 kmem_free(resp, length);
5579 return (B_TRUE);
5580 }
5581 return (B_FALSE);
5582 }
5583
5584 /*
5585 * ibdm_get_node_record_by_port
5586 * Sends a SA query to get the NODE record for port GUID
5587 * Returns IBDM_SUCCESS if the port GID is reachable.
5588 *
5589 * Note: the caller must be responsible for freeing the resource
5590 * by calling kmem_free(resp, length) later.
5591 */
5592 static int
5593 ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
5594 sa_node_record_t **resp, size_t *length)
5595 {
5596 sa_node_record_t req;
5597 ibmf_saa_access_args_t args;
5598 int ret;
5599 ASSERT(resp != NULL && length != NULL);
5600
5601 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
5602 guid);
5603
5604 bzero(&req, sizeof (sa_node_record_t));
5605 req.NodeInfo.PortGUID = guid;
5606
5607 args.sq_attr_id = SA_NODERECORD_ATTRID;
5608 args.sq_access_type = IBMF_SAA_RETRIEVE;
5609 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID;
5610 args.sq_template = &req;
5611 args.sq_callback = NULL;
5612 args.sq_callback_arg = NULL;
5613
5614 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp);
5615 if (ret != IBMF_SUCCESS) {
5616 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
5617 " SA Retrieve Failed: %d", ret);
5618 return (IBDM_FAILURE);
5619 }
5620 if (*resp == NULL || *length == 0) {
5621 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
5622 return (IBDM_FAILURE);
5623 }
5624 /*
5625 * There is one NodeRecord on each endport on a subnet.
5626 */
5627 ASSERT(*length == sizeof (sa_node_record_t));
5628
5629 return (IBDM_SUCCESS);
5630 }
5631
5632
5633 /*
5634 * Update the gidlist for all affected IOCs when GID becomes
5635 * available/unavailable.
5636 *
5637 * Parameters :
5638 * gidinfo - Incoming / Outgoing GID.
5639 * add_flag - 1 for GID added, 0 for GID removed.
5640 * - (-1) : IOC gid list updated, ioc_list required.
5641 *
5642 * This function gets the GID for the node GUID corresponding to the
5643 * port GID. Gets the IOU info
5644 */
5645 static ibdm_ioc_info_t *
5646 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
5647 {
5648 ibdm_dp_gidinfo_t *node_gid = NULL;
5649 uint8_t niocs, ii;
5650 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp;
5651
5652 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");
5653
5654 switch (avail_flag) {
5655 case 1 :
5656 node_gid = ibdm_check_dest_nodeguid(gid_info);
5657 break;
5658 case 0 :
5659 node_gid = ibdm_handle_gid_rm(gid_info);
5660 break;
5661 case -1 :
5662 node_gid = gid_info;
5663 break;
5664 default :
5665 break;
5666 }
5667
5668 if (node_gid == NULL) {
5669 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
5670 "No node GID found, port gid 0x%p, avail_flag %d",
5671 gid_info, avail_flag);
5672 return (NULL);
5673 }
5674
5675 mutex_enter(&node_gid->gl_mutex);
5676 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
5677 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
5678 node_gid->gl_iou == NULL) {
5679 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
5680 "gl_state %x, gl_iou %p", node_gid->gl_state,
5681 node_gid->gl_iou);
5682 mutex_exit(&node_gid->gl_mutex);
5683 return (NULL);
5684 }
5685
5686 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
5687 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
5688 niocs);
5689 for (ii = 0; ii < niocs; ii++) {
5690 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
5691 /*
5692 * Skip IOCs for which probe is not complete or
5693 * reprobe is progress
5694 */
5695 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5696 tmp = ibdm_dup_ioc_info(ioc, node_gid);
5697 tmp->ioc_info_updated.ib_gid_prop_updated = 1;
5698 tmp->ioc_next = ioc_list;
5699 ioc_list = tmp;
5700 }
5701 }
5702 mutex_exit(&node_gid->gl_mutex);
5703
5704 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
5705 ioc_list);
5706 return (ioc_list);
5707 }
5708
5709 /*
5710 * ibdm_saa_event_cb :
5711 * Event handling which does *not* require ibdm_hl_mutex to be
5712 * held are executed in the same thread. This is to prevent
5713 * deadlocks with HCA port down notifications which hold the
5714 * ibdm_hl_mutex.
5715 *
5716 * GID_AVAILABLE event is handled here. A taskq is spawned to
5717 * handle GID_UNAVAILABLE.
5718 *
5719 * A new mutex ibdm_ibnex_mutex has been introduced to protect
5720 * ibnex_callback. This has been done to prevent any possible
5721 * deadlock (described above) while handling GID_AVAILABLE.
5722 *
5723 * IBMF calls the event callback for a HCA port. The SA handle
5724 * for this port would be valid, till the callback returns.
5725 * IBDM calling IBDM using the above SA handle should be valid.
5726 *
5727 * IBDM will additionally check (SA handle != NULL), before
5728 * calling IBMF.
5729 */
5730 /*ARGSUSED*/
5731 static void
5732 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
5733 ibmf_saa_subnet_event_t ibmf_saa_event,
5734 ibmf_saa_event_details_t *event_details, void *callback_arg)
5735 {
5736 ibdm_saa_event_arg_t *event_arg;
5737 ib_gid_t sgid, dgid;
5738 ibdm_port_attr_t *hca_port;
5739 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL;
5740 sa_node_record_t *nrec;
5741 size_t length;
5742
5743 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5744
5745 hca_port = (ibdm_port_attr_t *)callback_arg;
5746
5747 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
5748 ibmf_saa_handle, ibmf_saa_event, event_details,
5749 callback_arg);
5750
5751 #ifdef DEBUG
5752 if (ibdm_ignore_saa_event)
5753 return;
5754 #endif
5755
5756 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
5757 /*
5758 * Ensure no other probe / sweep fabric is in
5759 * progress.
5760 */
5761 mutex_enter(&ibdm.ibdm_mutex);
5762 while (ibdm.ibdm_busy & IBDM_BUSY)
5763 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5764 ibdm.ibdm_busy |= IBDM_BUSY;
5765 mutex_exit(&ibdm.ibdm_mutex);
5766
5767 /*
5768 * If we already know about this GID, return.
5769 * GID_AVAILABLE may be reported for multiple HCA
5770 * ports.
5771 */
5772 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
5773 event_details->ie_gid.gid_prefix)) != NULL) {
5774 mutex_enter(&ibdm.ibdm_mutex);
5775 ibdm.ibdm_busy &= ~IBDM_BUSY;
5776 cv_broadcast(&ibdm.ibdm_busy_cv);
5777 mutex_exit(&ibdm.ibdm_mutex);
5778 return;
5779 }
5780
5781 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5782 "Insertion notified",
5783 event_details->ie_gid.gid_prefix,
5784 event_details->ie_gid.gid_guid);
5785
5786 /* This is a new gid, insert it to GID list */
5787 sgid.gid_prefix = hca_port->pa_sn_prefix;
5788 sgid.gid_guid = hca_port->pa_port_guid;
5789 dgid.gid_prefix = event_details->ie_gid.gid_prefix;
5790 dgid.gid_guid = event_details->ie_gid.gid_guid;
5791 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
5792 if (gid_info == NULL) {
5793 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
5794 "create_gid_info returned NULL");
5795 mutex_enter(&ibdm.ibdm_mutex);
5796 ibdm.ibdm_busy &= ~IBDM_BUSY;
5797 cv_broadcast(&ibdm.ibdm_busy_cv);
5798 mutex_exit(&ibdm.ibdm_mutex);
5799 return;
5800 }
5801 mutex_enter(&gid_info->gl_mutex);
5802 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
5803 mutex_exit(&gid_info->gl_mutex);
5804
5805 /* Get the node GUID */
5806 if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid,
5807 &nrec, &length) != IBDM_SUCCESS) {
5808 /*
5809 * Set the state to PROBE_NOT_DONE for the
5810 * next sweep to probe it
5811 */
5812 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
5813 "Skipping GID : port GUID not found");
5814 mutex_enter(&gid_info->gl_mutex);
5815 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5816 mutex_exit(&gid_info->gl_mutex);
5817 mutex_enter(&ibdm.ibdm_mutex);
5818 ibdm.ibdm_busy &= ~IBDM_BUSY;
5819 cv_broadcast(&ibdm.ibdm_busy_cv);
5820 mutex_exit(&ibdm.ibdm_mutex);
5821 return;
5822 }
5823 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
5824 gid_info->gl_devid = nrec->NodeInfo.DeviceID;
5825 kmem_free(nrec, length);
5826 gid_info->gl_portguid = dgid.gid_guid;
5827
5828 /*
5829 * Get the gid info with the same node GUID.
5830 */
5831 mutex_enter(&ibdm.ibdm_mutex);
5832 node_gid_info = ibdm.ibdm_dp_gidlist_head;
5833 while (node_gid_info) {
5834 if (node_gid_info->gl_nodeguid ==
5835 gid_info->gl_nodeguid &&
5836 node_gid_info->gl_iou != NULL) {
5837 break;
5838 }
5839 node_gid_info = node_gid_info->gl_next;
5840 }
5841 mutex_exit(&ibdm.ibdm_mutex);
5842
5843 /*
5844 * Handling a new GID requires filling of gl_hca_list.
5845 * This require ibdm hca_list to be parsed and hence
5846 * holding the ibdm_hl_mutex. Spawning a new thread to
5847 * handle this.
5848 */
5849 if (node_gid_info == NULL) {
5850 if (taskq_dispatch(system_taskq,
5851 ibdm_saa_handle_new_gid, (void *)gid_info,
5852 TQ_NOSLEEP) == NULL) {
5853 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5854 "new_gid taskq_dispatch failed");
5855 return;
5856 }
5857 }
5858
5859 mutex_enter(&ibdm.ibdm_mutex);
5860 ibdm.ibdm_busy &= ~IBDM_BUSY;
5861 cv_broadcast(&ibdm.ibdm_busy_cv);
5862 mutex_exit(&ibdm.ibdm_mutex);
5863 return;
5864 }
5865
5866 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
5867 return;
5868
5869 /*
5870 * GID UNAVAIL EVENT: Try to locate the GID in the GID list.
5871 * If we don't find it we just return.
5872 */
5873 mutex_enter(&ibdm.ibdm_mutex);
5874 gid_info = ibdm.ibdm_dp_gidlist_head;
5875 while (gid_info) {
5876 if (gid_info->gl_portguid ==
5877 event_details->ie_gid.gid_guid) {
5878 break;
5879 }
5880 gid_info = gid_info->gl_next;
5881 }
5882 mutex_exit(&ibdm.ibdm_mutex);
5883 if (gid_info == NULL) {
5884 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5885 "GID for GUID %llX not found during GID UNAVAIL event",
5886 event_details->ie_gid.gid_guid);
5887 return;
5888 }
5889
5890 /*
5891 * If this GID is DM capable, we'll have to check whether this DGID
5892 * is reachable via another port.
5893 */
5894 if (gid_info->gl_is_dm_capable == B_TRUE) {
5895 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
5896 sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
5897 event_arg->ibmf_saa_handle = ibmf_saa_handle;
5898 event_arg->ibmf_saa_event = ibmf_saa_event;
5899 bcopy(event_details, &event_arg->event_details,
5900 sizeof (ibmf_saa_event_details_t));
5901 event_arg->callback_arg = callback_arg;
5902
5903 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
5904 (void *)event_arg, TQ_NOSLEEP) == NULL) {
5905 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5906 "taskq_dispatch failed");
5907 ibdm_free_saa_event_arg(event_arg);
5908 return;
5909 }
5910 }
5911 }
5912
5913 /*
5914 * Handle a new GID discovered by GID_AVAILABLE saa event.
5915 */
5916 void
5917 ibdm_saa_handle_new_gid(void *arg)
5918 {
5919 ibdm_dp_gidinfo_t *gid_info;
5920 ibdm_hca_list_t *hca_list = NULL;
5921 ibdm_port_attr_t *port = NULL;
5922 ibdm_ioc_info_t *ioc_list = NULL;
5923
5924 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);
5925
5926 gid_info = (ibdm_dp_gidinfo_t *)arg;
5927
5928 /*
5929 * Ensure that no other sweep / probe has completed
5930 * probing this gid.
5931 */
5932 mutex_enter(&gid_info->gl_mutex);
5933 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
5934 mutex_exit(&gid_info->gl_mutex);
5935 return;
5936 }
5937 mutex_exit(&gid_info->gl_mutex);
5938
5939 /*
5940 * Parse HCAs to fill gl_hca_list
5941 */
5942 mutex_enter(&ibdm.ibdm_hl_mutex);
5943 for (ibdm_get_next_port(&hca_list, &port, 1); port;
5944 ibdm_get_next_port(&hca_list, &port, 1)) {
5945 if (ibdm_port_reachable(port->pa_sa_hdl,
5946 gid_info->gl_portguid) == B_TRUE) {
5947 ibdm_addto_glhcalist(gid_info, hca_list);
5948 }
5949 }
5950 mutex_exit(&ibdm.ibdm_hl_mutex);
5951
5952 /*
5953 * Ensure no other probe / sweep fabric is in
5954 * progress.
5955 */
5956 mutex_enter(&ibdm.ibdm_mutex);
5957 while (ibdm.ibdm_busy & IBDM_BUSY)
5958 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5959 ibdm.ibdm_busy |= IBDM_BUSY;
5960 mutex_exit(&ibdm.ibdm_mutex);
5961
5962 /*
5963 * New IOU probe it, to check if new IOCs
5964 */
5965 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
5966 "new GID : probing");
5967 mutex_enter(&ibdm.ibdm_mutex);
5968 ibdm.ibdm_ngid_probes_in_progress++;
5969 mutex_exit(&ibdm.ibdm_mutex);
5970 mutex_enter(&gid_info->gl_mutex);
5971 gid_info->gl_reprobe_flag = 0;
5972 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5973 mutex_exit(&gid_info->gl_mutex);
5974 ibdm_probe_gid_thread((void *)gid_info);
5975
5976 mutex_enter(&ibdm.ibdm_mutex);
5977 ibdm_wait_probe_completion();
5978 mutex_exit(&ibdm.ibdm_mutex);
5979
5980 if (gid_info->gl_iou == NULL) {
5981 mutex_enter(&ibdm.ibdm_mutex);
5982 ibdm.ibdm_busy &= ~IBDM_BUSY;
5983 cv_broadcast(&ibdm.ibdm_busy_cv);
5984 mutex_exit(&ibdm.ibdm_mutex);
5985 return;
5986 }
5987
5988 /*
5989 * Update GID list in all IOCs affected by this
5990 */
5991 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);
5992
5993 /*
5994 * Pass on the IOCs with updated GIDs to IBnexus
5995 */
5996 if (ioc_list) {
5997 mutex_enter(&ibdm.ibdm_ibnex_mutex);
5998 if (ibdm.ibdm_ibnex_callback != NULL) {
5999 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6000 IBDM_EVENT_IOC_PROP_UPDATE);
6001 }
6002 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6003 }
6004
6005 mutex_enter(&ibdm.ibdm_mutex);
6006 ibdm.ibdm_busy &= ~IBDM_BUSY;
6007 cv_broadcast(&ibdm.ibdm_busy_cv);
6008 mutex_exit(&ibdm.ibdm_mutex);
6009 }
6010
6011 /*
6012 * ibdm_saa_event_taskq :
6013 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
6014 * held. The GID_UNAVAILABLE handling is done in a taskq to
6015 * prevent deadlocks with HCA port down notifications which hold
6016 * ibdm_hl_mutex.
6017 */
6018 void
6019 ibdm_saa_event_taskq(void *arg)
6020 {
6021 ibdm_saa_event_arg_t *event_arg;
6022 ibmf_saa_handle_t ibmf_saa_handle;
6023 ibmf_saa_subnet_event_t ibmf_saa_event;
6024 ibmf_saa_event_details_t *event_details;
6025 void *callback_arg;
6026
6027 ibdm_dp_gidinfo_t *gid_info;
6028 ibdm_port_attr_t *hca_port, *port = NULL;
6029 ibdm_hca_list_t *hca_list = NULL;
6030 int sa_handle_valid = 0;
6031 ibdm_ioc_info_t *ioc_list = NULL;
6032
6033 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
6034
6035 event_arg = (ibdm_saa_event_arg_t *)arg;
6036 ibmf_saa_handle = event_arg->ibmf_saa_handle;
6037 ibmf_saa_event = event_arg->ibmf_saa_event;
6038 event_details = &event_arg->event_details;
6039 callback_arg = event_arg->callback_arg;
6040
6041 ASSERT(callback_arg != NULL);
6042 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
6043 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
6044 ibmf_saa_handle, ibmf_saa_event, event_details,
6045 callback_arg);
6046
6047 hca_port = (ibdm_port_attr_t *)callback_arg;
6048
6049 /* Check if the port_attr is still valid */
6050 mutex_enter(&ibdm.ibdm_hl_mutex);
6051 for (ibdm_get_next_port(&hca_list, &port, 0); port;
6052 ibdm_get_next_port(&hca_list, &port, 0)) {
6053 if (port == hca_port && port->pa_port_guid ==
6054 hca_port->pa_port_guid) {
6055 if (ibmf_saa_handle == hca_port->pa_sa_hdl)
6056 sa_handle_valid = 1;
6057 break;
6058 }
6059 }
6060 mutex_exit(&ibdm.ibdm_hl_mutex);
6061 if (sa_handle_valid == 0) {
6062 ibdm_free_saa_event_arg(event_arg);
6063 return;
6064 }
6065
6066 if (hca_port && (hca_port->pa_sa_hdl == NULL ||
6067 ibmf_saa_handle != hca_port->pa_sa_hdl)) {
6068 ibdm_free_saa_event_arg(event_arg);
6069 return;
6070 }
6071 hca_list = NULL;
6072 port = NULL;
6073
6074 /*
6075 * Check if the GID is visible to other HCA ports.
6076 * Return if so.
6077 */
6078 mutex_enter(&ibdm.ibdm_hl_mutex);
6079 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6080 ibdm_get_next_port(&hca_list, &port, 1)) {
6081 if (ibdm_port_reachable(port->pa_sa_hdl,
6082 event_details->ie_gid.gid_guid) == B_TRUE) {
6083 mutex_exit(&ibdm.ibdm_hl_mutex);
6084 ibdm_free_saa_event_arg(event_arg);
6085 return;
6086 }
6087 }
6088 mutex_exit(&ibdm.ibdm_hl_mutex);
6089
6090 /*
6091 * Ensure no other probe / sweep fabric is in
6092 * progress.
6093 */
6094 mutex_enter(&ibdm.ibdm_mutex);
6095 while (ibdm.ibdm_busy & IBDM_BUSY)
6096 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
6097 ibdm.ibdm_busy |= IBDM_BUSY;
6098 mutex_exit(&ibdm.ibdm_mutex);
6099
6100 /*
6101 * If this GID is no longer in GID list, return
6102 * GID_UNAVAILABLE may be reported for multiple HCA
6103 * ports.
6104 */
6105 mutex_enter(&ibdm.ibdm_mutex);
6106 gid_info = ibdm.ibdm_dp_gidlist_head;
6107 while (gid_info) {
6108 if (gid_info->gl_portguid ==
6109 event_details->ie_gid.gid_guid) {
6110 break;
6111 }
6112 gid_info = gid_info->gl_next;
6113 }
6114 mutex_exit(&ibdm.ibdm_mutex);
6115 if (gid_info == NULL) {
6116 mutex_enter(&ibdm.ibdm_mutex);
6117 ibdm.ibdm_busy &= ~IBDM_BUSY;
6118 cv_broadcast(&ibdm.ibdm_busy_cv);
6119 mutex_exit(&ibdm.ibdm_mutex);
6120 ibdm_free_saa_event_arg(event_arg);
6121 return;
6122 }
6123
6124 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
6125 "Unavailable notification",
6126 event_details->ie_gid.gid_prefix,
6127 event_details->ie_gid.gid_guid);
6128
6129 /*
6130 * Update GID list in all IOCs affected by this
6131 */
6132 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
6133 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
6134 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6135
6136 /*
6137 * Remove GID from the global GID list
6138 * Handle the case where all port GIDs for an
6139 * IOU have been hot-removed. Check both gid_info
6140 * & ioc_info for checking ngids.
6141 */
6142 mutex_enter(&ibdm.ibdm_mutex);
6143 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6144 mutex_enter(&gid_info->gl_mutex);
6145 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6146 mutex_exit(&gid_info->gl_mutex);
6147 }
6148 if (gid_info->gl_prev != NULL)
6149 gid_info->gl_prev->gl_next = gid_info->gl_next;
6150 if (gid_info->gl_next != NULL)
6151 gid_info->gl_next->gl_prev = gid_info->gl_prev;
6152
6153 if (gid_info == ibdm.ibdm_dp_gidlist_head)
6154 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6155 if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6156 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6157 ibdm.ibdm_ngids--;
6158
6159 ibdm.ibdm_busy &= ~IBDM_BUSY;
6160 cv_broadcast(&ibdm.ibdm_busy_cv);
6161 mutex_exit(&ibdm.ibdm_mutex);
6162
6163 /* free the hca_list on this gid_info */
6164 ibdm_delete_glhca_list(gid_info);
6165
6166 mutex_destroy(&gid_info->gl_mutex);
6167 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6168
6169 /*
6170 * Pass on the IOCs with updated GIDs to IBnexus
6171 */
6172 if (ioc_list) {
6173 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
6174 "IOC_PROP_UPDATE for %p\n", ioc_list);
6175 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6176 if (ibdm.ibdm_ibnex_callback != NULL) {
6177 (*ibdm.ibdm_ibnex_callback)((void *)
6178 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6179 }
6180 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6181 }
6182
6183 ibdm_free_saa_event_arg(event_arg);
6184 }
6185
6186
6187 static int
6188 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
6189 {
6190 ibdm_gid_t *scan_new, *scan_prev;
6191 int cmp_failed = 0;
6192
6193 ASSERT(new != NULL);
6194 ASSERT(prev != NULL);
6195
6196 /*
6197 * Search for each new gid anywhere in the prev GID list.
6198 * Note that the gid list could have been re-ordered.
6199 */
6200 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
6201 for (scan_prev = prev, cmp_failed = 1; scan_prev;
6202 scan_prev = scan_prev->gid_next) {
6203 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
6204 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
6205 cmp_failed = 0;
6206 break;
6207 }
6208 }
6209
6210 if (cmp_failed)
6211 return (1);
6212 }
6213 return (0);
6214 }
6215
6216 /*
6217 * This is always called in a single thread
6218 * This function updates the gid_list and serv_list of IOC
6219 * The current gid_list is in ioc_info_t(contains only port
6220 * guids for which probe is done) & gidinfo_t(other port gids)
6221 * The gids in both locations are used for comparision.
6222 */
6223 static void
6224 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
6225 {
6226 ibdm_gid_t *cur_gid_list;
6227 uint_t cur_nportgids;
6228
6229 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
6230
6231 ioc->ioc_info_updated.ib_prop_updated = 0;
6232
6233
6234 /* Current GID list in gid_info only */
6235 cur_gid_list = gidinfo->gl_gid;
6236 cur_nportgids = gidinfo->gl_ngids;
6237
6238 if (ioc->ioc_prev_serv_cnt !=
6239 ioc->ioc_profile.ioc_service_entries ||
6240 ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0],
6241 ioc->ioc_prev_serv_cnt))
6242 ioc->ioc_info_updated.ib_srv_prop_updated = 1;
6243
6244 if (ioc->ioc_prev_nportgids != cur_nportgids ||
6245 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
6246 ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6247 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
6248 ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6249 }
6250
6251 /* Zero out previous entries */
6252 ibdm_free_gid_list(ioc->ioc_prev_gid_list);
6253 if (ioc->ioc_prev_serv)
6254 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
6255 sizeof (ibdm_srvents_info_t));
6256 ioc->ioc_prev_serv_cnt = 0;
6257 ioc->ioc_prev_nportgids = 0;
6258 ioc->ioc_prev_serv = NULL;
6259 ioc->ioc_prev_gid_list = NULL;
6260 }
6261
6262 /*
6263 * Handle GID removal. This returns gid_info of an GID for the same
6264 * node GUID, if found. For an GID with IOU information, the same
6265 * gid_info is returned if no gid_info with same node_guid is found.
6266 */
6267 static ibdm_dp_gidinfo_t *
6268 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
6269 {
6270 ibdm_dp_gidinfo_t *gid_list;
6271
6272 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);
6273
6274 if (rm_gid->gl_iou == NULL) {
6275 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
6276 /*
6277 * Search for a GID with same node_guid and
6278 * gl_iou != NULL
6279 */
6280 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6281 gid_list = gid_list->gl_next) {
6282 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
6283 == rm_gid->gl_nodeguid))
6284 break;
6285 }
6286
6287 if (gid_list)
6288 ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6289
6290 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6291 return (gid_list);
6292 } else {
6293 /*
6294 * Search for a GID with same node_guid and
6295 * gl_iou == NULL
6296 */
6297 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
6298 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6299 gid_list = gid_list->gl_next) {
6300 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
6301 == rm_gid->gl_nodeguid))
6302 break;
6303 }
6304
6305 if (gid_list) {
6306 /*
6307 * Copy the following fields from rm_gid :
6308 * 1. gl_state
6309 * 2. gl_iou
6310 * 3. gl_gid & gl_ngids
6311 *
6312 * Note : Function is synchronized by
6313 * ibdm_busy flag.
6314 *
6315 * Note : Redirect info is initialized if
6316 * any MADs for the GID fail
6317 */
6318 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
6319 "copying info to GID with gl_iou != NULl");
6320 gid_list->gl_state = rm_gid->gl_state;
6321 gid_list->gl_iou = rm_gid->gl_iou;
6322 gid_list->gl_gid = rm_gid->gl_gid;
6323 gid_list->gl_ngids = rm_gid->gl_ngids;
6324
6325 /* Remove the GID from gl_gid list */
6326 ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6327 } else {
6328 /*
6329 * Handle a case where all GIDs to the IOU have
6330 * been removed.
6331 */
6332 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
6333 "to IOU");
6334
6335 ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
6336 return (rm_gid);
6337 }
6338 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6339 return (gid_list);
6340 }
6341 }
6342
6343 static void
6344 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
6345 ibdm_dp_gidinfo_t *rm_gid)
6346 {
6347 ibdm_gid_t *tmp, *prev;
6348
6349 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
6350 gid_info, rm_gid);
6351
6352 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
6353 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
6354 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
6355 if (prev == NULL)
6356 gid_info->gl_gid = tmp->gid_next;
6357 else
6358 prev->gid_next = tmp->gid_next;
6359
6360 kmem_free(tmp, sizeof (ibdm_gid_t));
6361 gid_info->gl_ngids--;
6362 break;
6363 } else {
6364 prev = tmp;
6365 tmp = tmp->gid_next;
6366 }
6367 }
6368 }
6369
6370 static void
6371 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
6372 {
6373 ibdm_gid_t *head = NULL, *new, *tail;
6374
6375 /* First copy the destination */
6376 for (; dest; dest = dest->gid_next) {
6377 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6378 new->gid_dgid_hi = dest->gid_dgid_hi;
6379 new->gid_dgid_lo = dest->gid_dgid_lo;
6380 new->gid_next = head;
6381 head = new;
6382 }
6383
6384 /* Insert this to the source */
6385 if (*src_ptr == NULL)
6386 *src_ptr = head;
6387 else {
6388 for (tail = *src_ptr; tail->gid_next != NULL;
6389 tail = tail->gid_next)
6390 ;
6391
6392 tail->gid_next = head;
6393 }
6394 }
6395
6396 static void
6397 ibdm_free_gid_list(ibdm_gid_t *head)
6398 {
6399 ibdm_gid_t *delete;
6400
6401 for (delete = head; delete; ) {
6402 head = delete->gid_next;
6403 kmem_free(delete, sizeof (ibdm_gid_t));
6404 delete = head;
6405 }
6406 }
6407
6408 /*
6409 * This function rescans the DM capable GIDs (gl_state is
6410 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
6411 * basically checks if the DM capable GID is reachable. If
6412 * not this is handled the same way as GID_UNAVAILABLE,
6413 * except that notifications are not send to IBnexus.
6414 *
6415 * This function also initializes the ioc_prev_list for
6416 * a particular IOC (when called from probe_ioc, with
6417 * ioc_guidp != NULL) or all IOCs for the gid (called from
6418 * sweep_fabric, ioc_guidp == NULL).
6419 */
6420 static void
6421 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
6422 {
6423 ibdm_dp_gidinfo_t *gid_info, *tmp;
6424 int ii, niocs, found;
6425 ibdm_hca_list_t *hca_list = NULL;
6426 ibdm_port_attr_t *port = NULL;
6427 ibdm_ioc_info_t *ioc_list;
6428
6429 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6430 found = 0;
6431 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
6432 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
6433 gid_info = gid_info->gl_next;
6434 continue;
6435 }
6436
6437 /*
6438 * Check if the GID is visible to any HCA ports.
6439 * Return if so.
6440 */
6441 mutex_enter(&ibdm.ibdm_hl_mutex);
6442 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6443 ibdm_get_next_port(&hca_list, &port, 1)) {
6444 if (ibdm_port_reachable(port->pa_sa_hdl,
6445 gid_info->gl_dgid_lo) == B_TRUE) {
6446 found = 1;
6447 break;
6448 }
6449 }
6450 mutex_exit(&ibdm.ibdm_hl_mutex);
6451
6452 if (found) {
6453 if (gid_info->gl_iou == NULL) {
6454 gid_info = gid_info->gl_next;
6455 continue;
6456 }
6457
6458 /* Intialize the ioc_prev_gid_list */
6459 niocs =
6460 gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
6461 for (ii = 0; ii < niocs; ii++) {
6462 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);
6463
6464 if (ioc_guidp == NULL || (*ioc_guidp ==
6465 ioc_list->ioc_profile.ioc_guid)) {
6466 /* Add info of GIDs in gid_info also */
6467 ibdm_addto_gidlist(
6468 &ioc_list->ioc_prev_gid_list,
6469 gid_info->gl_gid);
6470 ioc_list->ioc_prev_nportgids =
6471 gid_info->gl_ngids;
6472 }
6473 }
6474 gid_info = gid_info->gl_next;
6475 continue;
6476 }
6477
6478 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6479 "deleted port GUID %llx",
6480 gid_info->gl_dgid_lo);
6481
6482 /*
6483 * Update GID list in all IOCs affected by this
6484 */
6485 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6486
6487 /*
6488 * Remove GID from the global GID list
6489 * Handle the case where all port GIDs for an
6490 * IOU have been hot-removed.
6491 */
6492 mutex_enter(&ibdm.ibdm_mutex);
6493 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6494 mutex_enter(&gid_info->gl_mutex);
6495 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6496 mutex_exit(&gid_info->gl_mutex);
6497 }
6498
6499 tmp = gid_info->gl_next;
6500 if (gid_info->gl_prev != NULL)
6501 gid_info->gl_prev->gl_next = gid_info->gl_next;
6502 if (gid_info->gl_next != NULL)
6503 gid_info->gl_next->gl_prev = gid_info->gl_prev;
6504
6505 if (gid_info == ibdm.ibdm_dp_gidlist_head)
6506 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6507 if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6508 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6509 ibdm.ibdm_ngids--;
6510 mutex_exit(&ibdm.ibdm_mutex);
6511
6512 /* free the hca_list on this gid_info */
6513 ibdm_delete_glhca_list(gid_info);
6514
6515 mutex_destroy(&gid_info->gl_mutex);
6516 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6517
6518 gid_info = tmp;
6519
6520 /*
6521 * Pass on the IOCs with updated GIDs to IBnexus
6522 */
6523 if (ioc_list) {
6524 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6525 "IOC_PROP_UPDATE for %p\n", ioc_list);
6526 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6527 if (ibdm.ibdm_ibnex_callback != NULL) {
6528 (*ibdm.ibdm_ibnex_callback)((void *)
6529 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6530 }
6531 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6532 }
6533 }
6534 }
6535
6536 /*
6537 * This function notifies IBnex of IOCs on this GID.
6538 * Notification is for GIDs with gl_reprobe_flag set.
6539 * The flag is set when IOC probe / fabric sweep
6540 * probes a GID starting from CLASS port info.
6541 *
6542 * IBnexus will have information of a reconnected IOC
6543 * if it had probed it before. If this is a new IOC,
6544 * IBnexus ignores the notification.
6545 *
6546 * This function should be called with no locks held.
6547 */
6548 static void
6549 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
6550 {
6551 ibdm_ioc_info_t *ioc_list;
6552
6553 if (gid_info->gl_reprobe_flag == 0 ||
6554 gid_info->gl_iou == NULL)
6555 return;
6556
6557 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);
6558
6559 /*
6560 * Pass on the IOCs with updated GIDs to IBnexus
6561 */
6562 if (ioc_list) {
6563 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6564 if (ibdm.ibdm_ibnex_callback != NULL) {
6565 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6566 IBDM_EVENT_IOC_PROP_UPDATE);
6567 }
6568 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6569 }
6570 }
6571
6572
6573 static void
6574 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
6575 {
6576 if (arg != NULL)
6577 kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
6578 }
6579
6580 /*
6581 * This function parses the list of HCAs and HCA ports
6582 * to return the port_attr of the next HCA port. A port
6583 * connected to IB fabric (port_state active) is returned,
6584 * if connected_flag is set.
6585 */
6586 static void
6587 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
6588 ibdm_port_attr_t **inp_portp, int connect_flag)
6589 {
6590 int ii;
6591 ibdm_port_attr_t *port, *next_port = NULL;
6592 ibdm_port_attr_t *inp_port;
6593 ibdm_hca_list_t *hca_list;
6594 int found = 0;
6595
6596 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6597 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
6598 inp_hcap, inp_portp, connect_flag);
6599
6600 hca_list = *inp_hcap;
6601 inp_port = *inp_portp;
6602
6603 if (hca_list == NULL)
6604 hca_list = ibdm.ibdm_hca_list_head;
6605
6606 for (; hca_list; hca_list = hca_list->hl_next) {
6607 for (ii = 0; ii < hca_list->hl_nports; ii++) {
6608 port = &hca_list->hl_port_attr[ii];
6609
6610 /*
6611 * inp_port != NULL;
6612 * Skip till we find the matching port
6613 */
6614 if (inp_port && !found) {
6615 if (inp_port == port)
6616 found = 1;
6617 continue;
6618 }
6619
6620 if (!connect_flag) {
6621 next_port = port;
6622 break;
6623 }
6624
6625 if (port->pa_sa_hdl == NULL)
6626 ibdm_initialize_port(port);
6627 if (port->pa_sa_hdl == NULL)
6628 (void) ibdm_fini_port(port);
6629 else if (next_port == NULL &&
6630 port->pa_sa_hdl != NULL &&
6631 port->pa_state == IBT_PORT_ACTIVE) {
6632 next_port = port;
6633 break;
6634 }
6635 }
6636
6637 if (next_port)
6638 break;
6639 }
6640
6641 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
6642 "returns hca_list %p port %p", hca_list, next_port);
6643 *inp_hcap = hca_list;
6644 *inp_portp = next_port;
6645 }
6646
6647 static void
6648 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
6649 {
6650 ibdm_gid_t *tmp;
6651
6652 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6653 tmp->gid_dgid_hi = addgid->gl_dgid_hi;
6654 tmp->gid_dgid_lo = addgid->gl_dgid_lo;
6655
6656 mutex_enter(&nodegid->gl_mutex);
6657 tmp->gid_next = nodegid->gl_gid;
6658 nodegid->gl_gid = tmp;
6659 nodegid->gl_ngids++;
6660 mutex_exit(&nodegid->gl_mutex);
6661 }
6662
6663 static void
6664 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
6665 ibdm_hca_list_t *hca)
6666 {
6667 ibdm_hca_list_t *head, *prev = NULL, *temp;
6668
6669 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
6670 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
6671 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6672
6673 mutex_enter(&gid_info->gl_mutex);
6674 head = gid_info->gl_hca_list;
6675 if (head == NULL) {
6676 head = ibdm_dup_hca_attr(hca);
6677 head->hl_next = NULL;
6678 gid_info->gl_hca_list = head;
6679 mutex_exit(&gid_info->gl_mutex);
6680 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6681 "gid %p, gl_hca_list %p", gid_info,
6682 gid_info->gl_hca_list);
6683 return;
6684 }
6685
6686 /* Check if already in the list */
6687 while (head) {
6688 if (head->hl_hca_guid == hca->hl_hca_guid) {
6689 mutex_exit(&gid_info->gl_mutex);
6690 IBTF_DPRINTF_L4(ibdm_string,
6691 "\taddto_glhcalist : gid %p hca %p dup",
6692 gid_info, hca);
6693 return;
6694 }
6695 prev = head;
6696 head = head->hl_next;
6697 }
6698
6699 /* Add this HCA to gl_hca_list */
6700 temp = ibdm_dup_hca_attr(hca);
6701 temp->hl_next = NULL;
6702 prev->hl_next = temp;
6703 mutex_exit(&gid_info->gl_mutex);
6704
6705 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6706 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
6707 }
6708
6709 static void
6710 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
6711 {
6712 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6713 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6714
6715 mutex_enter(&gid_info->gl_mutex);
6716 if (gid_info->gl_hca_list)
6717 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
6718 gid_info->gl_hca_list = NULL;
6719 mutex_exit(&gid_info->gl_mutex);
6720 }
6721
6722
6723 static void
6724 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
6725 {
6726 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
6727 port_sa_hdl);
6728
6729 if (ibdm_enumerate_iocs == 0)
6730 return;
6731
6732 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6733 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6734
6735 /* Check : Not busy in another probe / sweep */
6736 mutex_enter(&ibdm.ibdm_mutex);
6737 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
6738 ibdm_dp_gidinfo_t *gid_info;
6739
6740 ibdm.ibdm_busy |= IBDM_BUSY;
6741 mutex_exit(&ibdm.ibdm_mutex);
6742
6743 /*
6744 * Check if any GID is using the SA & IBMF handle
6745 * of HCA port going down. Reset ibdm_dp_gidinfo_t
6746 * using another HCA port which can reach the GID.
6747 * This is for DM capable GIDs only, no need to do
6748 * this for others
6749 *
6750 * Delete the GID if no alternate HCA port to reach
6751 * it is found.
6752 */
6753 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6754 ibdm_dp_gidinfo_t *tmp;
6755
6756 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
6757 "checking gidinfo %p", gid_info);
6758
6759 if (gid_info->gl_sa_hdl == port_sa_hdl) {
6760 IBTF_DPRINTF_L3(ibdm_string,
6761 "\tevent_hdlr: down HCA port hdl "
6762 "matches gid %p", gid_info);
6763
6764 /*
6765 * The non-DM GIDs can come back
6766 * with a new subnet prefix, when
6767 * the HCA port commes up again. To
6768 * avoid issues, delete non-DM
6769 * capable GIDs, if the gid was
6770 * discovered using the HCA port
6771 * going down. This is ensured by
6772 * setting gl_disconnected to 1.
6773 */
6774 if (gid_info->gl_is_dm_capable == B_FALSE)
6775 gid_info->gl_disconnected = 1;
6776 else
6777 ibdm_reset_gidinfo(gid_info);
6778
6779 if (gid_info->gl_disconnected) {
6780 IBTF_DPRINTF_L3(ibdm_string,
6781 "\tevent_hdlr: deleting"
6782 " gid %p", gid_info);
6783 tmp = gid_info;
6784 gid_info = gid_info->gl_next;
6785 ibdm_delete_gidinfo(tmp);
6786 } else
6787 gid_info = gid_info->gl_next;
6788 } else
6789 gid_info = gid_info->gl_next;
6790 }
6791
6792 mutex_enter(&ibdm.ibdm_mutex);
6793 ibdm.ibdm_busy &= ~IBDM_BUSY;
6794 cv_signal(&ibdm.ibdm_busy_cv);
6795 }
6796 mutex_exit(&ibdm.ibdm_mutex);
6797 }
6798
6799 static void
6800 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6801 {
6802 ibdm_hca_list_t *hca_list = NULL;
6803 ibdm_port_attr_t *port = NULL;
6804 int gid_reinited = 0;
6805 sa_node_record_t *nr, *tmp;
6806 sa_portinfo_record_t *pi;
6807 size_t nr_len = 0, pi_len = 0;
6808 size_t path_len;
6809 ib_gid_t sgid, dgid;
6810 int ret, ii, nrecords;
6811 sa_path_record_t *path;
6812 uint8_t npaths = 1;
6813 ibdm_pkey_tbl_t *pkey_tbl;
6814
6815 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);
6816
6817 /*
6818 * Get list of all the ports reachable from the local known HCA
6819 * ports which are active
6820 */
6821 mutex_enter(&ibdm.ibdm_hl_mutex);
6822 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6823 ibdm_get_next_port(&hca_list, &port, 1)) {
6824
6825
6826 /*
6827 * Get the path and re-populate the gidinfo.
6828 * Getting the path is the same probe_ioc
6829 * Init the gid info as in ibdm_create_gidinfo()
6830 */
6831 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
6832 gidinfo->gl_nodeguid);
6833 if (nr == NULL) {
6834 IBTF_DPRINTF_L4(ibdm_string,
6835 "\treset_gidinfo : no records");
6836 continue;
6837 }
6838
6839 nrecords = (nr_len / sizeof (sa_node_record_t));
6840 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) {
6841 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
6842 break;
6843 }
6844
6845 if (ii == nrecords) {
6846 IBTF_DPRINTF_L4(ibdm_string,
6847 "\treset_gidinfo : no record for portguid");
6848 kmem_free(nr, nr_len);
6849 continue;
6850 }
6851
6852 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
6853 if (pi == NULL) {
6854 IBTF_DPRINTF_L4(ibdm_string,
6855 "\treset_gidinfo : no portinfo");
6856 kmem_free(nr, nr_len);
6857 continue;
6858 }
6859
6860 sgid.gid_prefix = port->pa_sn_prefix;
6861 sgid.gid_guid = port->pa_port_guid;
6862 dgid.gid_prefix = pi->PortInfo.GidPrefix;
6863 dgid.gid_guid = tmp->NodeInfo.PortGUID;
6864
6865 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
6866 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);
6867
6868 if ((ret != IBMF_SUCCESS) || path == NULL) {
6869 IBTF_DPRINTF_L4(ibdm_string,
6870 "\treset_gidinfo : no paths");
6871 kmem_free(pi, pi_len);
6872 kmem_free(nr, nr_len);
6873 continue;
6874 }
6875
6876 gidinfo->gl_dgid_hi = path->DGID.gid_prefix;
6877 gidinfo->gl_dgid_lo = path->DGID.gid_guid;
6878 gidinfo->gl_sgid_hi = path->SGID.gid_prefix;
6879 gidinfo->gl_sgid_lo = path->SGID.gid_guid;
6880 gidinfo->gl_p_key = path->P_Key;
6881 gidinfo->gl_sa_hdl = port->pa_sa_hdl;
6882 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl;
6883 gidinfo->gl_slid = path->SLID;
6884 gidinfo->gl_dlid = path->DLID;
6885 /* Reset redirect info, next MAD will set if redirected */
6886 gidinfo->gl_redirected = 0;
6887 gidinfo->gl_devid = (*tmp).NodeInfo.DeviceID;
6888 gidinfo->gl_SL = path->SL;
6889
6890 gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
6891 for (ii = 0; ii < port->pa_npkeys; ii++) {
6892 if (port->pa_pkey_tbl == NULL)
6893 break;
6894
6895 pkey_tbl = &port->pa_pkey_tbl[ii];
6896 if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) &&
6897 (pkey_tbl->pt_qp_hdl != NULL)) {
6898 gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
6899 break;
6900 }
6901 }
6902
6903 if (gidinfo->gl_qp_hdl == NULL)
6904 IBTF_DPRINTF_L2(ibdm_string,
6905 "\treset_gid_info: No matching Pkey");
6906 else
6907 gid_reinited = 1;
6908
6909 kmem_free(path, path_len);
6910 kmem_free(pi, pi_len);
6911 kmem_free(nr, nr_len);
6912 break;
6913 }
6914 mutex_exit(&ibdm.ibdm_hl_mutex);
6915
6916 if (!gid_reinited)
6917 gidinfo->gl_disconnected = 1;
6918 }
6919
6920 static void
6921 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6922 {
6923 ibdm_ioc_info_t *ioc_list;
6924 int in_gidlist = 0;
6925
6926 /*
6927 * Check if gidinfo has been inserted into the
6928 * ibdm_dp_gidlist_head list. gl_next or gl_prev
6929 * != NULL, if gidinfo is the list.
6930 */
6931 if (gidinfo->gl_prev != NULL ||
6932 gidinfo->gl_next != NULL ||
6933 ibdm.ibdm_dp_gidlist_head == gidinfo)
6934 in_gidlist = 1;
6935
6936 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);
6937
6938 /*
6939 * Remove GID from the global GID list
6940 * Handle the case where all port GIDs for an
6941 * IOU have been hot-removed.
6942 */
6943 mutex_enter(&ibdm.ibdm_mutex);
6944 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
6945 mutex_enter(&gidinfo->gl_mutex);
6946 (void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou);
6947 mutex_exit(&gidinfo->gl_mutex);
6948 }
6949
6950 /* Delete gl_hca_list */
6951 mutex_exit(&ibdm.ibdm_mutex);
6952 ibdm_delete_glhca_list(gidinfo);
6953 mutex_enter(&ibdm.ibdm_mutex);
6954
6955 if (in_gidlist) {
6956 if (gidinfo->gl_prev != NULL)
6957 gidinfo->gl_prev->gl_next = gidinfo->gl_next;
6958 if (gidinfo->gl_next != NULL)
6959 gidinfo->gl_next->gl_prev = gidinfo->gl_prev;
6960
6961 if (gidinfo == ibdm.ibdm_dp_gidlist_head)
6962 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
6963 if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
6964 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
6965 ibdm.ibdm_ngids--;
6966 }
6967 mutex_exit(&ibdm.ibdm_mutex);
6968
6969 mutex_destroy(&gidinfo->gl_mutex);
6970 cv_destroy(&gidinfo->gl_probe_cv);
6971 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));
6972
6973 /*
6974 * Pass on the IOCs with updated GIDs to IBnexus
6975 */
6976 if (ioc_list) {
6977 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
6978 "IOC_PROP_UPDATE for %p\n", ioc_list);
6979 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6980 if (ibdm.ibdm_ibnex_callback != NULL) {
6981 (*ibdm.ibdm_ibnex_callback)((void *)
6982 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6983 }
6984 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6985 }
6986 }
6987
6988
6989 static void
6990 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
6991 {
6992 uint32_t attr_mod;
6993
6994 attr_mod = (cb_args->cb_ioc_num + 1) << 16;
6995 attr_mod |= cb_args->cb_srvents_start;
6996 attr_mod |= (cb_args->cb_srvents_end) << 8;
6997 hdr->AttributeModifier = h2b32(attr_mod);
6998 }
6999
7000 static void
7001 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
7002 {
7003 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
7004 gid_info->gl_transactionID++;
7005 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
7006 IBTF_DPRINTF_L4(ibdm_string,
7007 "\tbump_transactionID(%p), wrapup", gid_info);
7008 gid_info->gl_transactionID = gid_info->gl_min_transactionID;
7009 }
7010 }
7011
7012 /*
7013 * gl_prev_iou is set for *non-reprobe* sweeep requests, which
7014 * detected that ChangeID in IOU info has changed. The service
7015 * entry also may have changed. Check if service entry in IOC
7016 * has changed wrt the prev iou, if so notify to IB Nexus.
7017 */
7018 static ibdm_ioc_info_t *
7019 ibdm_handle_prev_iou()
7020 {
7021 ibdm_dp_gidinfo_t *gid_info;
7022 ibdm_ioc_info_t *ioc_list_head = NULL, *ioc_list;
7023 ibdm_ioc_info_t *prev_ioc, *ioc;
7024 int ii, jj, niocs, prev_niocs;
7025
7026 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
7027
7028 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter");
7029 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
7030 gid_info = gid_info->gl_next) {
7031 if (gid_info->gl_prev_iou == NULL)
7032 continue;
7033
7034 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p",
7035 gid_info);
7036 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
7037 prev_niocs =
7038 gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots;
7039 for (ii = 0; ii < niocs; ii++) {
7040 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
7041
7042 /* Find matching IOC */
7043 for (jj = 0; jj < prev_niocs; jj++) {
7044 prev_ioc = (ibdm_ioc_info_t *)
7045 &gid_info->gl_prev_iou->iou_ioc_info[jj];
7046 if (prev_ioc->ioc_profile.ioc_guid ==
7047 ioc->ioc_profile.ioc_guid)
7048 break;
7049 }
7050 if (jj == prev_niocs)
7051 prev_ioc = NULL;
7052 if (ioc == NULL || prev_ioc == NULL)
7053 continue;
7054 if ((ioc->ioc_profile.ioc_service_entries !=
7055 prev_ioc->ioc_profile.ioc_service_entries) ||
7056 ibdm_serv_cmp(&ioc->ioc_serv[0],
7057 &prev_ioc->ioc_serv[0],
7058 ioc->ioc_profile.ioc_service_entries) != 0) {
7059 IBTF_DPRINTF_L4(ibdm_string,
7060 "/thandle_prev_iou modified IOC: "
7061 "current ioc %p, old ioc %p",
7062 ioc, prev_ioc);
7063 mutex_enter(&gid_info->gl_mutex);
7064 ioc_list = ibdm_dup_ioc_info(ioc, gid_info);
7065 mutex_exit(&gid_info->gl_mutex);
7066 ioc_list->ioc_info_updated.ib_prop_updated
7067 = 0;
7068 ioc_list->ioc_info_updated.ib_srv_prop_updated
7069 = 1;
7070
7071 if (ioc_list_head == NULL)
7072 ioc_list_head = ioc_list;
7073 else {
7074 ioc_list_head->ioc_next = ioc_list;
7075 ioc_list_head = ioc_list;
7076 }
7077 }
7078 }
7079
7080 mutex_enter(&gid_info->gl_mutex);
7081 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou);
7082 mutex_exit(&gid_info->gl_mutex);
7083 }
7084 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p",
7085 ioc_list_head);
7086 return (ioc_list_head);
7087 }
7088
7089 /*
7090 * Compares two service entries lists, returns 0 if same, returns 1
7091 * if no match.
7092 */
7093 static int
7094 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2,
7095 int nserv)
7096 {
7097 int ii;
7098
7099 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter");
7100 for (ii = 0; ii < nserv; ii++, serv1++, serv2++) {
7101 if (serv1->se_attr.srv_id != serv2->se_attr.srv_id ||
7102 bcmp(serv1->se_attr.srv_name,
7103 serv2->se_attr.srv_name,
7104 IB_DM_MAX_SVC_NAME_LEN) != 0) {
7105 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1");
7106 return (1);
7107 }
7108 }
7109 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0");
7110 return (0);
7111 }
7112
7113 /* For debugging purpose only */
7114 #ifdef DEBUG
7115 void
7116 ibdm_dump_mad_hdr(ib_mad_hdr_t *mad_hdr)
7117 {
7118 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
7119 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");
7120
7121 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x"
7122 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
7123 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
7124 "\tR Method : 0x%x",
7125 mad_hdr->ClassVersion, mad_hdr->R_Method);
7126 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x"
7127 "\tTransaction ID : 0x%llx",
7128 b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID));
7129 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x"
7130 "\tAttribute Modified : 0x%lx",
7131 b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier));
7132 }
7133
7134
7135 void
7136 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
7137 {
7138 ib_mad_hdr_t *mad_hdr;
7139
7140 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
7141 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------");
7142
7143 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x"
7144 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
7145 ibmf_msg->im_local_addr.ia_remote_lid,
7146 ibmf_msg->im_local_addr.ia_remote_qno);
7147 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x"
7148 " SL : 0x%x", ibmf_msg->im_local_addr.ia_p_key,
7149 ibmf_msg->im_local_addr.ia_q_key,
7150 ibmf_msg->im_local_addr.ia_service_level);
7151
7152 if (flag)
7153 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
7154 else
7155 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);
7156
7157 ibdm_dump_mad_hdr(mad_hdr);
7158 }
7159
7160
7161 void
7162 ibdm_dump_path_info(sa_path_record_t *path)
7163 {
7164 IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
7165 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");
7166
7167 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx",
7168 path->DGID.gid_prefix, path->DGID.gid_guid);
7169 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx",
7170 path->SGID.gid_prefix, path->SGID.gid_guid);
7171 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\t\tDlID : %x",
7172 path->SLID, path->DLID);
7173 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x\t\tSL : %x",
7174 path->P_Key, path->SL);
7175 }
7176
7177
7178 void
7179 ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo)
7180 {
7181 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
7182 IBTF_DPRINTF_L4("ibdm", "\t\t --------------");
7183
7184 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
7185 ((b2h32(classportinfo->RespTimeValue)) & 0x1F));
7186
7187 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%llx",
7188 b2h64(classportinfo->RedirectGID_hi));
7189 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%llx",
7190 b2h64(classportinfo->RedirectGID_lo));
7191 IBTF_DPRINTF_L4("ibdm", "\t Redirected TC : 0x%x",
7192 classportinfo->RedirectTC);
7193 IBTF_DPRINTF_L4("ibdm", "\t Redirected SL : 0x%x",
7194 classportinfo->RedirectSL);
7195 IBTF_DPRINTF_L4("ibdm", "\t Redirected FL : 0x%x",
7196 classportinfo->RedirectFL);
7197 IBTF_DPRINTF_L4("ibdm", "\t Redirected LID : 0x%x",
7198 b2h16(classportinfo->RedirectLID));
7199 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x",
7200 b2h16(classportinfo->RedirectP_Key));
7201 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x",
7202 classportinfo->RedirectQP);
7203 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x",
7204 b2h32(classportinfo->RedirectQ_Key));
7205 IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi : 0x%llx",
7206 b2h64(classportinfo->TrapGID_hi));
7207 IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo : 0x%llx",
7208 b2h64(classportinfo->TrapGID_lo));
7209 IBTF_DPRINTF_L4("ibdm", "\t Trap TC : 0x%x",
7210 classportinfo->TrapTC);
7211 IBTF_DPRINTF_L4("ibdm", "\t Trap SL : 0x%x",
7212 classportinfo->TrapSL);
7213 IBTF_DPRINTF_L4("ibdm", "\t Trap FL : 0x%x",
7214 classportinfo->TrapFL);
7215 IBTF_DPRINTF_L4("ibdm", "\t Trap LID : 0x%x",
7216 b2h16(classportinfo->TrapLID));
7217 IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key : 0x%x",
7218 b2h16(classportinfo->TrapP_Key));
7219 IBTF_DPRINTF_L4("ibdm", "\t Trap HL : 0x%x",
7220 classportinfo->TrapHL);
7221 IBTF_DPRINTF_L4("ibdm", "\t Trap QP : 0x%x",
7222 classportinfo->TrapQP);
7223 IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key : 0x%x",
7224 b2h32(classportinfo->TrapQ_Key));
7225 }
7226
7227
7228 void
7229 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
7230 {
7231 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
7232 IBTF_DPRINTF_L4("ibdm", "\t\t ------------");
7233
7234 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x",
7235 b2h16(iou_info->iou_changeid));
7236 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d",
7237 iou_info->iou_num_ctrl_slots);
7238 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x",
7239 iou_info->iou_flag);
7240 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x",
7241 iou_info->iou_ctrl_list[0]);
7242 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x",
7243 iou_info->iou_ctrl_list[1]);
7244 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x",
7245 iou_info->iou_ctrl_list[2]);
7246 }
7247
7248
7249 void
7250 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
7251 {
7252 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
7253 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");
7254
7255 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid);
7256 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid);
7257 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid);
7258 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver);
7259 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id);
7260 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class);
7261 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
7262 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol);
7263 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver);
7264 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth);
7265 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
7266 ioc->ioc_rdma_read_qdepth);
7267 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz);
7268 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
7269 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x",
7270 ioc->ioc_ctrl_opcap_mask);
7271 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries);
7272 }
7273
7274
7275 void
7276 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
7277 {
7278 IBTF_DPRINTF_L4("ibdm",
7279 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);
7280
7281 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
7282 "Service Name : %s", srv_ents->srv_name);
7283 }
7284
7285 int ibdm_allow_sweep_fabric_timestamp = 1;
7286
7287 void
7288 ibdm_dump_sweep_fabric_timestamp(int flag)
7289 {
7290 static hrtime_t x;
7291 if (flag) {
7292 if (ibdm_allow_sweep_fabric_timestamp) {
7293 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
7294 "sweep %lld ms", ((gethrtime() - x)/ 1000000));
7295 }
7296 x = 0;
7297 } else
7298 x = gethrtime();
7299 }
7300 #endif