Print this page
7127 remove -Wno-missing-braces from Makefile.uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c
+++ new/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24 /*
25 25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 26 */
27 27
28 28 /*
29 29 * ibdm.c
30 30 *
31 31 * This file contains the InifiniBand Device Manager (IBDM) support functions.
32 32 * IB nexus driver will only be the client for the IBDM module.
33 33 *
34 34 * IBDM registers with IBTF for HCA arrival/removal notification.
35 35 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
36 36 * the IOU's.
37 37 *
38 38 * IB nexus driver registers with IBDM to find the information about the
39 39 * HCA's and IOC's (behind the IOU) present on the IB fabric.
40 40 */
41 41
42 42 #include <sys/sysmacros.h>
43 43 #include <sys/systm.h>
44 44 #include <sys/taskq.h>
45 45 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
46 46 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
47 47 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
48 48 #include <sys/modctl.h>
49 49
50 50 /* Function Prototype declarations */
51 51 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
52 52 static int ibdm_fini(void);
53 53 static int ibdm_init(void);
54 54 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
55 55 ibdm_hca_list_t *);
56 56 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
57 57 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
58 58 static boolean_t ibdm_is_cisco(ib_guid_t);
59 59 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
60 60 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
61 61 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
62 62 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
63 63 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
64 64 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
65 65 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
66 66 ib_guid_t *, ib_guid_t *);
67 67 static int ibdm_retry_command(ibdm_timeout_cb_args_t *);
68 68 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
69 69 static int ibdm_verify_mad_status(ib_mad_hdr_t *);
70 70 static int ibdm_handle_redirection(ibmf_msg_t *,
71 71 ibdm_dp_gidinfo_t *, int *);
72 72 static void ibdm_wait_probe_completion(void);
73 73 static void ibdm_sweep_fabric(int);
74 74 static void ibdm_probe_gid_thread(void *);
75 75 static void ibdm_wakeup_probe_gid_cv(void);
76 76 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
77 77 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
78 78 static void ibdm_update_port_attr(ibdm_port_attr_t *);
79 79 static void ibdm_handle_hca_attach(ib_guid_t);
80 80 static void ibdm_handle_srventry_mad(ibmf_msg_t *,
81 81 ibdm_dp_gidinfo_t *, int *);
82 82 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
83 83 static void ibdm_recv_incoming_mad(void *);
84 84 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
85 85 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
86 86 static void ibdm_pkt_timeout_hdlr(void *arg);
87 87 static void ibdm_initialize_port(ibdm_port_attr_t *);
88 88 static void ibdm_update_port_pkeys(ibdm_port_attr_t *port);
89 89 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
90 90 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *);
91 91 static void ibdm_alloc_send_buffers(ibmf_msg_t *);
92 92 static void ibdm_free_send_buffers(ibmf_msg_t *);
93 93 static void ibdm_handle_hca_detach(ib_guid_t);
94 94 static void ibdm_handle_port_change_event(ibt_async_event_t *);
95 95 static int ibdm_fini_port(ibdm_port_attr_t *);
96 96 static int ibdm_uninit_hca(ibdm_hca_list_t *);
97 97 static void ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *,
98 98 ibdm_dp_gidinfo_t *, int *);
99 99 static void ibdm_handle_iounitinfo(ibmf_handle_t,
100 100 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
101 101 static void ibdm_handle_ioc_profile(ibmf_handle_t,
102 102 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
103 103 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t,
104 104 ibt_async_code_t, ibt_async_event_t *);
105 105 static void ibdm_handle_classportinfo(ibmf_handle_t,
106 106 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
107 107 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
108 108 ibdm_dp_gidinfo_t *);
109 109
110 110 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *);
111 111 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *,
112 112 ibdm_dp_gidinfo_t *gid_list);
113 113 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
114 114 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t,
115 115 ibdm_dp_gidinfo_t *, int *);
116 116 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *,
117 117 ibdm_hca_list_t **);
118 118 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t,
119 119 size_t *, ib_guid_t);
120 120 static int ibdm_get_node_record_by_port(ibmf_saa_handle_t,
121 121 ib_guid_t, sa_node_record_t **, size_t *);
122 122 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
123 123 ib_lid_t);
124 124 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *,
125 125 ib_gid_t, ib_gid_t);
126 126 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t);
127 127 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
128 128 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
129 129 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
130 130 ibmf_saa_event_details_t *, void *);
131 131 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
132 132 ibdm_dp_gidinfo_t *);
133 133 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
134 134 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
135 135 ibdm_dp_gidinfo_t *);
136 136 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
137 137 static void ibdm_free_gid_list(ibdm_gid_t *);
138 138 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
139 139 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
140 140 static void ibdm_saa_event_taskq(void *);
141 141 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
142 142 static void ibdm_get_next_port(ibdm_hca_list_t **,
143 143 ibdm_port_attr_t **, int);
144 144 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
145 145 ibdm_dp_gidinfo_t *);
146 146 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
147 147 ibdm_hca_list_t *);
148 148 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
149 149 static void ibdm_saa_handle_new_gid(void *);
150 150 static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
151 151 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
152 152 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
153 153 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
154 154 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
155 155 static ibdm_ioc_info_t *ibdm_handle_prev_iou();
156 156 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *,
157 157 int);
158 158 static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t,
159 159 ibdm_dp_gidinfo_t **);
160 160
161 161 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT;
162 162 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES;
163 163 #ifdef DEBUG
164 164 int ibdm_ignore_saa_event = 0;
165 165 #endif
|
↓ open down ↓ |
165 lines elided |
↑ open up ↑ |
166 166 int ibdm_enumerate_iocs = 0;
167 167
168 168 /* Modload support */
169 169 static struct modlmisc ibdm_modlmisc = {
170 170 &mod_miscops,
171 171 "InfiniBand Device Manager"
172 172 };
173 173
174 174 struct modlinkage ibdm_modlinkage = {
175 175 MODREV_1,
176 - (void *)&ibdm_modlmisc,
177 - NULL
176 + { (void *)&ibdm_modlmisc, NULL }
178 177 };
179 178
180 179 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
181 180 IBTI_V_CURR,
182 181 IBT_DM,
183 182 ibdm_event_hdlr,
184 183 NULL,
185 184 "ibdm"
186 185 };
187 186
188 187 /* Global variables */
189 188 ibdm_t ibdm;
190 189 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
191 190 char *ibdm_string = "ibdm";
192 191
193 192 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
194 193 ibdm.ibdm_dp_gidlist_head))
195 194
196 195 /*
197 196 * _init
198 197 * Loadable module init, called before any other module.
199 198 * Initialize mutex
200 199 * Register with IBTF
201 200 */
202 201 int
203 202 _init(void)
204 203 {
205 204 int err;
206 205
207 206 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);
208 207
209 208 if ((err = ibdm_init()) != IBDM_SUCCESS) {
210 209 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
211 210 (void) ibdm_fini();
212 211 return (DDI_FAILURE);
213 212 }
214 213
215 214 if ((err = mod_install(&ibdm_modlinkage)) != 0) {
216 215 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
217 216 (void) ibdm_fini();
218 217 }
219 218 return (err);
220 219 }
221 220
222 221
223 222 int
224 223 _fini(void)
225 224 {
226 225 int err;
227 226
228 227 if ((err = ibdm_fini()) != IBDM_SUCCESS) {
229 228 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
230 229 (void) ibdm_init();
231 230 return (EBUSY);
232 231 }
233 232
234 233 if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
235 234 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
236 235 (void) ibdm_init();
237 236 }
238 237 return (err);
239 238 }
240 239
241 240
242 241 int
243 242 _info(struct modinfo *modinfop)
244 243 {
245 244 return (mod_info(&ibdm_modlinkage, modinfop));
246 245 }
247 246
248 247
249 248 /*
250 249 * ibdm_init():
251 250 * Register with IBTF
252 251 * Allocate memory for the HCAs
253 252 * Allocate minor-nodes for the HCAs
254 253 */
255 254 static int
256 255 ibdm_init(void)
257 256 {
258 257 int i, hca_count;
259 258 ib_guid_t *hca_guids;
260 259 ibt_status_t status;
261 260
262 261 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
263 262 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
264 263 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
265 264 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
266 265 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
267 266 cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL);
268 267 mutex_enter(&ibdm.ibdm_mutex);
269 268 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
270 269 }
271 270
272 271 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
273 272 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
274 273 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
275 274 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
276 275 "failed %x", status);
277 276 mutex_exit(&ibdm.ibdm_mutex);
278 277 return (IBDM_FAILURE);
279 278 }
280 279
281 280 ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
282 281 mutex_exit(&ibdm.ibdm_mutex);
283 282 }
284 283
285 284
286 285 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
287 286 hca_count = ibt_get_hca_list(&hca_guids);
288 287 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
289 288 for (i = 0; i < hca_count; i++)
290 289 (void) ibdm_handle_hca_attach(hca_guids[i]);
291 290 if (hca_count)
292 291 ibt_free_hca_list(hca_guids, hca_count);
293 292
294 293 mutex_enter(&ibdm.ibdm_mutex);
295 294 ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
296 295 mutex_exit(&ibdm.ibdm_mutex);
297 296 }
298 297
299 298 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
300 299 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
301 300 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
302 301 mutex_enter(&ibdm.ibdm_mutex);
303 302 ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
304 303 mutex_exit(&ibdm.ibdm_mutex);
305 304 }
306 305 return (IBDM_SUCCESS);
307 306 }
308 307
309 308
310 309 static int
311 310 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup)
312 311 {
313 312 int ii, k, niocs;
314 313 size_t size;
315 314 ibdm_gid_t *delete, *head;
316 315 timeout_id_t timeout_id;
317 316 ibdm_ioc_info_t *ioc;
318 317 ibdm_iou_info_t *gl_iou = *ioup;
319 318
320 319 ASSERT(mutex_owned(&gid_info->gl_mutex));
321 320 if (gl_iou == NULL) {
322 321 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
323 322 return (0);
324 323 }
325 324
326 325 niocs = gl_iou->iou_info.iou_num_ctrl_slots;
327 326 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
328 327 gid_info, niocs);
329 328
330 329 for (ii = 0; ii < niocs; ii++) {
331 330 ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii];
332 331
333 332 /* handle the case where an ioc_timeout_id is scheduled */
334 333 if (ioc->ioc_timeout_id) {
335 334 timeout_id = ioc->ioc_timeout_id;
336 335 ioc->ioc_timeout_id = 0;
337 336 mutex_exit(&gid_info->gl_mutex);
338 337 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
339 338 "ioc_timeout_id = 0x%x", timeout_id);
340 339 if (untimeout(timeout_id) == -1) {
341 340 IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
342 341 "untimeout ioc_timeout_id failed");
343 342 mutex_enter(&gid_info->gl_mutex);
344 343 return (-1);
345 344 }
346 345 mutex_enter(&gid_info->gl_mutex);
347 346 }
348 347
349 348 /* handle the case where an ioc_dc_timeout_id is scheduled */
350 349 if (ioc->ioc_dc_timeout_id) {
351 350 timeout_id = ioc->ioc_dc_timeout_id;
352 351 ioc->ioc_dc_timeout_id = 0;
353 352 mutex_exit(&gid_info->gl_mutex);
354 353 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
355 354 "ioc_dc_timeout_id = 0x%x", timeout_id);
356 355 if (untimeout(timeout_id) == -1) {
357 356 IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
358 357 "untimeout ioc_dc_timeout_id failed");
359 358 mutex_enter(&gid_info->gl_mutex);
360 359 return (-1);
361 360 }
362 361 mutex_enter(&gid_info->gl_mutex);
363 362 }
364 363
365 364 /* handle the case where serv[k].se_timeout_id is scheduled */
366 365 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
367 366 if (ioc->ioc_serv[k].se_timeout_id) {
368 367 timeout_id = ioc->ioc_serv[k].se_timeout_id;
369 368 ioc->ioc_serv[k].se_timeout_id = 0;
370 369 mutex_exit(&gid_info->gl_mutex);
371 370 IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
372 371 "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
373 372 k, timeout_id);
374 373 if (untimeout(timeout_id) == -1) {
375 374 IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
376 375 " untimeout se_timeout_id failed");
377 376 mutex_enter(&gid_info->gl_mutex);
378 377 return (-1);
379 378 }
380 379 mutex_enter(&gid_info->gl_mutex);
381 380 }
382 381 }
383 382
384 383 /* delete GID list in IOC */
385 384 head = ioc->ioc_gid_list;
386 385 while (head) {
387 386 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
388 387 "Deleting gid_list struct %p", head);
389 388 delete = head;
390 389 head = head->gid_next;
391 390 kmem_free(delete, sizeof (ibdm_gid_t));
392 391 }
393 392 ioc->ioc_gid_list = NULL;
394 393
395 394 /* delete ioc_serv */
396 395 size = ioc->ioc_profile.ioc_service_entries *
397 396 sizeof (ibdm_srvents_info_t);
398 397 if (ioc->ioc_serv && size) {
399 398 kmem_free(ioc->ioc_serv, size);
400 399 ioc->ioc_serv = NULL;
401 400 }
402 401 }
403 402 /*
404 403 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information
405 404 * via the switch during the probe process.
406 405 */
407 406 gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE;
408 407
409 408 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
410 409 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
411 410 kmem_free(gl_iou, size);
412 411 *ioup = NULL;
413 412 return (0);
414 413 }
415 414
416 415
417 416 /*
418 417 * ibdm_fini():
419 418 * Un-register with IBTF
420 419 * De allocate memory for the GID info
421 420 */
422 421 static int
423 422 ibdm_fini()
424 423 {
425 424 int ii;
426 425 ibdm_hca_list_t *hca_list, *temp;
427 426 ibdm_dp_gidinfo_t *gid_info, *tmp;
428 427 ibdm_gid_t *head, *delete;
429 428
430 429 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");
431 430
432 431 mutex_enter(&ibdm.ibdm_hl_mutex);
433 432 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
434 433 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
435 434 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
436 435 mutex_exit(&ibdm.ibdm_hl_mutex);
437 436 return (IBDM_FAILURE);
438 437 }
439 438 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
440 439 ibdm.ibdm_ibt_clnt_hdl = NULL;
441 440 }
442 441
443 442 hca_list = ibdm.ibdm_hca_list_head;
444 443 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
445 444 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
446 445 temp = hca_list;
447 446 hca_list = hca_list->hl_next;
448 447 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
449 448 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
450 449 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
451 450 "uninit_hca %p failed", temp);
452 451 mutex_exit(&ibdm.ibdm_hl_mutex);
453 452 return (IBDM_FAILURE);
454 453 }
455 454 }
456 455 mutex_exit(&ibdm.ibdm_hl_mutex);
457 456
458 457 mutex_enter(&ibdm.ibdm_mutex);
459 458 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
460 459 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;
461 460
462 461 gid_info = ibdm.ibdm_dp_gidlist_head;
463 462 while (gid_info) {
464 463 mutex_enter(&gid_info->gl_mutex);
465 464 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
466 465 mutex_exit(&gid_info->gl_mutex);
467 466 ibdm_delete_glhca_list(gid_info);
468 467
469 468 tmp = gid_info;
470 469 gid_info = gid_info->gl_next;
471 470 mutex_destroy(&tmp->gl_mutex);
472 471 head = tmp->gl_gid;
473 472 while (head) {
474 473 IBTF_DPRINTF_L4("ibdm",
475 474 "\tibdm_fini: Deleting gid structs");
476 475 delete = head;
477 476 head = head->gid_next;
478 477 kmem_free(delete, sizeof (ibdm_gid_t));
479 478 }
480 479 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
481 480 }
482 481 mutex_exit(&ibdm.ibdm_mutex);
483 482
484 483 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
485 484 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
486 485 mutex_destroy(&ibdm.ibdm_mutex);
487 486 mutex_destroy(&ibdm.ibdm_hl_mutex);
488 487 mutex_destroy(&ibdm.ibdm_ibnex_mutex);
489 488 cv_destroy(&ibdm.ibdm_port_settle_cv);
490 489 }
491 490 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
492 491 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
493 492 cv_destroy(&ibdm.ibdm_probe_cv);
494 493 cv_destroy(&ibdm.ibdm_busy_cv);
495 494 }
496 495 return (IBDM_SUCCESS);
497 496 }
498 497
499 498
500 499 /*
501 500 * ibdm_event_hdlr()
502 501 *
503 502 * IBDM registers this asynchronous event handler at the time of
504 503 * ibt_attach. IBDM support the following async events. For other
505 504 * event, simply returns success.
506 505 * IBT_HCA_ATTACH_EVENT:
507 506 * Retrieves the information about all the port that are
508 507 * present on this HCA, allocates the port attributes
509 508 * structure and calls IB nexus callback routine with
510 509 * the port attributes structure as an input argument.
511 510 * IBT_HCA_DETACH_EVENT:
512 511 * Retrieves the information about all the ports that are
513 512 * present on this HCA and calls IB nexus callback with
514 513 * port guid as an argument
515 514 * IBT_EVENT_PORT_UP:
516 515 * Register with IBMF and SA access
517 516 * Setup IBMF receive callback routine
518 517 * IBT_EVENT_PORT_DOWN:
519 518 * Un-Register with IBMF and SA access
520 519 * Teardown IBMF receive callback routine
521 520 */
522 521 /*ARGSUSED*/
523 522 static void
524 523 ibdm_event_hdlr(void *clnt_hdl,
525 524 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
526 525 {
527 526 ibdm_hca_list_t *hca_list;
528 527 ibdm_port_attr_t *port;
529 528 ibmf_saa_handle_t port_sa_hdl;
530 529
531 530 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);
532 531
533 532 switch (code) {
534 533 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */
535 534 ibdm_handle_hca_attach(event->ev_hca_guid);
536 535 break;
537 536
538 537 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */
539 538 ibdm_handle_hca_detach(event->ev_hca_guid);
540 539 mutex_enter(&ibdm.ibdm_ibnex_mutex);
541 540 if (ibdm.ibdm_ibnex_callback != NULL) {
542 541 (*ibdm.ibdm_ibnex_callback)((void *)
543 542 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
544 543 }
545 544 mutex_exit(&ibdm.ibdm_ibnex_mutex);
546 545 break;
547 546
548 547 case IBT_EVENT_PORT_UP:
549 548 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
550 549 mutex_enter(&ibdm.ibdm_hl_mutex);
551 550 port = ibdm_get_port_attr(event, &hca_list);
552 551 if (port == NULL) {
553 552 IBTF_DPRINTF_L2("ibdm",
554 553 "\tevent_hdlr: HCA not present");
555 554 mutex_exit(&ibdm.ibdm_hl_mutex);
556 555 break;
557 556 }
558 557 ibdm_initialize_port(port);
559 558 hca_list->hl_nports_active++;
560 559 cv_broadcast(&ibdm.ibdm_port_settle_cv);
561 560 mutex_exit(&ibdm.ibdm_hl_mutex);
562 561
563 562 /* Inform IB nexus driver */
564 563 mutex_enter(&ibdm.ibdm_ibnex_mutex);
565 564 if (ibdm.ibdm_ibnex_callback != NULL) {
566 565 (*ibdm.ibdm_ibnex_callback)((void *)
567 566 &event->ev_hca_guid, IBDM_EVENT_PORT_UP);
568 567 }
569 568 mutex_exit(&ibdm.ibdm_ibnex_mutex);
570 569 break;
571 570
572 571 case IBT_ERROR_PORT_DOWN:
573 572 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
574 573 mutex_enter(&ibdm.ibdm_hl_mutex);
575 574 port = ibdm_get_port_attr(event, &hca_list);
576 575 if (port == NULL) {
577 576 IBTF_DPRINTF_L2("ibdm",
578 577 "\tevent_hdlr: HCA not present");
579 578 mutex_exit(&ibdm.ibdm_hl_mutex);
580 579 break;
581 580 }
582 581 hca_list->hl_nports_active--;
583 582 port_sa_hdl = port->pa_sa_hdl;
584 583 (void) ibdm_fini_port(port);
585 584 port->pa_state = IBT_PORT_DOWN;
586 585 cv_broadcast(&ibdm.ibdm_port_settle_cv);
587 586 mutex_exit(&ibdm.ibdm_hl_mutex);
588 587 ibdm_reset_all_dgids(port_sa_hdl);
589 588 break;
590 589
591 590 case IBT_PORT_CHANGE_EVENT:
592 591 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE");
593 592 if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY)
594 593 ibdm_handle_port_change_event(event);
595 594 break;
596 595
597 596 default: /* Ignore all other events/errors */
598 597 break;
599 598 }
600 599 }
601 600
602 601 static void
603 602 ibdm_handle_port_change_event(ibt_async_event_t *event)
604 603 {
605 604 ibdm_port_attr_t *port;
606 605 ibdm_hca_list_t *hca_list;
607 606
608 607 IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:"
609 608 " HCA guid %llx", event->ev_hca_guid);
610 609 mutex_enter(&ibdm.ibdm_hl_mutex);
611 610 port = ibdm_get_port_attr(event, &hca_list);
612 611 if (port == NULL) {
613 612 IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present");
614 613 mutex_exit(&ibdm.ibdm_hl_mutex);
615 614 return;
616 615 }
617 616 ibdm_update_port_pkeys(port);
618 617 cv_broadcast(&ibdm.ibdm_port_settle_cv);
619 618 mutex_exit(&ibdm.ibdm_hl_mutex);
620 619
621 620 /* Inform IB nexus driver */
622 621 mutex_enter(&ibdm.ibdm_ibnex_mutex);
623 622 if (ibdm.ibdm_ibnex_callback != NULL) {
624 623 (*ibdm.ibdm_ibnex_callback)((void *)
625 624 &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE);
626 625 }
627 626 mutex_exit(&ibdm.ibdm_ibnex_mutex);
628 627 }
629 628
630 629 /*
631 630 * ibdm_update_port_pkeys()
632 631 * Update the pkey table
633 632 * Update the port attributes
634 633 */
635 634 static void
636 635 ibdm_update_port_pkeys(ibdm_port_attr_t *port)
637 636 {
638 637 uint_t nports, size;
639 638 uint_t pkey_idx, opkey_idx;
640 639 uint16_t npkeys;
641 640 ibt_hca_portinfo_t *pinfop;
642 641 ib_pkey_t pkey;
643 642 ibdm_pkey_tbl_t *pkey_tbl;
644 643 ibdm_port_attr_t newport;
645 644
646 645 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:");
647 646 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
648 647
649 648 /* Check whether the port is active */
650 649 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
651 650 NULL) != IBT_SUCCESS)
652 651 return;
653 652
654 653 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
655 654 &pinfop, &nports, &size) != IBT_SUCCESS) {
656 655 /* This should not occur */
657 656 port->pa_npkeys = 0;
658 657 port->pa_pkey_tbl = NULL;
659 658 return;
660 659 }
661 660
662 661 npkeys = pinfop->p_pkey_tbl_sz;
663 662 pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
664 663 newport.pa_pkey_tbl = pkey_tbl;
665 664 newport.pa_ibmf_hdl = port->pa_ibmf_hdl;
666 665
667 666 for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) {
668 667 pkey = pkey_tbl[pkey_idx].pt_pkey =
669 668 pinfop->p_pkey_tbl[pkey_idx];
670 669 /*
671 670 * Is this pkey present in the current table ?
672 671 */
673 672 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
674 673 if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) {
675 674 pkey_tbl[pkey_idx].pt_qp_hdl =
676 675 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl;
677 676 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL;
678 677 break;
679 678 }
680 679 }
681 680
682 681 if (opkey_idx == port->pa_npkeys) {
683 682 pkey = pkey_tbl[pkey_idx].pt_pkey;
684 683 if (IBDM_INVALID_PKEY(pkey)) {
685 684 pkey_tbl[pkey_idx].pt_qp_hdl = NULL;
686 685 continue;
687 686 }
688 687 ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx);
689 688 }
690 689 }
691 690
692 691 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
693 692 if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) {
694 693 if (ibdm_port_attr_ibmf_fini(port, opkey_idx) !=
695 694 IBDM_SUCCESS) {
696 695 IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: "
697 696 "ibdm_port_attr_ibmf_fini failed for "
698 697 "port pkey 0x%x",
699 698 port->pa_pkey_tbl[opkey_idx].pt_pkey);
700 699 }
701 700 }
702 701 }
703 702
704 703 if (port->pa_pkey_tbl != NULL) {
705 704 kmem_free(port->pa_pkey_tbl,
706 705 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
707 706 }
708 707
709 708 port->pa_npkeys = npkeys;
710 709 port->pa_pkey_tbl = pkey_tbl;
711 710 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
712 711 port->pa_state = pinfop->p_linkstate;
713 712 ibt_free_portinfo(pinfop, size);
714 713 }
715 714
716 715 /*
717 716 * ibdm_initialize_port()
718 717 * Register with IBMF
719 718 * Register with SA access
720 719 * Register a receive callback routine with IBMF. IBMF invokes
721 720 * this routine whenever a MAD arrives at this port.
722 721 * Update the port attributes
723 722 */
724 723 static void
725 724 ibdm_initialize_port(ibdm_port_attr_t *port)
726 725 {
727 726 int ii;
728 727 uint_t nports, size;
729 728 uint_t pkey_idx;
730 729 ib_pkey_t pkey;
731 730 ibt_hca_portinfo_t *pinfop;
732 731 ibmf_register_info_t ibmf_reg;
733 732 ibmf_saa_subnet_event_args_t event_args;
734 733
735 734 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
736 735 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
737 736
738 737 /* Check whether the port is active */
739 738 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
740 739 NULL) != IBT_SUCCESS)
741 740 return;
742 741
743 742 if (port->pa_sa_hdl != NULL || port->pa_pkey_tbl != NULL)
744 743 return;
745 744
746 745 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
747 746 &pinfop, &nports, &size) != IBT_SUCCESS) {
748 747 /* This should not occur */
749 748 port->pa_npkeys = 0;
750 749 port->pa_pkey_tbl = NULL;
751 750 return;
752 751 }
753 752 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
754 753
755 754 port->pa_state = pinfop->p_linkstate;
756 755 port->pa_npkeys = pinfop->p_pkey_tbl_sz;
757 756 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc(
758 757 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
759 758
760 759 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
761 760 port->pa_pkey_tbl[pkey_idx].pt_pkey =
762 761 pinfop->p_pkey_tbl[pkey_idx];
763 762
764 763 ibt_free_portinfo(pinfop, size);
765 764
766 765 if (ibdm_enumerate_iocs) {
767 766 event_args.is_event_callback = ibdm_saa_event_cb;
768 767 event_args.is_event_callback_arg = port;
769 768 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
770 769 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
771 770 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
772 771 "sa access registration failed");
773 772 (void) ibdm_fini_port(port);
774 773 return;
775 774 }
776 775
777 776 ibmf_reg.ir_ci_guid = port->pa_hca_guid;
778 777 ibmf_reg.ir_port_num = port->pa_port_num;
779 778 ibmf_reg.ir_client_class = DEV_MGT_MANAGER;
780 779
781 780 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
782 781 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
783 782 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
784 783 "IBMF registration failed");
785 784 (void) ibdm_fini_port(port);
786 785 return;
787 786 }
788 787
789 788 if (ibmf_setup_async_cb(port->pa_ibmf_hdl,
790 789 IBMF_QP_HANDLE_DEFAULT,
791 790 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
792 791 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
793 792 "IBMF setup recv cb failed");
794 793 (void) ibdm_fini_port(port);
795 794 return;
796 795 }
797 796 } else {
798 797 port->pa_sa_hdl = NULL;
799 798 port->pa_ibmf_hdl = NULL;
800 799 }
801 800
802 801 for (ii = 0; ii < port->pa_npkeys; ii++) {
803 802 pkey = port->pa_pkey_tbl[ii].pt_pkey;
804 803 if (IBDM_INVALID_PKEY(pkey)) {
805 804 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
806 805 continue;
807 806 }
808 807 ibdm_port_attr_ibmf_init(port, pkey, ii);
809 808 }
810 809 }
811 810
812 811
813 812 /*
814 813 * ibdm_port_attr_ibmf_init:
815 814 * With IBMF - Alloc QP Handle and Setup Async callback
816 815 */
817 816 static void
818 817 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
819 818 {
820 819 int ret;
821 820
822 821 if (ibdm_enumerate_iocs == 0) {
823 822 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
824 823 return;
825 824 }
826 825
827 826 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
828 827 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
829 828 IBMF_SUCCESS) {
830 829 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
831 830 "IBMF failed to alloc qp %d", ret);
832 831 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
833 832 return;
834 833 }
835 834
836 835 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
837 836 port->pa_ibmf_hdl);
838 837
839 838 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
840 839 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
841 840 IBMF_SUCCESS) {
842 841 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
843 842 "IBMF setup recv cb failed %d", ret);
844 843 (void) ibmf_free_qp(port->pa_ibmf_hdl,
845 844 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
846 845 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
847 846 }
848 847 }
849 848
850 849
851 850 /*
852 851 * ibdm_get_port_attr()
853 852 * Get port attributes from HCA guid and port number
854 853 * Return pointer to ibdm_port_attr_t on Success
855 854 * and NULL on failure
856 855 */
857 856 static ibdm_port_attr_t *
858 857 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
859 858 {
860 859 ibdm_hca_list_t *hca_list;
861 860 ibdm_port_attr_t *port_attr;
862 861 int ii;
863 862
864 863 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
865 864 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
866 865 hca_list = ibdm.ibdm_hca_list_head;
867 866 while (hca_list) {
868 867 if (hca_list->hl_hca_guid == event->ev_hca_guid) {
869 868 for (ii = 0; ii < hca_list->hl_nports; ii++) {
870 869 port_attr = &hca_list->hl_port_attr[ii];
871 870 if (port_attr->pa_port_num == event->ev_port) {
872 871 *retval = hca_list;
873 872 return (port_attr);
874 873 }
875 874 }
876 875 }
877 876 hca_list = hca_list->hl_next;
878 877 }
879 878 return (NULL);
880 879 }
881 880
882 881
883 882 /*
884 883 * ibdm_update_port_attr()
885 884 * Update the port attributes
886 885 */
887 886 static void
888 887 ibdm_update_port_attr(ibdm_port_attr_t *port)
889 888 {
890 889 uint_t nports, size;
891 890 uint_t pkey_idx;
892 891 ibt_hca_portinfo_t *portinfop;
893 892
894 893 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
895 894 if (ibt_query_hca_ports(port->pa_hca_hdl,
896 895 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
897 896 /* This should not occur */
898 897 port->pa_npkeys = 0;
899 898 port->pa_pkey_tbl = NULL;
900 899 return;
901 900 }
902 901 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;
903 902
904 903 port->pa_state = portinfop->p_linkstate;
905 904
906 905 /*
907 906 * PKey information in portinfo valid only if port is
908 907 * ACTIVE. Bail out if not.
909 908 */
910 909 if (port->pa_state != IBT_PORT_ACTIVE) {
911 910 port->pa_npkeys = 0;
912 911 port->pa_pkey_tbl = NULL;
913 912 ibt_free_portinfo(portinfop, size);
914 913 return;
915 914 }
916 915
917 916 port->pa_npkeys = portinfop->p_pkey_tbl_sz;
918 917 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc(
919 918 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
920 919
921 920 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
922 921 port->pa_pkey_tbl[pkey_idx].pt_pkey =
923 922 portinfop->p_pkey_tbl[pkey_idx];
924 923 }
925 924 ibt_free_portinfo(portinfop, size);
926 925 }
927 926
928 927
929 928 /*
930 929 * ibdm_handle_hca_attach()
931 930 */
932 931 static void
933 932 ibdm_handle_hca_attach(ib_guid_t hca_guid)
934 933 {
935 934 uint_t size;
936 935 uint_t ii, nports;
937 936 ibt_status_t status;
938 937 ibt_hca_hdl_t hca_hdl;
939 938 ibt_hca_attr_t *hca_attr;
940 939 ibdm_hca_list_t *hca_list, *temp;
941 940 ibdm_port_attr_t *port_attr;
942 941 ibt_hca_portinfo_t *portinfop;
943 942
944 943 IBTF_DPRINTF_L4("ibdm",
945 944 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);
946 945
947 946 /* open the HCA first */
948 947 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
949 948 &hca_hdl)) != IBT_SUCCESS) {
950 949 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
951 950 "open_hca failed, status 0x%x", status);
952 951 return;
953 952 }
954 953
955 954 hca_attr = (ibt_hca_attr_t *)
956 955 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
957 956 /* ibt_query_hca always returns IBT_SUCCESS */
958 957 (void) ibt_query_hca(hca_hdl, hca_attr);
959 958
960 959 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
961 960 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
962 961 hca_attr->hca_version_id, hca_attr->hca_nports);
963 962
964 963 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
965 964 &size)) != IBT_SUCCESS) {
966 965 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
967 966 "ibt_query_hca_ports failed, status 0x%x", status);
968 967 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
969 968 (void) ibt_close_hca(hca_hdl);
970 969 return;
971 970 }
972 971 hca_list = (ibdm_hca_list_t *)
973 972 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
974 973 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
975 974 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
976 975 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
977 976 hca_list->hl_nports = hca_attr->hca_nports;
978 977 hca_list->hl_attach_time = gethrtime();
979 978 hca_list->hl_hca_hdl = hca_hdl;
980 979
981 980 /*
982 981 * Init a dummy port attribute for the HCA node
983 982 * This is for Per-HCA Node. Initialize port_attr :
984 983 * hca_guid & port_guid -> hca_guid
985 984 * npkeys, pkey_tbl is NULL
986 985 * port_num, sn_prefix is 0
987 986 * vendorid, product_id, dev_version from HCA
988 987 * pa_state is IBT_PORT_ACTIVE
989 988 */
990 989 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
991 990 sizeof (ibdm_port_attr_t), KM_SLEEP);
992 991 port_attr = hca_list->hl_hca_port_attr;
993 992 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
994 993 port_attr->pa_productid = hca_attr->hca_device_id;
995 994 port_attr->pa_dev_version = hca_attr->hca_version_id;
996 995 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
997 996 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
998 997 port_attr->pa_port_guid = hca_attr->hca_node_guid;
999 998 port_attr->pa_state = IBT_PORT_ACTIVE;
1000 999
1001 1000
1002 1001 for (ii = 0; ii < nports; ii++) {
1003 1002 port_attr = &hca_list->hl_port_attr[ii];
1004 1003 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
1005 1004 port_attr->pa_productid = hca_attr->hca_device_id;
1006 1005 port_attr->pa_dev_version = hca_attr->hca_version_id;
1007 1006 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
1008 1007 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
1009 1008 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid;
1010 1009 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix;
1011 1010 port_attr->pa_port_num = portinfop[ii].p_port_num;
1012 1011 port_attr->pa_state = portinfop[ii].p_linkstate;
1013 1012
1014 1013 /*
1015 1014 * Register with IBMF, SA access when the port is in
1016 1015 * ACTIVE state. Also register a callback routine
1017 1016 * with IBMF to receive incoming DM MAD's.
1018 1017 * The IBDM event handler takes care of registration of
1019 1018 * port which are not active.
1020 1019 */
1021 1020 IBTF_DPRINTF_L4("ibdm",
1022 1021 "\thandle_hca_attach: port guid %llx Port state 0x%x",
1023 1022 port_attr->pa_port_guid, portinfop[ii].p_linkstate);
1024 1023
1025 1024 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
1026 1025 mutex_enter(&ibdm.ibdm_hl_mutex);
1027 1026 hca_list->hl_nports_active++;
1028 1027 ibdm_initialize_port(port_attr);
1029 1028 cv_broadcast(&ibdm.ibdm_port_settle_cv);
1030 1029 mutex_exit(&ibdm.ibdm_hl_mutex);
1031 1030 }
1032 1031 }
1033 1032 mutex_enter(&ibdm.ibdm_hl_mutex);
1034 1033 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
1035 1034 if (temp->hl_hca_guid == hca_guid) {
1036 1035 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
1037 1036 "already seen by IBDM", hca_guid);
1038 1037 mutex_exit(&ibdm.ibdm_hl_mutex);
1039 1038 (void) ibdm_uninit_hca(hca_list);
1040 1039 return;
1041 1040 }
1042 1041 }
1043 1042 ibdm.ibdm_hca_count++;
1044 1043 if (ibdm.ibdm_hca_list_head == NULL) {
1045 1044 ibdm.ibdm_hca_list_head = hca_list;
1046 1045 ibdm.ibdm_hca_list_tail = hca_list;
1047 1046 } else {
1048 1047 ibdm.ibdm_hca_list_tail->hl_next = hca_list;
1049 1048 ibdm.ibdm_hca_list_tail = hca_list;
1050 1049 }
1051 1050 mutex_exit(&ibdm.ibdm_hl_mutex);
1052 1051 mutex_enter(&ibdm.ibdm_ibnex_mutex);
1053 1052 if (ibdm.ibdm_ibnex_callback != NULL) {
1054 1053 (*ibdm.ibdm_ibnex_callback)((void *)
1055 1054 &hca_guid, IBDM_EVENT_HCA_ADDED);
1056 1055 }
1057 1056 mutex_exit(&ibdm.ibdm_ibnex_mutex);
1058 1057
1059 1058 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1060 1059 ibt_free_portinfo(portinfop, size);
1061 1060 }
1062 1061
1063 1062
1064 1063 /*
1065 1064 * ibdm_handle_hca_detach()
1066 1065 */
1067 1066 static void
1068 1067 ibdm_handle_hca_detach(ib_guid_t hca_guid)
1069 1068 {
1070 1069 ibdm_hca_list_t *head, *prev = NULL;
1071 1070 size_t len;
1072 1071 ibdm_dp_gidinfo_t *gidinfo;
1073 1072 ibdm_port_attr_t *port_attr;
1074 1073 int i;
1075 1074
1076 1075 IBTF_DPRINTF_L4("ibdm",
1077 1076 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);
1078 1077
1079 1078 /* Make sure no probes are running */
1080 1079 mutex_enter(&ibdm.ibdm_mutex);
1081 1080 while (ibdm.ibdm_busy & IBDM_BUSY)
1082 1081 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1083 1082 ibdm.ibdm_busy |= IBDM_BUSY;
1084 1083 mutex_exit(&ibdm.ibdm_mutex);
1085 1084
1086 1085 mutex_enter(&ibdm.ibdm_hl_mutex);
1087 1086 head = ibdm.ibdm_hca_list_head;
1088 1087 while (head) {
1089 1088 if (head->hl_hca_guid == hca_guid) {
1090 1089 if (prev == NULL)
1091 1090 ibdm.ibdm_hca_list_head = head->hl_next;
1092 1091 else
1093 1092 prev->hl_next = head->hl_next;
1094 1093 if (ibdm.ibdm_hca_list_tail == head)
1095 1094 ibdm.ibdm_hca_list_tail = prev;
1096 1095 ibdm.ibdm_hca_count--;
1097 1096 break;
1098 1097 }
1099 1098 prev = head;
1100 1099 head = head->hl_next;
1101 1100 }
1102 1101 mutex_exit(&ibdm.ibdm_hl_mutex);
1103 1102 if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
1104 1103 (void) ibdm_handle_hca_attach(hca_guid);
1105 1104
1106 1105 #ifdef DEBUG
1107 1106 if (ibdm_enumerate_iocs == 0) {
1108 1107 ASSERT(ibdm.ibdm_dp_gidlist_head == NULL);
1109 1108 }
1110 1109 #endif
1111 1110
1112 1111 /*
1113 1112 * Now clean up the HCA lists in the gidlist.
1114 1113 */
1115 1114 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
1116 1115 gidinfo->gl_next) {
1117 1116 prev = NULL;
1118 1117 head = gidinfo->gl_hca_list;
1119 1118 while (head) {
1120 1119 if (head->hl_hca_guid == hca_guid) {
1121 1120 if (prev == NULL)
1122 1121 gidinfo->gl_hca_list =
1123 1122 head->hl_next;
1124 1123 else
1125 1124 prev->hl_next = head->hl_next;
1126 1125 for (i = 0; i < head->hl_nports; i++) {
1127 1126 port_attr = &head->hl_port_attr[i];
1128 1127 if (port_attr->pa_pkey_tbl != NULL)
1129 1128 kmem_free(
1130 1129 port_attr->pa_pkey_tbl,
1131 1130 port_attr->pa_npkeys *
1132 1131 sizeof (ibdm_pkey_tbl_t));
1133 1132 }
1134 1133 len = sizeof (ibdm_hca_list_t) +
1135 1134 (head->hl_nports *
1136 1135 sizeof (ibdm_port_attr_t));
1137 1136 kmem_free(head, len);
1138 1137
1139 1138 break;
1140 1139 }
1141 1140 prev = head;
1142 1141 head = head->hl_next;
1143 1142 }
1144 1143 }
1145 1144
1146 1145 mutex_enter(&ibdm.ibdm_mutex);
1147 1146 ibdm.ibdm_busy &= ~IBDM_BUSY;
1148 1147 cv_broadcast(&ibdm.ibdm_busy_cv);
1149 1148 mutex_exit(&ibdm.ibdm_mutex);
1150 1149 }
1151 1150
1152 1151
1153 1152 static int
1154 1153 ibdm_uninit_hca(ibdm_hca_list_t *head)
1155 1154 {
1156 1155 int ii;
1157 1156 ibdm_port_attr_t *port_attr;
1158 1157
1159 1158 for (ii = 0; ii < head->hl_nports; ii++) {
1160 1159 port_attr = &head->hl_port_attr[ii];
1161 1160 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
1162 1161 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
1163 1162 "ibdm_fini_port() failed", head, ii);
1164 1163 return (IBDM_FAILURE);
1165 1164 }
1166 1165 }
1167 1166 if (head->hl_hca_hdl)
1168 1167 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) {
1169 1168 IBTF_DPRINTF_L2("ibdm", "uninit_hca: "
1170 1169 "ibt_close_hca() failed");
1171 1170 return (IBDM_FAILURE);
1172 1171 }
1173 1172 kmem_free(head->hl_port_attr,
1174 1173 head->hl_nports * sizeof (ibdm_port_attr_t));
1175 1174 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
1176 1175 kmem_free(head, sizeof (ibdm_hca_list_t));
1177 1176 return (IBDM_SUCCESS);
1178 1177 }
1179 1178
1180 1179
1181 1180 /*
1182 1181 * For each port on the HCA,
1183 1182 * 1) Teardown IBMF receive callback function
1184 1183 * 2) Unregister with IBMF
1185 1184 * 3) Unregister with SA access
1186 1185 */
1187 1186 static int
1188 1187 ibdm_fini_port(ibdm_port_attr_t *port_attr)
1189 1188 {
1190 1189 int ii, ibmf_status;
1191 1190
1192 1191 for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
1193 1192 if (port_attr->pa_pkey_tbl == NULL)
1194 1193 break;
1195 1194 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
1196 1195 continue;
1197 1196 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
1198 1197 IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1199 1198 "ibdm_port_attr_ibmf_fini failed for "
1200 1199 "port pkey 0x%x", ii);
1201 1200 return (IBDM_FAILURE);
1202 1201 }
1203 1202 }
1204 1203
1205 1204 if (port_attr->pa_ibmf_hdl) {
1206 1205 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1207 1206 IBMF_QP_HANDLE_DEFAULT, 0);
1208 1207 if (ibmf_status != IBMF_SUCCESS) {
1209 1208 IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1210 1209 "ibmf_tear_down_async_cb failed %d", ibmf_status);
1211 1210 return (IBDM_FAILURE);
1212 1211 }
1213 1212
1214 1213 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
1215 1214 if (ibmf_status != IBMF_SUCCESS) {
1216 1215 IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1217 1216 "ibmf_unregister failed %d", ibmf_status);
1218 1217 return (IBDM_FAILURE);
1219 1218 }
1220 1219
1221 1220 port_attr->pa_ibmf_hdl = NULL;
1222 1221 }
1223 1222
1224 1223 if (port_attr->pa_sa_hdl) {
1225 1224 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
1226 1225 if (ibmf_status != IBMF_SUCCESS) {
1227 1226 IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1228 1227 "ibmf_sa_session_close failed %d", ibmf_status);
1229 1228 return (IBDM_FAILURE);
1230 1229 }
1231 1230 port_attr->pa_sa_hdl = NULL;
1232 1231 }
1233 1232
1234 1233 if (port_attr->pa_pkey_tbl != NULL) {
1235 1234 kmem_free(port_attr->pa_pkey_tbl,
1236 1235 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
1237 1236 port_attr->pa_pkey_tbl = NULL;
1238 1237 port_attr->pa_npkeys = 0;
1239 1238 }
1240 1239
1241 1240 return (IBDM_SUCCESS);
1242 1241 }
1243 1242
1244 1243
1245 1244 /*
1246 1245 * ibdm_port_attr_ibmf_fini:
1247 1246 * With IBMF - Tear down Async callback and free QP Handle
1248 1247 */
1249 1248 static int
1250 1249 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
1251 1250 {
1252 1251 int ibmf_status;
1253 1252
1254 1253 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");
1255 1254
1256 1255 if (ibdm_enumerate_iocs == 0) {
1257 1256 ASSERT(port_attr->pa_pkey_tbl[ii].pt_qp_hdl == NULL);
1258 1257 return (IBDM_SUCCESS);
1259 1258 }
1260 1259
1261 1260 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
1262 1261 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1263 1262 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1264 1263 if (ibmf_status != IBMF_SUCCESS) {
1265 1264 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1266 1265 "ibmf_tear_down_async_cb failed %d", ibmf_status);
1267 1266 return (IBDM_FAILURE);
1268 1267 }
1269 1268 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
1270 1269 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1271 1270 if (ibmf_status != IBMF_SUCCESS) {
1272 1271 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1273 1272 "ibmf_free_qp failed %d", ibmf_status);
1274 1273 return (IBDM_FAILURE);
1275 1274 }
1276 1275 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
1277 1276 }
1278 1277 return (IBDM_SUCCESS);
1279 1278 }
1280 1279
1281 1280
1282 1281 /*
1283 1282 * ibdm_gid_decr_pending:
1284 1283 * decrement gl_pending_cmds. If zero wakeup sleeping threads
1285 1284 */
1286 1285 static void
1287 1286 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
1288 1287 {
1289 1288 mutex_enter(&ibdm.ibdm_mutex);
1290 1289 mutex_enter(&gidinfo->gl_mutex);
1291 1290 if (--gidinfo->gl_pending_cmds == 0) {
1292 1291 /*
1293 1292 * Handle DGID getting removed.
1294 1293 */
1295 1294 if (gidinfo->gl_disconnected) {
1296 1295 mutex_exit(&gidinfo->gl_mutex);
1297 1296 mutex_exit(&ibdm.ibdm_mutex);
1298 1297
1299 1298 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
1300 1299 "gidinfo %p hot removal", gidinfo);
1301 1300 ibdm_delete_gidinfo(gidinfo);
1302 1301
1303 1302 mutex_enter(&ibdm.ibdm_mutex);
1304 1303 ibdm.ibdm_ngid_probes_in_progress--;
1305 1304 ibdm_wait_probe_completion();
1306 1305 mutex_exit(&ibdm.ibdm_mutex);
1307 1306 return;
1308 1307 }
1309 1308 mutex_exit(&gidinfo->gl_mutex);
1310 1309 mutex_exit(&ibdm.ibdm_mutex);
1311 1310 ibdm_notify_newgid_iocs(gidinfo);
1312 1311 mutex_enter(&ibdm.ibdm_mutex);
1313 1312 mutex_enter(&gidinfo->gl_mutex);
1314 1313
1315 1314 ibdm.ibdm_ngid_probes_in_progress--;
1316 1315 ibdm_wait_probe_completion();
1317 1316 }
1318 1317 mutex_exit(&gidinfo->gl_mutex);
1319 1318 mutex_exit(&ibdm.ibdm_mutex);
1320 1319 }
1321 1320
1322 1321
1323 1322 /*
1324 1323 * ibdm_wait_probe_completion:
1325 1324 * wait for probing to complete
1326 1325 */
1327 1326 static void
1328 1327 ibdm_wait_probe_completion(void)
1329 1328 {
1330 1329 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1331 1330 if (ibdm.ibdm_ngid_probes_in_progress) {
1332 1331 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete");
1333 1332 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
1334 1333 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
1335 1334 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
1336 1335 }
1337 1336 }
1338 1337
1339 1338
1340 1339 /*
1341 1340 * ibdm_wait_cisco_probe_completion:
1342 1341 * wait for the reply from the Cisco FC GW switch after a setclassportinfo
1343 1342 * request is sent. This wait can be achieved on each gid.
1344 1343 */
1345 1344 static void
1346 1345 ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo)
1347 1346 {
1348 1347 ASSERT(MUTEX_HELD(&gidinfo->gl_mutex));
1349 1348 IBTF_DPRINTF_L4("ibdm", "\twait for cisco probe complete");
1350 1349 gidinfo->gl_flag |= IBDM_CISCO_PROBE;
1351 1350 while (gidinfo->gl_flag & IBDM_CISCO_PROBE)
1352 1351 cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex);
1353 1352 }
1354 1353
1355 1354
1356 1355 /*
1357 1356 * ibdm_wakeup_probe_gid_cv:
1358 1357 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
1359 1358 */
1360 1359 static void
1361 1360 ibdm_wakeup_probe_gid_cv(void)
1362 1361 {
1363 1362 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1364 1363 if (!ibdm.ibdm_ngid_probes_in_progress) {
1365 1364 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
1366 1365 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
1367 1366 cv_broadcast(&ibdm.ibdm_probe_cv);
1368 1367 }
1369 1368
1370 1369 }
1371 1370
1372 1371
1373 1372 /*
1374 1373 * ibdm_sweep_fabric(reprobe_flag)
1375 1374 * Find all possible Managed IOU's and their IOC's that are visible
1376 1375 * to the host. The algorithm used is as follows
1377 1376 *
1378 1377 * Send a "bus walk" request for each port on the host HCA to SA access
1379 1378 * SA returns complete set of GID's that are reachable from
1380 1379 * source port. This is done in parallel.
1381 1380 *
1382 1381 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE
1383 1382 *
1384 1383 * Sort the GID list and eliminate duplicate GID's
1385 1384 * 1) Use DGID for sorting
1386 1385 * 2) use PortGuid for sorting
1387 1386 * Send SA query to retrieve NodeRecord and
1388 1387 * extract PortGuid from that.
1389 1388 *
1390 1389 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
1391 1390 * support DM MAD's
1392 1391 * Send a "Portinfo" query to get the port capabilities and
1393 1392 * then check for DM MAD's support
1394 1393 *
1395 1394 * Send "ClassPortInfo" request for all the GID's in parallel,
1396 1395 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
1397 1396 * cv_signal to complete.
1398 1397 *
1399 1398 * When DM agent on the remote GID sends back the response, IBMF
1400 1399 * invokes DM callback routine.
1401 1400 *
1402 1401 * If the response is proper, send "IOUnitInfo" request and set
1403 1402 * GID state to IBDM_GET_IOUNITINFO.
1404 1403 *
1405 1404 * If the response is proper, send "IocProfileInfo" request to
1406 1405 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
1407 1406 *
1408 1407 * Send request to get Service entries simultaneously
1409 1408 *
1410 1409 * Signal the waiting thread when received response for all the commands.
1411 1410 *
1412 1411 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error
1413 1412 * response during the probing period.
1414 1413 *
1415 1414 * Note:
1416 1415 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
1417 1416 * keep track of number commands in progress at any point of time.
1418 1417 * MAD transaction ID is used to identify a particular GID
1419 1418 * TBD: Consider registering the IBMF receive callback on demand
1420 1419 *
1421 1420 * Note: This routine must be called with ibdm.ibdm_mutex held
1422 1421 * TBD: Re probe the failure GID (for certain failures) when requested
1423 1422 * for fabric sweep next time
1424 1423 *
1425 1424 * Parameters : If reprobe_flag is set, All IOCs will be reprobed.
1426 1425 */
1427 1426 static void
1428 1427 ibdm_sweep_fabric(int reprobe_flag)
1429 1428 {
1430 1429 int ii;
1431 1430 int new_paths = 0;
1432 1431 uint8_t niocs;
1433 1432 taskqid_t tid;
1434 1433 ibdm_ioc_info_t *ioc;
1435 1434 ibdm_hca_list_t *hca_list = NULL;
1436 1435 ibdm_port_attr_t *port = NULL;
1437 1436 ibdm_dp_gidinfo_t *gid_info;
1438 1437
1439 1438 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
1440 1439 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1441 1440
1442 1441 /*
1443 1442 * Check whether a sweep already in progress. If so, just
1444 1443 * wait for the fabric sweep to complete
1445 1444 */
1446 1445 while (ibdm.ibdm_busy & IBDM_BUSY)
1447 1446 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1448 1447 ibdm.ibdm_busy |= IBDM_BUSY;
1449 1448 mutex_exit(&ibdm.ibdm_mutex);
1450 1449
1451 1450 ibdm_dump_sweep_fabric_timestamp(0);
1452 1451
1453 1452 /* Rescan the GID list for any removed GIDs for reprobe */
1454 1453 if (reprobe_flag)
1455 1454 ibdm_rescan_gidlist(NULL);
1456 1455
1457 1456 /*
1458 1457 * Get list of all the ports reachable from the local known HCA
1459 1458 * ports which are active
1460 1459 */
1461 1460 mutex_enter(&ibdm.ibdm_hl_mutex);
1462 1461 for (ibdm_get_next_port(&hca_list, &port, 1); port;
1463 1462 ibdm_get_next_port(&hca_list, &port, 1)) {
1464 1463 /*
1465 1464 * Get PATHS to all the reachable ports from
1466 1465 * SGID and update the global ibdm structure.
1467 1466 */
1468 1467 new_paths = ibdm_get_reachable_ports(port, hca_list);
1469 1468 ibdm.ibdm_ngids += new_paths;
1470 1469 }
1471 1470 mutex_exit(&ibdm.ibdm_hl_mutex);
1472 1471
1473 1472 mutex_enter(&ibdm.ibdm_mutex);
1474 1473 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
1475 1474 mutex_exit(&ibdm.ibdm_mutex);
1476 1475
1477 1476 /* Send a request to probe GIDs asynchronously. */
1478 1477 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1479 1478 gid_info = gid_info->gl_next) {
1480 1479 mutex_enter(&gid_info->gl_mutex);
1481 1480 gid_info->gl_reprobe_flag = reprobe_flag;
1482 1481 mutex_exit(&gid_info->gl_mutex);
1483 1482
1484 1483 /* process newly encountered GIDs */
1485 1484 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
1486 1485 (void *)gid_info, TQ_NOSLEEP);
1487 1486 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
1488 1487 " taskq_id = %x", gid_info, tid);
1489 1488 /* taskq failed to dispatch call it directly */
1490 1489 if (tid == NULL)
1491 1490 ibdm_probe_gid_thread((void *)gid_info);
1492 1491 }
1493 1492
1494 1493 mutex_enter(&ibdm.ibdm_mutex);
1495 1494 ibdm_wait_probe_completion();
1496 1495
1497 1496 /*
1498 1497 * Update the properties, if reprobe_flag is set
1499 1498 * Skip if gl_reprobe_flag is set, this will be
1500 1499 * a re-inserted / new GID, for which notifications
1501 1500 * have already been send.
1502 1501 */
1503 1502 if (reprobe_flag) {
1504 1503 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1505 1504 gid_info = gid_info->gl_next) {
1506 1505 if (gid_info->gl_iou == NULL)
1507 1506 continue;
1508 1507 if (gid_info->gl_reprobe_flag) {
1509 1508 gid_info->gl_reprobe_flag = 0;
1510 1509 continue;
1511 1510 }
1512 1511
1513 1512 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1514 1513 for (ii = 0; ii < niocs; ii++) {
1515 1514 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1516 1515 if (ioc)
1517 1516 ibdm_reprobe_update_port_srv(ioc,
1518 1517 gid_info);
1519 1518 }
1520 1519 }
1521 1520 } else if (ibdm.ibdm_prev_iou) {
1522 1521 ibdm_ioc_info_t *ioc_list;
1523 1522
1524 1523 /*
1525 1524 * Get the list of IOCs which have changed.
1526 1525 * If any IOCs have changed, Notify IBNexus
1527 1526 */
1528 1527 ibdm.ibdm_prev_iou = 0;
1529 1528 ioc_list = ibdm_handle_prev_iou();
1530 1529 if (ioc_list) {
1531 1530 if (ibdm.ibdm_ibnex_callback != NULL) {
1532 1531 (*ibdm.ibdm_ibnex_callback)(
1533 1532 (void *)ioc_list,
1534 1533 IBDM_EVENT_IOC_PROP_UPDATE);
1535 1534 }
1536 1535 }
1537 1536 }
1538 1537
1539 1538 ibdm_dump_sweep_fabric_timestamp(1);
1540 1539
1541 1540 ibdm.ibdm_busy &= ~IBDM_BUSY;
1542 1541 cv_broadcast(&ibdm.ibdm_busy_cv);
1543 1542 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
1544 1543 }
1545 1544
1546 1545
1547 1546 /*
1548 1547 * ibdm_is_cisco:
1549 1548 * Check if this is a Cisco device or not.
1550 1549 */
1551 1550 static boolean_t
1552 1551 ibdm_is_cisco(ib_guid_t guid)
1553 1552 {
1554 1553 if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID)
1555 1554 return (B_TRUE);
1556 1555 return (B_FALSE);
1557 1556 }
1558 1557
1559 1558
1560 1559 /*
1561 1560 * ibdm_is_cisco_switch:
1562 1561 * Check if this switch is a CISCO switch or not.
1563 1562 * Note that if this switch is already activated, ibdm_is_cisco_switch()
1564 1563 * returns B_FALSE not to re-activate it again.
1565 1564 */
1566 1565 static boolean_t
1567 1566 ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info)
1568 1567 {
1569 1568 int company_id, device_id;
1570 1569 ASSERT(gid_info != 0);
1571 1570 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
1572 1571
1573 1572 /*
1574 1573 * If this switch is already activated, don't re-activate it.
1575 1574 */
1576 1575 if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE)
1577 1576 return (B_FALSE);
1578 1577
1579 1578 /*
1580 1579 * Check if this switch is a Cisco FC GW or not.
1581 1580 * Use the node guid (the OUI part) instead of the vendor id
1582 1581 * since the vendor id is zero in practice.
1583 1582 */
1584 1583 company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT;
1585 1584 device_id = gid_info->gl_devid;
1586 1585
1587 1586 if (company_id == IBDM_CISCO_COMPANY_ID &&
1588 1587 device_id == IBDM_CISCO_DEVICE_ID)
1589 1588 return (B_TRUE);
1590 1589 return (B_FALSE);
1591 1590 }
1592 1591
1593 1592
1594 1593 /*
1595 1594 * ibdm_probe_gid_thread:
1596 1595 * thread that does the actual work for sweeping the fabric
1597 1596 * for a given GID
1598 1597 */
1599 1598 static void
1600 1599 ibdm_probe_gid_thread(void *args)
1601 1600 {
1602 1601 int reprobe_flag;
1603 1602 ib_guid_t node_guid;
1604 1603 ib_guid_t port_guid;
1605 1604 ibdm_dp_gidinfo_t *gid_info;
1606 1605
1607 1606 gid_info = (ibdm_dp_gidinfo_t *)args;
1608 1607 reprobe_flag = gid_info->gl_reprobe_flag;
1609 1608 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
1610 1609 gid_info, reprobe_flag);
1611 1610 ASSERT(gid_info != NULL);
1612 1611 ASSERT(gid_info->gl_pending_cmds == 0);
1613 1612
1614 1613 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
1615 1614 reprobe_flag == 0) {
1616 1615 /*
1617 1616 * This GID may have been already probed. Send
1618 1617 * in a CLP to check if IOUnitInfo changed?
1619 1618 * Explicitly set gl_reprobe_flag to 0 so that
1620 1619 * IBnex is not notified on completion
1621 1620 */
1622 1621 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
1623 1622 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
1624 1623 "get new IOCs information");
1625 1624 mutex_enter(&gid_info->gl_mutex);
1626 1625 gid_info->gl_pending_cmds++;
1627 1626 gid_info->gl_state = IBDM_GET_IOUNITINFO;
1628 1627 gid_info->gl_reprobe_flag = 0;
1629 1628 mutex_exit(&gid_info->gl_mutex);
1630 1629 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
1631 1630 mutex_enter(&gid_info->gl_mutex);
1632 1631 --gid_info->gl_pending_cmds;
1633 1632 mutex_exit(&gid_info->gl_mutex);
1634 1633 mutex_enter(&ibdm.ibdm_mutex);
1635 1634 --ibdm.ibdm_ngid_probes_in_progress;
1636 1635 ibdm_wakeup_probe_gid_cv();
1637 1636 mutex_exit(&ibdm.ibdm_mutex);
1638 1637 }
1639 1638 } else {
1640 1639 mutex_enter(&ibdm.ibdm_mutex);
1641 1640 --ibdm.ibdm_ngid_probes_in_progress;
1642 1641 ibdm_wakeup_probe_gid_cv();
1643 1642 mutex_exit(&ibdm.ibdm_mutex);
1644 1643 }
1645 1644 return;
1646 1645 } else if (reprobe_flag && gid_info->gl_state ==
1647 1646 IBDM_GID_PROBING_COMPLETE) {
1648 1647 /*
1649 1648 * Reprobe all IOCs for the GID which has completed
1650 1649 * probe. Skip other port GIDs to same IOU.
1651 1650 * Explicitly set gl_reprobe_flag to 0 so that
1652 1651 * IBnex is not notified on completion
1653 1652 */
1654 1653 ibdm_ioc_info_t *ioc_info;
1655 1654 uint8_t niocs, ii;
1656 1655
1657 1656 ASSERT(gid_info->gl_iou);
1658 1657 mutex_enter(&gid_info->gl_mutex);
1659 1658 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1660 1659 gid_info->gl_state = IBDM_GET_IOC_DETAILS;
1661 1660 gid_info->gl_pending_cmds += niocs;
1662 1661 gid_info->gl_reprobe_flag = 0;
1663 1662 mutex_exit(&gid_info->gl_mutex);
1664 1663 for (ii = 0; ii < niocs; ii++) {
1665 1664 uchar_t slot_info;
1666 1665 ib_dm_io_unitinfo_t *giou_info;
1667 1666
1668 1667 /*
1669 1668 * Check whether IOC is present in the slot
1670 1669 * Series of nibbles (in the field
1671 1670 * iou_ctrl_list) represents a slot in the
1672 1671 * IOU.
1673 1672 * Byte format: 76543210
1674 1673 * Bits 0-3 of first byte represent Slot 2
1675 1674 * bits 4-7 of first byte represent slot 1,
1676 1675 * bits 0-3 of second byte represent slot 4
1677 1676 * and so on
1678 1677 * Each 4-bit nibble has the following meaning
1679 1678 * 0x0 : IOC not installed
1680 1679 * 0x1 : IOC is present
1681 1680 * 0xf : Slot does not exist
1682 1681 * and all other values are reserved.
1683 1682 */
1684 1683 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1685 1684 giou_info = &gid_info->gl_iou->iou_info;
1686 1685 slot_info = giou_info->iou_ctrl_list[(ii/2)];
1687 1686 if ((ii % 2) == 0)
1688 1687 slot_info = (slot_info >> 4);
1689 1688
1690 1689 if ((slot_info & 0xf) != 1) {
1691 1690 ioc_info->ioc_state =
1692 1691 IBDM_IOC_STATE_PROBE_FAILED;
1693 1692 ibdm_gid_decr_pending(gid_info);
1694 1693 continue;
1695 1694 }
1696 1695
1697 1696 if (ibdm_send_ioc_profile(gid_info, ii) !=
1698 1697 IBDM_SUCCESS) {
1699 1698 ibdm_gid_decr_pending(gid_info);
1700 1699 }
1701 1700 }
1702 1701
1703 1702 return;
1704 1703 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
1705 1704 mutex_enter(&ibdm.ibdm_mutex);
1706 1705 --ibdm.ibdm_ngid_probes_in_progress;
1707 1706 ibdm_wakeup_probe_gid_cv();
1708 1707 mutex_exit(&ibdm.ibdm_mutex);
1709 1708 return;
1710 1709 }
1711 1710
1712 1711 /*
1713 1712 * Check whether the destination GID supports DM agents. If
1714 1713 * not, stop probing the GID and continue with the next GID
1715 1714 * in the list.
1716 1715 */
1717 1716 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
1718 1717 mutex_enter(&gid_info->gl_mutex);
1719 1718 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1720 1719 gid_info->gl_is_dm_capable = B_FALSE;
1721 1720 mutex_exit(&gid_info->gl_mutex);
1722 1721 ibdm_delete_glhca_list(gid_info);
1723 1722 mutex_enter(&ibdm.ibdm_mutex);
1724 1723 --ibdm.ibdm_ngid_probes_in_progress;
1725 1724 ibdm_wakeup_probe_gid_cv();
1726 1725 mutex_exit(&ibdm.ibdm_mutex);
1727 1726 return;
1728 1727 }
1729 1728
1730 1729 /*
1731 1730 * This GID is Device management capable
1732 1731 */
1733 1732 mutex_enter(&gid_info->gl_mutex);
1734 1733 gid_info->gl_is_dm_capable = B_TRUE;
1735 1734 mutex_exit(&gid_info->gl_mutex);
1736 1735
1737 1736 /* Get the nodeguid and portguid of the port */
1738 1737 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
1739 1738 &node_guid, &port_guid) != IBDM_SUCCESS) {
1740 1739 mutex_enter(&gid_info->gl_mutex);
1741 1740 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1742 1741 mutex_exit(&gid_info->gl_mutex);
1743 1742 ibdm_delete_glhca_list(gid_info);
1744 1743 mutex_enter(&ibdm.ibdm_mutex);
1745 1744 --ibdm.ibdm_ngid_probes_in_progress;
1746 1745 ibdm_wakeup_probe_gid_cv();
1747 1746 mutex_exit(&ibdm.ibdm_mutex);
1748 1747 return;
1749 1748 }
1750 1749
1751 1750 /*
1752 1751 * Check whether we already knew about this NodeGuid
1753 1752 * If so, do not probe the GID and continue with the
1754 1753 * next GID in the gid list. Set the GID state to
1755 1754 * probing done.
1756 1755 */
1757 1756 mutex_enter(&ibdm.ibdm_mutex);
1758 1757 gid_info->gl_nodeguid = node_guid;
1759 1758 gid_info->gl_portguid = port_guid;
1760 1759 if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
1761 1760 mutex_exit(&ibdm.ibdm_mutex);
1762 1761 mutex_enter(&gid_info->gl_mutex);
1763 1762 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
1764 1763 mutex_exit(&gid_info->gl_mutex);
1765 1764 ibdm_delete_glhca_list(gid_info);
1766 1765 mutex_enter(&ibdm.ibdm_mutex);
1767 1766 --ibdm.ibdm_ngid_probes_in_progress;
1768 1767 ibdm_wakeup_probe_gid_cv();
1769 1768 mutex_exit(&ibdm.ibdm_mutex);
1770 1769 return;
1771 1770 }
1772 1771 ibdm_add_to_gl_gid(gid_info, gid_info);
1773 1772 mutex_exit(&ibdm.ibdm_mutex);
1774 1773
1775 1774 /*
1776 1775 * New or reinserted GID : Enable notification to IBnex
1777 1776 */
1778 1777 mutex_enter(&gid_info->gl_mutex);
1779 1778 gid_info->gl_reprobe_flag = 1;
1780 1779
1781 1780 /*
1782 1781 * A Cisco FC GW needs the special handling to get IOUnitInfo.
1783 1782 */
1784 1783 if (ibdm_is_cisco_switch(gid_info)) {
1785 1784 gid_info->gl_pending_cmds++;
1786 1785 gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
1787 1786 mutex_exit(&gid_info->gl_mutex);
1788 1787
1789 1788 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
1790 1789 mutex_enter(&gid_info->gl_mutex);
1791 1790 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1792 1791 --gid_info->gl_pending_cmds;
1793 1792 mutex_exit(&gid_info->gl_mutex);
1794 1793
1795 1794 /* free the hca_list on this gid_info */
1796 1795 ibdm_delete_glhca_list(gid_info);
1797 1796
1798 1797 mutex_enter(&ibdm.ibdm_mutex);
1799 1798 --ibdm.ibdm_ngid_probes_in_progress;
1800 1799 ibdm_wakeup_probe_gid_cv();
1801 1800 mutex_exit(&ibdm.ibdm_mutex);
1802 1801
1803 1802 return;
1804 1803 }
1805 1804
1806 1805 mutex_enter(&gid_info->gl_mutex);
1807 1806 ibdm_wait_cisco_probe_completion(gid_info);
1808 1807
1809 1808 IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: "
1810 1809 "CISCO Wakeup signal received");
1811 1810 }
1812 1811
1813 1812 /* move on to the 'GET_CLASSPORTINFO' stage */
1814 1813 gid_info->gl_pending_cmds++;
1815 1814 gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
1816 1815 mutex_exit(&gid_info->gl_mutex);
1817 1816
1818 1817 IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: "
1819 1818 "%d: gid_info %p gl_state %d pending_cmds %d",
1820 1819 __LINE__, gid_info, gid_info->gl_state,
1821 1820 gid_info->gl_pending_cmds);
1822 1821
1823 1822 /*
1824 1823 * Send ClassPortInfo request to the GID asynchronously.
1825 1824 */
1826 1825 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
1827 1826
1828 1827 mutex_enter(&gid_info->gl_mutex);
1829 1828 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1830 1829 --gid_info->gl_pending_cmds;
1831 1830 mutex_exit(&gid_info->gl_mutex);
1832 1831
1833 1832 /* free the hca_list on this gid_info */
1834 1833 ibdm_delete_glhca_list(gid_info);
1835 1834
1836 1835 mutex_enter(&ibdm.ibdm_mutex);
1837 1836 --ibdm.ibdm_ngid_probes_in_progress;
1838 1837 ibdm_wakeup_probe_gid_cv();
1839 1838 mutex_exit(&ibdm.ibdm_mutex);
1840 1839
1841 1840 return;
1842 1841 }
1843 1842 }
1844 1843
1845 1844
1846 1845 /*
1847 1846 * ibdm_check_dest_nodeguid
1848 1847 * Searches for the NodeGuid in the GID list
1849 1848 * Returns matching gid_info if found and otherwise NULL
1850 1849 *
1851 1850 * This function is called to handle new GIDs discovered
1852 1851 * during device sweep / probe or for GID_AVAILABLE event.
1853 1852 *
1854 1853 * Parameter :
1855 1854 * gid_info GID to check
1856 1855 */
1857 1856 static ibdm_dp_gidinfo_t *
1858 1857 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
1859 1858 {
1860 1859 ibdm_dp_gidinfo_t *gid_list;
1861 1860 ibdm_gid_t *tmp;
1862 1861
1863 1862 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");
1864 1863
1865 1864 gid_list = ibdm.ibdm_dp_gidlist_head;
1866 1865 while (gid_list) {
1867 1866 if ((gid_list != gid_info) &&
1868 1867 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
1869 1868 IBTF_DPRINTF_L4("ibdm",
1870 1869 "\tcheck_dest_nodeguid: NodeGuid is present");
1871 1870
1872 1871 /* Add to gid_list */
1873 1872 tmp = kmem_zalloc(sizeof (ibdm_gid_t),
1874 1873 KM_SLEEP);
1875 1874 tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
1876 1875 tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
1877 1876 tmp->gid_next = gid_list->gl_gid;
1878 1877 gid_list->gl_gid = tmp;
1879 1878 gid_list->gl_ngids++;
1880 1879 return (gid_list);
1881 1880 }
1882 1881
1883 1882 gid_list = gid_list->gl_next;
1884 1883 }
1885 1884
1886 1885 return (NULL);
1887 1886 }
1888 1887
1889 1888
1890 1889 /*
1891 1890 * ibdm_is_dev_mgt_supported
1892 1891 * Get the PortInfo attribute (SA Query)
1893 1892 * Check "CompatabilityMask" field in the Portinfo.
1894 1893 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
1895 1894 * by the port, otherwise IBDM_FAILURE
1896 1895 */
1897 1896 static int
1898 1897 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
1899 1898 {
1900 1899 int ret;
1901 1900 size_t length = 0;
1902 1901 sa_portinfo_record_t req, *resp = NULL;
1903 1902 ibmf_saa_access_args_t qargs;
1904 1903
1905 1904 bzero(&req, sizeof (sa_portinfo_record_t));
1906 1905 req.EndportLID = gid_info->gl_dlid;
1907 1906
1908 1907 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID;
1909 1908 qargs.sq_access_type = IBMF_SAA_RETRIEVE;
1910 1909 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
1911 1910 qargs.sq_template = &req;
1912 1911 qargs.sq_callback = NULL;
1913 1912 qargs.sq_callback_arg = NULL;
1914 1913
1915 1914 ret = ibmf_sa_access(gid_info->gl_sa_hdl,
1916 1915 &qargs, 0, &length, (void **)&resp);
1917 1916
1918 1917 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1919 1918 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
1920 1919 "failed to get PORTINFO attribute %d", ret);
1921 1920 return (IBDM_FAILURE);
1922 1921 }
1923 1922
1924 1923 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
1925 1924 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
1926 1925 ret = IBDM_SUCCESS;
1927 1926 } else {
1928 1927 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
1929 1928 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
1930 1929 ret = IBDM_FAILURE;
1931 1930 }
1932 1931 kmem_free(resp, length);
1933 1932 return (ret);
1934 1933 }
1935 1934
1936 1935
1937 1936 /*
1938 1937 * ibdm_get_node_port_guids()
1939 1938 * Get the NodeInfoRecord of the port
1940 1939 * Save NodeGuid and PortGUID values in the GID list structure.
1941 1940 * Return IBDM_SUCCESS/IBDM_FAILURE
1942 1941 */
1943 1942 static int
1944 1943 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
1945 1944 ib_guid_t *node_guid, ib_guid_t *port_guid)
1946 1945 {
1947 1946 int ret;
1948 1947 size_t length = 0;
1949 1948 sa_node_record_t req, *resp = NULL;
1950 1949 ibmf_saa_access_args_t qargs;
1951 1950
1952 1951 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");
1953 1952
1954 1953 bzero(&req, sizeof (sa_node_record_t));
1955 1954 req.LID = dlid;
1956 1955
1957 1956 qargs.sq_attr_id = SA_NODERECORD_ATTRID;
1958 1957 qargs.sq_access_type = IBMF_SAA_RETRIEVE;
1959 1958 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
1960 1959 qargs.sq_template = &req;
1961 1960 qargs.sq_callback = NULL;
1962 1961 qargs.sq_callback_arg = NULL;
1963 1962
1964 1963 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
1965 1964 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1966 1965 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
1967 1966 " SA Retrieve Failed: %d", ret);
1968 1967 return (IBDM_FAILURE);
1969 1968 }
1970 1969 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
1971 1970 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);
1972 1971
1973 1972 *node_guid = resp->NodeInfo.NodeGUID;
1974 1973 *port_guid = resp->NodeInfo.PortGUID;
1975 1974 kmem_free(resp, length);
1976 1975 return (IBDM_SUCCESS);
1977 1976 }
1978 1977
1979 1978
1980 1979 /*
1981 1980 * ibdm_get_reachable_ports()
1982 1981 * Get list of the destination GID (and its path records) by
1983 1982 * querying the SA access.
1984 1983 *
1985 1984 * Returns Number paths
1986 1985 */
1987 1986 static int
1988 1987 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
1989 1988 {
1990 1989 uint_t ii, jj, nrecs;
1991 1990 uint_t npaths = 0;
1992 1991 size_t length;
1993 1992 ib_gid_t sgid;
1994 1993 ibdm_pkey_tbl_t *pkey_tbl;
1995 1994 sa_path_record_t *result;
1996 1995 sa_path_record_t *precp;
1997 1996 ibdm_dp_gidinfo_t *gid_info;
1998 1997
1999 1998 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
2000 1999 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);
2001 2000
2002 2001 sgid.gid_prefix = portinfo->pa_sn_prefix;
2003 2002 sgid.gid_guid = portinfo->pa_port_guid;
2004 2003
2005 2004 /* get reversible paths */
2006 2005 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
2007 2006 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
2008 2007 != IBMF_SUCCESS) {
2009 2008 IBTF_DPRINTF_L2("ibdm",
2010 2009 "\tget_reachable_ports: Getting path records failed");
2011 2010 return (0);
2012 2011 }
2013 2012
2014 2013 for (ii = 0; ii < nrecs; ii++) {
2015 2014 sa_node_record_t *nrec;
2016 2015 size_t length;
2017 2016
2018 2017 precp = &result[ii];
2019 2018 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
2020 2019 precp->DGID.gid_prefix)) != NULL) {
2021 2020 IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: "
2022 2021 "Already exists nrecs %d, ii %d", nrecs, ii);
2023 2022 ibdm_addto_glhcalist(gid_info, hca);
2024 2023 continue;
2025 2024 }
2026 2025 /*
2027 2026 * This is a new GID. Allocate a GID structure and
2028 2027 * initialize the structure
2029 2028 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
2030 2029 * by kmem_zalloc call
2031 2030 */
2032 2031 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
2033 2032 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
2034 2033 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
2035 2034 gid_info->gl_dgid_hi = precp->DGID.gid_prefix;
2036 2035 gid_info->gl_dgid_lo = precp->DGID.gid_guid;
2037 2036 gid_info->gl_sgid_hi = precp->SGID.gid_prefix;
2038 2037 gid_info->gl_sgid_lo = precp->SGID.gid_guid;
2039 2038 gid_info->gl_p_key = precp->P_Key;
2040 2039 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl;
2041 2040 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl;
2042 2041 gid_info->gl_slid = precp->SLID;
2043 2042 gid_info->gl_dlid = precp->DLID;
2044 2043 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID)
2045 2044 << IBDM_GID_TRANSACTIONID_SHIFT;
2046 2045 gid_info->gl_min_transactionID = gid_info->gl_transactionID;
2047 2046 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1)
2048 2047 << IBDM_GID_TRANSACTIONID_SHIFT;
2049 2048 gid_info->gl_SL = precp->SL;
2050 2049
2051 2050 /*
2052 2051 * get the node record with this guid if the destination
2053 2052 * device is a Cisco one.
2054 2053 */
2055 2054 if (ibdm_is_cisco(precp->DGID.gid_guid) &&
2056 2055 (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) &&
2057 2056 ibdm_get_node_record_by_port(portinfo->pa_sa_hdl,
2058 2057 precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) {
2059 2058 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
2060 2059 gid_info->gl_devid = nrec->NodeInfo.DeviceID;
2061 2060 kmem_free(nrec, length);
2062 2061 }
2063 2062
2064 2063 ibdm_addto_glhcalist(gid_info, hca);
2065 2064
2066 2065 ibdm_dump_path_info(precp);
2067 2066
2068 2067 gid_info->gl_qp_hdl = NULL;
2069 2068 ASSERT(portinfo->pa_pkey_tbl != NULL &&
2070 2069 portinfo->pa_npkeys != 0);
2071 2070
2072 2071 for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
2073 2072 pkey_tbl = &portinfo->pa_pkey_tbl[jj];
2074 2073 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
2075 2074 (pkey_tbl->pt_qp_hdl != NULL)) {
2076 2075 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
2077 2076 break;
2078 2077 }
2079 2078 }
2080 2079
2081 2080 /*
2082 2081 * QP handle for GID not initialized. No matching Pkey
2083 2082 * was found!! ibdm should *not* hit this case. Flag an
2084 2083 * error and drop the GID if ibdm does encounter this.
2085 2084 */
2086 2085 if (gid_info->gl_qp_hdl == NULL) {
2087 2086 IBTF_DPRINTF_L2(ibdm_string,
2088 2087 "\tget_reachable_ports: No matching Pkey");
2089 2088 ibdm_delete_gidinfo(gid_info);
2090 2089 continue;
2091 2090 }
2092 2091 if (ibdm.ibdm_dp_gidlist_head == NULL) {
2093 2092 ibdm.ibdm_dp_gidlist_head = gid_info;
2094 2093 ibdm.ibdm_dp_gidlist_tail = gid_info;
2095 2094 } else {
2096 2095 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
2097 2096 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
2098 2097 ibdm.ibdm_dp_gidlist_tail = gid_info;
2099 2098 }
2100 2099 npaths++;
2101 2100 }
2102 2101 kmem_free(result, length);
2103 2102 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
2104 2103 return (npaths);
2105 2104 }
2106 2105
2107 2106
2108 2107 /*
2109 2108 * ibdm_check_dgid()
2110 2109 * Look in the global list to check whether we know this DGID already
2111 2110 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
2112 2111 */
2113 2112 static ibdm_dp_gidinfo_t *
2114 2113 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
2115 2114 {
2116 2115 ibdm_dp_gidinfo_t *gid_list;
2117 2116
2118 2117 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2119 2118 gid_list = gid_list->gl_next) {
2120 2119 if ((guid == gid_list->gl_dgid_lo) &&
2121 2120 (prefix == gid_list->gl_dgid_hi)) {
2122 2121 break;
2123 2122 }
2124 2123 }
2125 2124 return (gid_list);
2126 2125 }
2127 2126
2128 2127
2129 2128 /*
2130 2129 * ibdm_find_gid()
2131 2130 * Look in the global list to find a GID entry with matching
2132 2131 * port & node GUID.
2133 2132 * Return pointer to gidinfo if found, else return NULL
2134 2133 */
2135 2134 static ibdm_dp_gidinfo_t *
2136 2135 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
2137 2136 {
2138 2137 ibdm_dp_gidinfo_t *gid_list;
2139 2138
2140 2139 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
2141 2140 nodeguid, portguid);
2142 2141
2143 2142 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2144 2143 gid_list = gid_list->gl_next) {
2145 2144 if ((portguid == gid_list->gl_portguid) &&
2146 2145 (nodeguid == gid_list->gl_nodeguid)) {
2147 2146 break;
2148 2147 }
2149 2148 }
2150 2149
2151 2150 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
2152 2151 gid_list);
2153 2152 return (gid_list);
2154 2153 }
2155 2154
2156 2155
2157 2156 /*
2158 2157 * ibdm_set_classportinfo()
2159 2158 * ibdm_set_classportinfo() is a function to activate a Cisco FC GW
2160 2159 * by sending the setClassPortInfo request with the trapLID, trapGID
2161 2160 * and etc. to the gateway since the gateway doesn't provide the IO
2162 2161 * Unit Information othewise. This behavior is the Cisco specific one,
2163 2162 * and this function is called to a Cisco FC GW only.
2164 2163 * Returns IBDM_SUCCESS/IBDM_FAILURE
2165 2164 */
2166 2165 static int
2167 2166 ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2168 2167 {
2169 2168 ibmf_msg_t *msg;
2170 2169 ib_mad_hdr_t *hdr;
2171 2170 ibdm_timeout_cb_args_t *cb_args;
2172 2171 void *data;
2173 2172 ib_mad_classportinfo_t *cpi;
2174 2173
2175 2174 IBTF_DPRINTF_L4("ibdm",
2176 2175 "\tset_classportinfo: gid info 0x%p", gid_info);
2177 2176
2178 2177 /*
2179 2178 * Send command to set classportinfo attribute. Allocate a IBMF
2180 2179 * packet and initialize the packet.
2181 2180 */
2182 2181 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2183 2182 &msg) != IBMF_SUCCESS) {
2184 2183 IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail");
2185 2184 return (IBDM_FAILURE);
2186 2185 }
2187 2186
2188 2187 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2189 2188 ibdm_alloc_send_buffers(msg);
2190 2189 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2191 2190
2192 2191 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2193 2192 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2194 2193 msg->im_local_addr.ia_remote_qno = 1;
2195 2194 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2196 2195 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2197 2196 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2198 2197
2199 2198 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2200 2199 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2201 2200 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2202 2201 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2203 2202 hdr->R_Method = IB_DM_DEVMGT_METHOD_SET;
2204 2203 hdr->Status = 0;
2205 2204 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2206 2205 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
2207 2206 hdr->AttributeModifier = 0;
2208 2207
2209 2208 data = msg->im_msgbufs_send.im_bufs_cl_data;
2210 2209 cpi = (ib_mad_classportinfo_t *)data;
2211 2210
2212 2211 /*
2213 2212 * Set the classportinfo values to activate this Cisco FC GW.
2214 2213 */
2215 2214 cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi);
2216 2215 cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo);
2217 2216 cpi->TrapLID = h2b16(gid_info->gl_slid);
2218 2217 cpi->TrapSL = gid_info->gl_SL;
2219 2218 cpi->TrapP_Key = h2b16(gid_info->gl_p_key);
2220 2219 cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn));
2221 2220 cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *)
2222 2221 gid_info->gl_qp_hdl)->isq_qkey));
2223 2222
2224 2223 cb_args = &gid_info->gl_cpi_cb_args;
2225 2224 cb_args->cb_gid_info = gid_info;
2226 2225 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2227 2226 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2228 2227
2229 2228 mutex_enter(&gid_info->gl_mutex);
2230 2229 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2231 2230 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2232 2231 mutex_exit(&gid_info->gl_mutex);
2233 2232
2234 2233 IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: "
2235 2234 "timeout id %x", gid_info->gl_timeout_id);
2236 2235
2237 2236 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2238 2237 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2239 2238 IBTF_DPRINTF_L2("ibdm",
2240 2239 "\tset_classportinfo: ibmf send failed");
2241 2240 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2242 2241 }
2243 2242
2244 2243 return (IBDM_SUCCESS);
2245 2244 }
2246 2245
2247 2246
2248 2247 /*
2249 2248 * ibdm_send_classportinfo()
2250 2249 * Send classportinfo request. When the request is completed
2251 2250 * IBMF calls ibdm_classportinfo_cb routine to inform about
2252 2251 * the completion.
2253 2252 * Returns IBDM_SUCCESS/IBDM_FAILURE
2254 2253 */
2255 2254 static int
2256 2255 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2257 2256 {
2258 2257 ibmf_msg_t *msg;
2259 2258 ib_mad_hdr_t *hdr;
2260 2259 ibdm_timeout_cb_args_t *cb_args;
2261 2260
2262 2261 IBTF_DPRINTF_L4("ibdm",
2263 2262 "\tsend_classportinfo: gid info 0x%p", gid_info);
2264 2263
2265 2264 /*
2266 2265 * Send command to get classportinfo attribute. Allocate a IBMF
2267 2266 * packet and initialize the packet.
2268 2267 */
2269 2268 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2270 2269 &msg) != IBMF_SUCCESS) {
2271 2270 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
2272 2271 return (IBDM_FAILURE);
2273 2272 }
2274 2273
2275 2274 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2276 2275 ibdm_alloc_send_buffers(msg);
2277 2276 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2278 2277
2279 2278 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2280 2279 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2281 2280 msg->im_local_addr.ia_remote_qno = 1;
2282 2281 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2283 2282 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2284 2283 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2285 2284
2286 2285 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2287 2286 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2288 2287 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2289 2288 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2290 2289 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2291 2290 hdr->Status = 0;
2292 2291 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2293 2292 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
2294 2293 hdr->AttributeModifier = 0;
2295 2294
2296 2295 cb_args = &gid_info->gl_cpi_cb_args;
2297 2296 cb_args->cb_gid_info = gid_info;
2298 2297 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2299 2298 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2300 2299
2301 2300 mutex_enter(&gid_info->gl_mutex);
2302 2301 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2303 2302 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2304 2303 mutex_exit(&gid_info->gl_mutex);
2305 2304
2306 2305 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
2307 2306 "timeout id %x", gid_info->gl_timeout_id);
2308 2307
2309 2308 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2310 2309 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2311 2310 IBTF_DPRINTF_L2("ibdm",
2312 2311 "\tsend_classportinfo: ibmf send failed");
2313 2312 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2314 2313 }
2315 2314
2316 2315 return (IBDM_SUCCESS);
2317 2316 }
2318 2317
2319 2318
2320 2319 /*
2321 2320 * ibdm_handle_setclassportinfo()
2322 2321 * Invoked by the IBMF when setClassPortInfo request is completed.
2323 2322 */
2324 2323 static void
2325 2324 ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl,
2326 2325 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2327 2326 {
2328 2327 void *data;
2329 2328 timeout_id_t timeout_id;
2330 2329 ib_mad_classportinfo_t *cpi;
2331 2330
2332 2331 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl "
2333 2332 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2334 2333
2335 2334 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2336 2335 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: "
2337 2336 "Not a ClassPortInfo resp");
2338 2337 *flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2339 2338 return;
2340 2339 }
2341 2340
2342 2341 /*
2343 2342 * Verify whether timeout handler is created/active.
2344 2343 * If created/ active, cancel the timeout handler
2345 2344 */
2346 2345 mutex_enter(&gid_info->gl_mutex);
2347 2346 if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) {
2348 2347 IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp");
2349 2348 *flag |= IBDM_IBMF_PKT_DUP_RESP;
2350 2349 mutex_exit(&gid_info->gl_mutex);
2351 2350 return;
2352 2351 }
2353 2352 ibdm_bump_transactionID(gid_info);
2354 2353
2355 2354 gid_info->gl_iou_cb_args.cb_req_type = 0;
2356 2355 if (gid_info->gl_timeout_id) {
2357 2356 timeout_id = gid_info->gl_timeout_id;
2358 2357 mutex_exit(&gid_info->gl_mutex);
2359 2358 IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: "
2360 2359 "gl_timeout_id = 0x%x", timeout_id);
2361 2360 if (untimeout(timeout_id) == -1) {
2362 2361 IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: "
2363 2362 "untimeout gl_timeout_id failed");
2364 2363 }
2365 2364 mutex_enter(&gid_info->gl_mutex);
2366 2365 gid_info->gl_timeout_id = 0;
2367 2366 }
2368 2367 mutex_exit(&gid_info->gl_mutex);
2369 2368
2370 2369 data = msg->im_msgbufs_recv.im_bufs_cl_data;
2371 2370 cpi = (ib_mad_classportinfo_t *)data;
2372 2371
2373 2372 ibdm_dump_classportinfo(cpi);
2374 2373 }
2375 2374
2376 2375
2377 2376 /*
2378 2377 * ibdm_handle_classportinfo()
2379 2378 * Invoked by the IBMF when the classportinfo request is completed.
2380 2379 */
2381 2380 static void
2382 2381 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
2383 2382 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2384 2383 {
2385 2384 void *data;
2386 2385 timeout_id_t timeout_id;
2387 2386 ib_mad_hdr_t *hdr;
2388 2387 ib_mad_classportinfo_t *cpi;
2389 2388
2390 2389 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
2391 2390 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2392 2391
2393 2392 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2394 2393 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
2395 2394 "Not a ClassPortInfo resp");
2396 2395 *flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2397 2396 return;
2398 2397 }
2399 2398
2400 2399 /*
2401 2400 * Verify whether timeout handler is created/active.
2402 2401 * If created/ active, cancel the timeout handler
2403 2402 */
2404 2403 mutex_enter(&gid_info->gl_mutex);
2405 2404 ibdm_bump_transactionID(gid_info);
2406 2405 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
2407 2406 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
2408 2407 *flag |= IBDM_IBMF_PKT_DUP_RESP;
2409 2408 mutex_exit(&gid_info->gl_mutex);
2410 2409 return;
2411 2410 }
2412 2411 gid_info->gl_iou_cb_args.cb_req_type = 0;
2413 2412 if (gid_info->gl_timeout_id) {
2414 2413 timeout_id = gid_info->gl_timeout_id;
2415 2414 mutex_exit(&gid_info->gl_mutex);
2416 2415 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
2417 2416 "gl_timeout_id = 0x%x", timeout_id);
2418 2417 if (untimeout(timeout_id) == -1) {
2419 2418 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
2420 2419 "untimeout gl_timeout_id failed");
2421 2420 }
2422 2421 mutex_enter(&gid_info->gl_mutex);
2423 2422 gid_info->gl_timeout_id = 0;
2424 2423 }
2425 2424 gid_info->gl_state = IBDM_GET_IOUNITINFO;
2426 2425 gid_info->gl_pending_cmds++;
2427 2426 mutex_exit(&gid_info->gl_mutex);
2428 2427
2429 2428 data = msg->im_msgbufs_recv.im_bufs_cl_data;
2430 2429 cpi = (ib_mad_classportinfo_t *)data;
2431 2430
2432 2431 /*
2433 2432 * Cache the "RespTimeValue" and redirection information in the
2434 2433 * global gid list data structure. This cached information will
2435 2434 * be used to send any further requests to the GID.
2436 2435 */
2437 2436 gid_info->gl_resp_timeout =
2438 2437 (b2h32(cpi->RespTimeValue) & 0x1F);
2439 2438
2440 2439 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) &
2441 2440 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
2442 2441 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID);
2443 2442 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff);
2444 2443 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key);
2445 2444 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key);
2446 2445 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi);
2447 2446 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo);
2448 2447 gid_info->gl_redirectSL = cpi->RedirectSL;
2449 2448
2450 2449 ibdm_dump_classportinfo(cpi);
2451 2450
2452 2451 /*
2453 2452 * Send IOUnitInfo request
2454 2453 * Reuse previously allocated IBMF packet for sending ClassPortInfo
2455 2454 * Check whether DM agent on the remote node requested redirection
2456 2455 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
2457 2456 */
2458 2457 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2459 2458 ibdm_alloc_send_buffers(msg);
2460 2459 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2461 2460 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2462 2461 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2463 2462
2464 2463 if (gid_info->gl_redirected == B_TRUE) {
2465 2464 if (gid_info->gl_redirect_dlid != 0) {
2466 2465 msg->im_local_addr.ia_remote_lid =
2467 2466 gid_info->gl_redirect_dlid;
2468 2467 }
2469 2468 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
2470 2469 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
2471 2470 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
2472 2471 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
2473 2472 } else {
2474 2473 msg->im_local_addr.ia_remote_qno = 1;
2475 2474 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2476 2475 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2477 2476 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2478 2477 }
2479 2478
2480 2479 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2481 2480 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2482 2481 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2483 2482 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2484 2483 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2485 2484 hdr->Status = 0;
2486 2485 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2487 2486 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
2488 2487 hdr->AttributeModifier = 0;
2489 2488
2490 2489 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2491 2490 gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2492 2491 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2493 2492
2494 2493 mutex_enter(&gid_info->gl_mutex);
2495 2494 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2496 2495 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2497 2496 mutex_exit(&gid_info->gl_mutex);
2498 2497
2499 2498 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
2500 2499 "timeout %x", gid_info->gl_timeout_id);
2501 2500
2502 2501 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
2503 2502 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
2504 2503 IBTF_DPRINTF_L2("ibdm",
2505 2504 "\thandle_classportinfo: msg transport failed");
2506 2505 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
2507 2506 }
2508 2507 (*flag) |= IBDM_IBMF_PKT_REUSED;
2509 2508 }
2510 2509
2511 2510
2512 2511 /*
2513 2512 * ibdm_send_iounitinfo:
2514 2513 * Sends a DM request to get IOU unitinfo.
2515 2514 */
2516 2515 static int
2517 2516 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
2518 2517 {
2519 2518 ibmf_msg_t *msg;
2520 2519 ib_mad_hdr_t *hdr;
2521 2520
2522 2521 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);
2523 2522
2524 2523 /*
2525 2524 * Send command to get iounitinfo attribute. Allocate a IBMF
2526 2525 * packet and initialize the packet.
2527 2526 */
2528 2527 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
2529 2528 IBMF_SUCCESS) {
2530 2529 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
2531 2530 return (IBDM_FAILURE);
2532 2531 }
2533 2532
2534 2533 mutex_enter(&gid_info->gl_mutex);
2535 2534 ibdm_bump_transactionID(gid_info);
2536 2535 mutex_exit(&gid_info->gl_mutex);
2537 2536
2538 2537
2539 2538 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2540 2539 ibdm_alloc_send_buffers(msg);
2541 2540 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2542 2541 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2543 2542 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2544 2543 msg->im_local_addr.ia_remote_qno = 1;
2545 2544 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2546 2545 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2547 2546 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2548 2547
2549 2548 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2550 2549 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2551 2550 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2552 2551 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2553 2552 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2554 2553 hdr->Status = 0;
2555 2554 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2556 2555 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
2557 2556 hdr->AttributeModifier = 0;
2558 2557
2559 2558 gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2560 2559 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2561 2560 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2562 2561
2563 2562 mutex_enter(&gid_info->gl_mutex);
2564 2563 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2565 2564 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2566 2565 mutex_exit(&gid_info->gl_mutex);
2567 2566
2568 2567 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
2569 2568 "timeout %x", gid_info->gl_timeout_id);
2570 2569
2571 2570 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
2572 2571 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
2573 2572 IBMF_SUCCESS) {
2574 2573 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
2575 2574 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
2576 2575 msg, &gid_info->gl_iou_cb_args);
2577 2576 }
2578 2577 return (IBDM_SUCCESS);
2579 2578 }
2580 2579
2581 2580 /*
2582 2581 * ibdm_handle_iounitinfo()
2583 2582 * Invoked by the IBMF when IO Unitinfo request is completed.
2584 2583 */
2585 2584 static void
2586 2585 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
2587 2586 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2588 2587 {
2589 2588 int ii, first = B_TRUE;
2590 2589 int num_iocs;
2591 2590 size_t size;
2592 2591 uchar_t slot_info;
2593 2592 timeout_id_t timeout_id;
2594 2593 ib_mad_hdr_t *hdr;
2595 2594 ibdm_ioc_info_t *ioc_info;
2596 2595 ib_dm_io_unitinfo_t *iou_info;
2597 2596 ib_dm_io_unitinfo_t *giou_info;
2598 2597 ibdm_timeout_cb_args_t *cb_args;
2599 2598
2600 2599 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
2601 2600 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);
2602 2601
2603 2602 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
2604 2603 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
2605 2604 "Unexpected response");
2606 2605 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2607 2606 return;
2608 2607 }
2609 2608
2610 2609 mutex_enter(&gid_info->gl_mutex);
2611 2610 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
2612 2611 IBTF_DPRINTF_L4("ibdm",
2613 2612 "\thandle_iounitinfo: DUP resp");
2614 2613 mutex_exit(&gid_info->gl_mutex);
2615 2614 (*flag) = IBDM_IBMF_PKT_DUP_RESP;
2616 2615 return;
2617 2616 }
2618 2617 gid_info->gl_iou_cb_args.cb_req_type = 0;
2619 2618 if (gid_info->gl_timeout_id) {
2620 2619 timeout_id = gid_info->gl_timeout_id;
2621 2620 mutex_exit(&gid_info->gl_mutex);
2622 2621 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
2623 2622 "gl_timeout_id = 0x%x", timeout_id);
2624 2623 if (untimeout(timeout_id) == -1) {
2625 2624 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
2626 2625 "untimeout gl_timeout_id failed");
2627 2626 }
2628 2627 mutex_enter(&gid_info->gl_mutex);
2629 2628 gid_info->gl_timeout_id = 0;
2630 2629 }
2631 2630 gid_info->gl_state = IBDM_GET_IOC_DETAILS;
2632 2631
2633 2632 iou_info = IBDM_IN_IBMFMSG2IOU(msg);
2634 2633 ibdm_dump_iounitinfo(iou_info);
2635 2634 num_iocs = iou_info->iou_num_ctrl_slots;
2636 2635 /*
2637 2636 * check if number of IOCs reported is zero? if yes, return.
2638 2637 * when num_iocs are reported zero internal IOC database needs
2639 2638 * to be updated. To ensure that save the number of IOCs in
2640 2639 * the new field "gl_num_iocs". Use a new field instead of
2641 2640 * "giou_info->iou_num_ctrl_slots" as that would prevent
2642 2641 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
2643 2642 */
2644 2643 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
2645 2644 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
2646 2645 mutex_exit(&gid_info->gl_mutex);
2647 2646 return;
2648 2647 }
2649 2648 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);
2650 2649
2651 2650 /*
2652 2651 * if there is an existing gl_iou (IOU has been probed before)
2653 2652 * check if the "iou_changeid" is same as saved entry in
2654 2653 * "giou_info->iou_changeid".
2655 2654 * (note: this logic can prevent IOC enumeration if a given
2656 2655 * vendor doesn't support setting iou_changeid field for its IOU)
2657 2656 *
2658 2657 * if there is an existing gl_iou and iou_changeid has changed :
2659 2658 * free up existing gl_iou info and its related structures.
2660 2659 * reallocate gl_iou info all over again.
2661 2660 * if we donot free this up; then this leads to memory leaks
2662 2661 */
2663 2662 if (gid_info->gl_iou) {
2664 2663 giou_info = &gid_info->gl_iou->iou_info;
2665 2664 if (b2h16(iou_info->iou_changeid) ==
2666 2665 giou_info->iou_changeid) {
2667 2666 IBTF_DPRINTF_L3("ibdm",
2668 2667 "\thandle_iounitinfo: no IOCs changed");
2669 2668 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2670 2669 mutex_exit(&gid_info->gl_mutex);
2671 2670 return;
2672 2671 }
2673 2672
2674 2673 /*
2675 2674 * Store the iou info as prev_iou to be used after
2676 2675 * sweep is done.
2677 2676 */
2678 2677 ASSERT(gid_info->gl_prev_iou == NULL);
2679 2678 IBTF_DPRINTF_L4(ibdm_string,
2680 2679 "\thandle_iounitinfo: setting gl_prev_iou %p",
2681 2680 gid_info->gl_prev_iou);
2682 2681 gid_info->gl_prev_iou = gid_info->gl_iou;
2683 2682 ibdm.ibdm_prev_iou = 1;
2684 2683 gid_info->gl_iou = NULL;
2685 2684 }
2686 2685
2687 2686 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
2688 2687 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
2689 2688 giou_info = &gid_info->gl_iou->iou_info;
2690 2689 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
2691 2690 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));
2692 2691
2693 2692 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs;
2694 2693 giou_info->iou_flag = iou_info->iou_flag;
2695 2694 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
2696 2695 giou_info->iou_changeid = b2h16(iou_info->iou_changeid);
2697 2696 gid_info->gl_pending_cmds++; /* for diag code */
2698 2697 mutex_exit(&gid_info->gl_mutex);
2699 2698
2700 2699 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
2701 2700 mutex_enter(&gid_info->gl_mutex);
2702 2701 gid_info->gl_pending_cmds--;
2703 2702 mutex_exit(&gid_info->gl_mutex);
2704 2703 }
2705 2704 /*
2706 2705 * Parallelize getting IOC controller profiles from here.
2707 2706 * Allocate IBMF packets and send commands to get IOC profile for
2708 2707 * each IOC present on the IOU.
2709 2708 */
2710 2709 for (ii = 0; ii < num_iocs; ii++) {
2711 2710 /*
2712 2711 * Check whether IOC is present in the slot
2713 2712 * Series of nibbles (in the field iou_ctrl_list) represents
2714 2713 * a slot in the IOU.
2715 2714 * Byte format: 76543210
2716 2715 * Bits 0-3 of first byte represent Slot 2
2717 2716 * bits 4-7 of first byte represent slot 1,
2718 2717 * bits 0-3 of second byte represent slot 4 and so on
2719 2718 * Each 4-bit nibble has the following meaning
2720 2719 * 0x0 : IOC not installed
2721 2720 * 0x1 : IOC is present
2722 2721 * 0xf : Slot does not exist
2723 2722 * and all other values are reserved.
2724 2723 */
2725 2724 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
2726 2725 slot_info = giou_info->iou_ctrl_list[(ii/2)];
2727 2726 if ((ii % 2) == 0)
2728 2727 slot_info = (slot_info >> 4);
2729 2728
2730 2729 if ((slot_info & 0xf) != 1) {
2731 2730 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2732 2731 "No IOC is present in the slot = %d", ii);
2733 2732 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
2734 2733 continue;
2735 2734 }
2736 2735
2737 2736 mutex_enter(&gid_info->gl_mutex);
2738 2737 ibdm_bump_transactionID(gid_info);
2739 2738 mutex_exit(&gid_info->gl_mutex);
2740 2739
2741 2740 /*
2742 2741 * Re use the already allocated packet (for IOUnitinfo) to
2743 2742 * send the first IOC controller attribute. Allocate new
2744 2743 * IBMF packets for the rest of the IOC's
2745 2744 */
2746 2745 if (first != B_TRUE) {
2747 2746 msg = NULL;
2748 2747 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2749 2748 &msg) != IBMF_SUCCESS) {
2750 2749 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2751 2750 "IBMF packet allocation failed");
2752 2751 continue;
2753 2752 }
2754 2753
2755 2754 }
2756 2755
2757 2756 /* allocate send buffers for all messages */
2758 2757 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2759 2758 ibdm_alloc_send_buffers(msg);
2760 2759 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2761 2760
2762 2761 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2763 2762 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2764 2763 if (gid_info->gl_redirected == B_TRUE) {
2765 2764 if (gid_info->gl_redirect_dlid != 0) {
2766 2765 msg->im_local_addr.ia_remote_lid =
2767 2766 gid_info->gl_redirect_dlid;
2768 2767 }
2769 2768 msg->im_local_addr.ia_remote_qno =
2770 2769 gid_info->gl_redirect_QP;
2771 2770 msg->im_local_addr.ia_p_key =
2772 2771 gid_info->gl_redirect_pkey;
2773 2772 msg->im_local_addr.ia_q_key =
2774 2773 gid_info->gl_redirect_qkey;
2775 2774 msg->im_local_addr.ia_service_level =
2776 2775 gid_info->gl_redirectSL;
2777 2776 } else {
2778 2777 msg->im_local_addr.ia_remote_qno = 1;
2779 2778 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2780 2779 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2781 2780 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2782 2781 }
2783 2782
2784 2783 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
2785 2784 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
2786 2785 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
2787 2786 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
2788 2787 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
2789 2788 hdr->Status = 0;
2790 2789 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
2791 2790 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
2792 2791 hdr->AttributeModifier = h2b32(ii + 1);
2793 2792
2794 2793 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID;
2795 2794 cb_args = &ioc_info->ioc_cb_args;
2796 2795 cb_args->cb_gid_info = gid_info;
2797 2796 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
2798 2797 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO;
2799 2798 cb_args->cb_ioc_num = ii;
2800 2799
2801 2800 mutex_enter(&gid_info->gl_mutex);
2802 2801 gid_info->gl_pending_cmds++; /* for diag code */
2803 2802
2804 2803 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2805 2804 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2806 2805 mutex_exit(&gid_info->gl_mutex);
2807 2806
2808 2807 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
2809 2808 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);
2810 2809
2811 2810 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2812 2811 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2813 2812 IBTF_DPRINTF_L2("ibdm",
2814 2813 "\thandle_iounitinfo: msg transport failed");
2815 2814 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2816 2815 }
2817 2816 (*flag) |= IBDM_IBMF_PKT_REUSED;
2818 2817 first = B_FALSE;
2819 2818 gid_info->gl_iou->iou_niocs_probe_in_progress++;
2820 2819 }
2821 2820 }
2822 2821
2823 2822
2824 2823 /*
2825 2824 * ibdm_handle_ioc_profile()
2826 2825 * Invoked by the IBMF when the IOCControllerProfile request
2827 2826 * gets completed
2828 2827 */
2829 2828 static void
2830 2829 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
2831 2830 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2832 2831 {
2833 2832 int first = B_TRUE, reprobe = 0;
2834 2833 uint_t ii, ioc_no, srv_start;
2835 2834 uint_t nserv_entries;
2836 2835 timeout_id_t timeout_id;
2837 2836 ib_mad_hdr_t *hdr;
2838 2837 ibdm_ioc_info_t *ioc_info;
2839 2838 ibdm_timeout_cb_args_t *cb_args;
2840 2839 ib_dm_ioc_ctrl_profile_t *ioc, *gioc;
2841 2840
2842 2841 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2843 2842 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2844 2843
2845 2844 ioc = IBDM_IN_IBMFMSG2IOC(msg);
2846 2845 /*
2847 2846 * Check whether we know this IOC already
2848 2847 * This will return NULL if reprobe is in progress
2849 2848 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
2850 2849 * Do not hold mutexes here.
2851 2850 */
2852 2851 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
2853 2852 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2854 2853 "IOC guid %llx is present", ioc->ioc_guid);
2855 2854 return;
2856 2855 }
2857 2856 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2858 2857 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);
2859 2858
2860 2859 /* Make sure that IOC index is with the valid range */
2861 2860 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
2862 2861 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
2863 2862 "IOC index Out of range, index %d", ioc);
2864 2863 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2865 2864 return;
2866 2865 }
2867 2866 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
2868 2867 ioc_info->ioc_iou_info = gid_info->gl_iou;
2869 2868
2870 2869 mutex_enter(&gid_info->gl_mutex);
2871 2870 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
2872 2871 reprobe = 1;
2873 2872 ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
2874 2873 ioc_info->ioc_serv = NULL;
2875 2874 ioc_info->ioc_prev_serv_cnt =
2876 2875 ioc_info->ioc_profile.ioc_service_entries;
2877 2876 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
2878 2877 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
2879 2878 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
2880 2879 mutex_exit(&gid_info->gl_mutex);
2881 2880 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2882 2881 return;
2883 2882 }
2884 2883 ioc_info->ioc_cb_args.cb_req_type = 0;
2885 2884 if (ioc_info->ioc_timeout_id) {
2886 2885 timeout_id = ioc_info->ioc_timeout_id;
2887 2886 ioc_info->ioc_timeout_id = 0;
2888 2887 mutex_exit(&gid_info->gl_mutex);
2889 2888 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
2890 2889 "ioc_timeout_id = 0x%x", timeout_id);
2891 2890 if (untimeout(timeout_id) == -1) {
2892 2891 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
2893 2892 "untimeout ioc_timeout_id failed");
2894 2893 }
2895 2894 mutex_enter(&gid_info->gl_mutex);
2896 2895 }
2897 2896
2898 2897 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
2899 2898 if (reprobe == 0) {
2900 2899 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
2901 2900 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
2902 2901 }
2903 2902
2904 2903 /*
2905 2904 * Save all the IOC information in the global structures.
2906 2905 * Note the wire format is Big Endian and the Sparc process also
2907 2906 * big endian. So, there is no need to convert the data fields
2908 2907 * The conversion routines used below are ineffective on Sparc
2909 2908 * machines where as they will be effective on little endian
2910 2909 * machines such as Intel processors.
2911 2910 */
2912 2911 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;
2913 2912
2914 2913 /*
2915 2914 * Restrict updates to onlyport GIDs and service entries during reprobe
2916 2915 */
2917 2916 if (reprobe == 0) {
2918 2917 gioc->ioc_guid = b2h64(ioc->ioc_guid);
2919 2918 gioc->ioc_vendorid =
2920 2919 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
2921 2920 >> IB_DM_VENDORID_SHIFT);
2922 2921 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid);
2923 2922 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver);
2924 2923 gioc->ioc_subsys_vendorid =
2925 2924 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
2926 2925 >> IB_DM_VENDORID_SHIFT);
2927 2926 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id);
2928 2927 gioc->ioc_io_class = b2h16(ioc->ioc_io_class);
2929 2928 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass);
2930 2929 gioc->ioc_protocol = b2h16(ioc->ioc_protocol);
2931 2930 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver);
2932 2931 gioc->ioc_send_msg_qdepth =
2933 2932 b2h16(ioc->ioc_send_msg_qdepth);
2934 2933 gioc->ioc_rdma_read_qdepth =
2935 2934 b2h16(ioc->ioc_rdma_read_qdepth);
2936 2935 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz);
2937 2936 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz);
2938 2937 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask;
2939 2938 bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
2940 2939 IB_DM_IOC_ID_STRING_LEN);
2941 2940
2942 2941 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
2943 2942 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
2944 2943 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
2945 2944 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;
2946 2945
2947 2946 if (ioc_info->ioc_diagdeviceid == B_TRUE) {
2948 2947 gid_info->gl_pending_cmds++;
2949 2948 IBTF_DPRINTF_L3(ibdm_string,
2950 2949 "\tibdm_handle_ioc_profile: "
2951 2950 "%d: gid_info %p gl_state %d pending_cmds %d",
2952 2951 __LINE__, gid_info, gid_info->gl_state,
2953 2952 gid_info->gl_pending_cmds);
2954 2953 }
2955 2954 }
2956 2955 gioc->ioc_service_entries = ioc->ioc_service_entries;
2957 2956 mutex_exit(&gid_info->gl_mutex);
2958 2957
2959 2958 ibdm_dump_ioc_profile(gioc);
2960 2959
2961 2960 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
2962 2961 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
2963 2962 mutex_enter(&gid_info->gl_mutex);
2964 2963 gid_info->gl_pending_cmds--;
2965 2964 mutex_exit(&gid_info->gl_mutex);
2966 2965 }
2967 2966 }
2968 2967 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
2969 2968 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
2970 2969 KM_SLEEP);
2971 2970
2972 2971 /*
2973 2972 * In one single request, maximum number of requests that can be
2974 2973 * obtained is 4. If number of service entries are more than four,
2975 2974 * calculate number requests needed and send them parallelly.
2976 2975 */
2977 2976 nserv_entries = ioc->ioc_service_entries;
2978 2977 ii = 0;
2979 2978 while (nserv_entries) {
2980 2979 mutex_enter(&gid_info->gl_mutex);
2981 2980 gid_info->gl_pending_cmds++;
2982 2981 ibdm_bump_transactionID(gid_info);
2983 2982 mutex_exit(&gid_info->gl_mutex);
2984 2983
2985 2984 if (first != B_TRUE) {
2986 2985 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2987 2986 &msg) != IBMF_SUCCESS) {
2988 2987 continue;
2989 2988 }
2990 2989
2991 2990 }
2992 2991 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2993 2992 ibdm_alloc_send_buffers(msg);
2994 2993 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2995 2994 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
2996 2995 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
2997 2996 if (gid_info->gl_redirected == B_TRUE) {
2998 2997 if (gid_info->gl_redirect_dlid != 0) {
2999 2998 msg->im_local_addr.ia_remote_lid =
3000 2999 gid_info->gl_redirect_dlid;
3001 3000 }
3002 3001 msg->im_local_addr.ia_remote_qno =
3003 3002 gid_info->gl_redirect_QP;
3004 3003 msg->im_local_addr.ia_p_key =
3005 3004 gid_info->gl_redirect_pkey;
3006 3005 msg->im_local_addr.ia_q_key =
3007 3006 gid_info->gl_redirect_qkey;
3008 3007 msg->im_local_addr.ia_service_level =
3009 3008 gid_info->gl_redirectSL;
3010 3009 } else {
3011 3010 msg->im_local_addr.ia_remote_qno = 1;
3012 3011 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3013 3012 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3014 3013 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3015 3014 }
3016 3015
3017 3016 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3018 3017 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3019 3018 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3020 3019 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3021 3020 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3022 3021 hdr->Status = 0;
3023 3022 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3024 3023 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
3025 3024
3026 3025 srv_start = ii * 4;
3027 3026 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
3028 3027 cb_args->cb_gid_info = gid_info;
3029 3028 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
3030 3029 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS;
3031 3030 cb_args->cb_srvents_start = srv_start;
3032 3031 cb_args->cb_ioc_num = ioc_no - 1;
3033 3032
3034 3033 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
3035 3034 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
3036 3035 cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
3037 3036 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
3038 3037 } else {
3039 3038 cb_args->cb_srvents_end =
3040 3039 (cb_args->cb_srvents_start + nserv_entries - 1);
3041 3040 nserv_entries = 0;
3042 3041 }
3043 3042 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3044 3043 ibdm_fill_srv_attr_mod(hdr, cb_args);
3045 3044 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3046 3045
3047 3046 mutex_enter(&gid_info->gl_mutex);
3048 3047 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
3049 3048 ibdm_pkt_timeout_hdlr, cb_args,
3050 3049 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3051 3050 mutex_exit(&gid_info->gl_mutex);
3052 3051
3053 3052 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
3054 3053 "timeout %x, ioc %d srv %d",
3055 3054 ioc_info->ioc_serv[srv_start].se_timeout_id,
3056 3055 ioc_no - 1, srv_start);
3057 3056
3058 3057 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
3059 3058 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3060 3059 IBTF_DPRINTF_L2("ibdm",
3061 3060 "\thandle_ioc_profile: msg send failed");
3062 3061 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
3063 3062 }
3064 3063 (*flag) |= IBDM_IBMF_PKT_REUSED;
3065 3064 first = B_FALSE;
3066 3065 ii++;
3067 3066 }
3068 3067 }
3069 3068
3070 3069
3071 3070 /*
3072 3071 * ibdm_handle_srventry_mad()
3073 3072 */
3074 3073 static void
3075 3074 ibdm_handle_srventry_mad(ibmf_msg_t *msg,
3076 3075 ibdm_dp_gidinfo_t *gid_info, int *flag)
3077 3076 {
3078 3077 uint_t ii, ioc_no, attrmod;
3079 3078 uint_t nentries, start, end;
3080 3079 timeout_id_t timeout_id;
3081 3080 ib_dm_srv_t *srv_ents;
3082 3081 ibdm_ioc_info_t *ioc_info;
3083 3082 ibdm_srvents_info_t *gsrv_ents;
3084 3083
3085 3084 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
3086 3085 " IBMF msg %p gid info %p", msg, gid_info);
3087 3086
3088 3087 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
3089 3088 /*
3090 3089 * Get the start and end index of the service entries
3091 3090 * Upper 16 bits identify the IOC
3092 3091 * Lower 16 bits specify the range of service entries
3093 3092 * LSB specifies (Big endian) end of the range
3094 3093 * MSB specifies (Big endian) start of the range
3095 3094 */
3096 3095 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3097 3096 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK);
3098 3097 end = ((attrmod >> 8) & IBDM_8_BIT_MASK);
3099 3098 start = (attrmod & IBDM_8_BIT_MASK);
3100 3099
3101 3100 /* Make sure that IOC index is with the valid range */
3102 3101 if ((ioc_no < 1) |
3103 3102 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
3104 3103 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3105 3104 "IOC index Out of range, index %d", ioc_no);
3106 3105 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3107 3106 return;
3108 3107 }
3109 3108 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3110 3109
3111 3110 /*
3112 3111 * Make sure that the "start" and "end" service indexes are
3113 3112 * with in the valid range
3114 3113 */
3115 3114 nentries = ioc_info->ioc_profile.ioc_service_entries;
3116 3115 if ((start > end) | (start >= nentries) | (end >= nentries)) {
3117 3116 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3118 3117 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
3119 3118 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3120 3119 return;
3121 3120 }
3122 3121 gsrv_ents = &ioc_info->ioc_serv[start];
3123 3122 mutex_enter(&gid_info->gl_mutex);
3124 3123 if (gsrv_ents->se_state != IBDM_SE_INVALID) {
3125 3124 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3126 3125 "already known, ioc %d, srv %d, se_state %x",
3127 3126 ioc_no - 1, start, gsrv_ents->se_state);
3128 3127 mutex_exit(&gid_info->gl_mutex);
3129 3128 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3130 3129 return;
3131 3130 }
3132 3131 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
3133 3132 if (ioc_info->ioc_serv[start].se_timeout_id) {
3134 3133 IBTF_DPRINTF_L2("ibdm",
3135 3134 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
3136 3135 timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
3137 3136 ioc_info->ioc_serv[start].se_timeout_id = 0;
3138 3137 mutex_exit(&gid_info->gl_mutex);
3139 3138 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
3140 3139 "se_timeout_id = 0x%x", timeout_id);
3141 3140 if (untimeout(timeout_id) == -1) {
3142 3141 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
3143 3142 "untimeout se_timeout_id failed");
3144 3143 }
3145 3144 mutex_enter(&gid_info->gl_mutex);
3146 3145 }
3147 3146
3148 3147 gsrv_ents->se_state = IBDM_SE_VALID;
3149 3148 mutex_exit(&gid_info->gl_mutex);
3150 3149 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
3151 3150 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
3152 3151 bcopy(srv_ents->srv_name,
3153 3152 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
3154 3153 ibdm_dump_service_entries(&gsrv_ents->se_attr);
3155 3154 }
3156 3155 }
3157 3156
3158 3157
3159 3158 /*
3160 3159 * ibdm_get_diagcode:
3161 3160 * Send request to get IOU/IOC diag code
3162 3161 * Returns IBDM_SUCCESS/IBDM_FAILURE
3163 3162 */
3164 3163 static int
3165 3164 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
3166 3165 {
3167 3166 ibmf_msg_t *msg;
3168 3167 ib_mad_hdr_t *hdr;
3169 3168 ibdm_ioc_info_t *ioc;
3170 3169 ibdm_timeout_cb_args_t *cb_args;
3171 3170 timeout_id_t *timeout_id;
3172 3171
3173 3172 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
3174 3173 gid_info, attr);
3175 3174
3176 3175 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
3177 3176 &msg) != IBMF_SUCCESS) {
3178 3177 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
3179 3178 return (IBDM_FAILURE);
3180 3179 }
3181 3180
3182 3181 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
3183 3182 ibdm_alloc_send_buffers(msg);
3184 3183 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
3185 3184
3186 3185 mutex_enter(&gid_info->gl_mutex);
3187 3186 ibdm_bump_transactionID(gid_info);
3188 3187 mutex_exit(&gid_info->gl_mutex);
3189 3188
3190 3189 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
3191 3190 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
3192 3191 if (gid_info->gl_redirected == B_TRUE) {
3193 3192 if (gid_info->gl_redirect_dlid != 0) {
3194 3193 msg->im_local_addr.ia_remote_lid =
3195 3194 gid_info->gl_redirect_dlid;
3196 3195 }
3197 3196
3198 3197 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3199 3198 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3200 3199 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3201 3200 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3202 3201 } else {
3203 3202 msg->im_local_addr.ia_remote_qno = 1;
3204 3203 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3205 3204 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3206 3205 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3207 3206 }
3208 3207
3209 3208 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3210 3209 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3211 3210 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3212 3211 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3213 3212 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3214 3213 hdr->Status = 0;
3215 3214 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3216 3215
3217 3216 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
3218 3217 hdr->AttributeModifier = h2b32(attr);
3219 3218
3220 3219 if (attr == 0) {
3221 3220 cb_args = &gid_info->gl_iou_cb_args;
3222 3221 gid_info->gl_iou->iou_dc_valid = B_FALSE;
3223 3222 cb_args->cb_ioc_num = 0;
3224 3223 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE;
3225 3224 timeout_id = &gid_info->gl_timeout_id;
3226 3225 } else {
3227 3226 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
3228 3227 ioc->ioc_dc_valid = B_FALSE;
3229 3228 cb_args = &ioc->ioc_dc_cb_args;
3230 3229 cb_args->cb_ioc_num = attr - 1;
3231 3230 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE;
3232 3231 timeout_id = &ioc->ioc_dc_timeout_id;
3233 3232 }
3234 3233 cb_args->cb_gid_info = gid_info;
3235 3234 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
3236 3235 cb_args->cb_srvents_start = 0;
3237 3236
3238 3237 mutex_enter(&gid_info->gl_mutex);
3239 3238 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3240 3239 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3241 3240 mutex_exit(&gid_info->gl_mutex);
3242 3241
3243 3242 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
3244 3243 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);
3245 3244
3246 3245 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3247 3246 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3248 3247 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
3249 3248 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3250 3249 }
3251 3250 return (IBDM_SUCCESS);
3252 3251 }
3253 3252
3254 3253 /*
3255 3254 * ibdm_handle_diagcode:
3256 3255 * Process the DiagCode MAD response and update local DM
3257 3256 * data structure.
3258 3257 */
3259 3258 static void
3260 3259 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
3261 3260 ibdm_dp_gidinfo_t *gid_info, int *flag)
3262 3261 {
3263 3262 uint16_t attrmod, *diagcode;
3264 3263 ibdm_iou_info_t *iou;
3265 3264 ibdm_ioc_info_t *ioc;
3266 3265 timeout_id_t timeout_id;
3267 3266 ibdm_timeout_cb_args_t *cb_args;
3268 3267
3269 3268 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;
3270 3269
3271 3270 mutex_enter(&gid_info->gl_mutex);
3272 3271 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
3273 3272 iou = gid_info->gl_iou;
3274 3273 if (attrmod == 0) {
3275 3274 if (iou->iou_dc_valid != B_FALSE) {
3276 3275 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3277 3276 IBTF_DPRINTF_L4("ibdm",
3278 3277 "\thandle_diagcode: Duplicate IOU DiagCode");
3279 3278 mutex_exit(&gid_info->gl_mutex);
3280 3279 return;
3281 3280 }
3282 3281 cb_args = &gid_info->gl_iou_cb_args;
3283 3282 cb_args->cb_req_type = 0;
3284 3283 iou->iou_diagcode = b2h16(*diagcode);
3285 3284 iou->iou_dc_valid = B_TRUE;
3286 3285 if (gid_info->gl_timeout_id) {
3287 3286 timeout_id = gid_info->gl_timeout_id;
3288 3287 mutex_exit(&gid_info->gl_mutex);
3289 3288 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
3290 3289 "gl_timeout_id = 0x%x", timeout_id);
3291 3290 if (untimeout(timeout_id) == -1) {
3292 3291 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
3293 3292 "untimeout gl_timeout_id failed");
3294 3293 }
3295 3294 mutex_enter(&gid_info->gl_mutex);
3296 3295 gid_info->gl_timeout_id = 0;
3297 3296 }
3298 3297 } else {
3299 3298 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
3300 3299 if (ioc->ioc_dc_valid != B_FALSE) {
3301 3300 (*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3302 3301 IBTF_DPRINTF_L4("ibdm",
3303 3302 "\thandle_diagcode: Duplicate IOC DiagCode");
3304 3303 mutex_exit(&gid_info->gl_mutex);
3305 3304 return;
3306 3305 }
3307 3306 cb_args = &ioc->ioc_dc_cb_args;
3308 3307 cb_args->cb_req_type = 0;
3309 3308 ioc->ioc_diagcode = b2h16(*diagcode);
3310 3309 ioc->ioc_dc_valid = B_TRUE;
3311 3310 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
3312 3311 if (timeout_id) {
3313 3312 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
3314 3313 mutex_exit(&gid_info->gl_mutex);
3315 3314 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
3316 3315 "timeout_id = 0x%x", timeout_id);
3317 3316 if (untimeout(timeout_id) == -1) {
3318 3317 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
3319 3318 "untimeout ioc_dc_timeout_id failed");
3320 3319 }
3321 3320 mutex_enter(&gid_info->gl_mutex);
3322 3321 }
3323 3322 }
3324 3323 mutex_exit(&gid_info->gl_mutex);
3325 3324
3326 3325 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
3327 3326 "attrmod : 0x%x", b2h16(*diagcode), attrmod);
3328 3327 }
3329 3328
3330 3329
3331 3330 /*
3332 3331 * ibdm_is_ioc_present()
3333 3332 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list
3334 3333 */
3335 3334 static ibdm_ioc_info_t *
3336 3335 ibdm_is_ioc_present(ib_guid_t ioc_guid,
3337 3336 ibdm_dp_gidinfo_t *gid_info, int *flag)
3338 3337 {
3339 3338 int ii;
3340 3339 ibdm_ioc_info_t *ioc;
3341 3340 ibdm_dp_gidinfo_t *head;
3342 3341 ib_dm_io_unitinfo_t *iou;
3343 3342
3344 3343 mutex_enter(&ibdm.ibdm_mutex);
3345 3344 head = ibdm.ibdm_dp_gidlist_head;
3346 3345 while (head) {
3347 3346 mutex_enter(&head->gl_mutex);
3348 3347 if (head->gl_iou == NULL) {
3349 3348 mutex_exit(&head->gl_mutex);
3350 3349 head = head->gl_next;
3351 3350 continue;
3352 3351 }
3353 3352 iou = &head->gl_iou->iou_info;
3354 3353 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
3355 3354 ioc = IBDM_GIDINFO2IOCINFO(head, ii);
3356 3355 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
3357 3356 (ioc->ioc_profile.ioc_guid == ioc_guid)) {
3358 3357 if (gid_info == head) {
3359 3358 *flag |= IBDM_IBMF_PKT_DUP_RESP;
3360 3359 } else if (ibdm_check_dgid(head->gl_dgid_lo,
3361 3360 head->gl_dgid_hi) != NULL) {
3362 3361 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
3363 3362 "present: gid not present");
3364 3363 ibdm_add_to_gl_gid(gid_info, head);
3365 3364 }
3366 3365 mutex_exit(&head->gl_mutex);
3367 3366 mutex_exit(&ibdm.ibdm_mutex);
3368 3367 return (ioc);
3369 3368 }
3370 3369 }
3371 3370 mutex_exit(&head->gl_mutex);
3372 3371 head = head->gl_next;
3373 3372 }
3374 3373 mutex_exit(&ibdm.ibdm_mutex);
3375 3374 return (NULL);
3376 3375 }
3377 3376
3378 3377
3379 3378 /*
3380 3379 * ibdm_ibmf_send_cb()
3381 3380 * IBMF invokes this callback routine after posting the DM MAD to
3382 3381 * the HCA.
3383 3382 */
3384 3383 /*ARGSUSED*/
3385 3384 static void
3386 3385 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
3387 3386 {
3388 3387 ibdm_dump_ibmf_msg(ibmf_msg, 1);
3389 3388 ibdm_free_send_buffers(ibmf_msg);
3390 3389 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
3391 3390 IBTF_DPRINTF_L4("ibdm",
3392 3391 "\tibmf_send_cb: IBMF free msg failed");
3393 3392 }
3394 3393 }
3395 3394
3396 3395
3397 3396 /*
3398 3397 * ibdm_ibmf_recv_cb()
3399 3398 * Invoked by the IBMF when a response to the one of the DM requests
3400 3399 * is received.
3401 3400 */
3402 3401 /*ARGSUSED*/
3403 3402 static void
3404 3403 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3405 3404 {
3406 3405 ibdm_taskq_args_t *taskq_args;
3407 3406
3408 3407 /*
3409 3408 * If the taskq enable is set then dispatch a taskq to process
3410 3409 * the MAD, otherwise just process it on this thread
3411 3410 */
3412 3411 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
3413 3412 ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
3414 3413 return;
3415 3414 }
3416 3415
3417 3416 /*
3418 3417 * create a taskq and dispatch it to process the incoming MAD
3419 3418 */
3420 3419 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
3421 3420 if (taskq_args == NULL) {
3422 3421 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
3423 3422 "taskq_args");
3424 3423 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3425 3424 IBTF_DPRINTF_L4("ibmf_recv_cb",
3426 3425 "\tibmf_recv_cb: IBMF free msg failed");
3427 3426 }
3428 3427 return;
3429 3428 }
3430 3429 taskq_args->tq_ibmf_handle = ibmf_hdl;
3431 3430 taskq_args->tq_ibmf_msg = msg;
3432 3431 taskq_args->tq_args = arg;
3433 3432
3434 3433 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
3435 3434 TQ_NOSLEEP) == 0) {
3436 3435 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
3437 3436 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3438 3437 IBTF_DPRINTF_L4("ibmf_recv_cb",
3439 3438 "\tibmf_recv_cb: IBMF free msg failed");
3440 3439 }
3441 3440 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3442 3441 return;
3443 3442 }
3444 3443
3445 3444 /* taskq_args are deleted in ibdm_recv_incoming_mad() */
3446 3445 }
3447 3446
3448 3447
3449 3448 void
3450 3449 ibdm_recv_incoming_mad(void *args)
3451 3450 {
3452 3451 ibdm_taskq_args_t *taskq_args;
3453 3452
3454 3453 taskq_args = (ibdm_taskq_args_t *)args;
3455 3454
3456 3455 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
3457 3456 "Processing incoming MAD via taskq");
3458 3457
3459 3458 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
3460 3459 taskq_args->tq_ibmf_msg, taskq_args->tq_args);
3461 3460
3462 3461 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3463 3462 }
3464 3463
3465 3464
3466 3465 /*
3467 3466 * Calls ibdm_process_incoming_mad with all function arguments extracted
3468 3467 * from args
3469 3468 */
3470 3469 /*ARGSUSED*/
3471 3470 static void
3472 3471 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3473 3472 {
3474 3473 int flag = 0;
3475 3474 int ret;
3476 3475 uint64_t transaction_id;
3477 3476 ib_mad_hdr_t *hdr;
3478 3477 ibdm_dp_gidinfo_t *gid_info = NULL;
3479 3478
3480 3479 IBTF_DPRINTF_L4("ibdm",
3481 3480 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
3482 3481 ibdm_dump_ibmf_msg(msg, 0);
3483 3482
3484 3483 /*
3485 3484 * IBMF calls this routine for every DM MAD that arrives at this port.
3486 3485 * But we handle only the responses for requests we sent. We drop all
3487 3486 * the DM packets that does not have response bit set in the MAD
3488 3487 * header(this eliminates all the requests sent to this port).
3489 3488 * We handle only DM class version 1 MAD's
3490 3489 */
3491 3490 hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
3492 3491 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
3493 3492 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3494 3493 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3495 3494 "IBMF free msg failed DM request drop it");
3496 3495 }
3497 3496 return;
3498 3497 }
3499 3498
3500 3499 transaction_id = b2h64(hdr->TransactionID);
3501 3500
3502 3501 mutex_enter(&ibdm.ibdm_mutex);
3503 3502 gid_info = ibdm.ibdm_dp_gidlist_head;
3504 3503 while (gid_info) {
3505 3504 if ((gid_info->gl_transactionID &
3506 3505 IBDM_GID_TRANSACTIONID_MASK) ==
3507 3506 (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
3508 3507 break;
3509 3508 gid_info = gid_info->gl_next;
3510 3509 }
3511 3510 mutex_exit(&ibdm.ibdm_mutex);
3512 3511
3513 3512 if (gid_info == NULL) {
3514 3513 /* Drop the packet */
3515 3514 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
3516 3515 " does not match: 0x%llx", transaction_id);
3517 3516 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3518 3517 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3519 3518 "IBMF free msg failed DM request drop it");
3520 3519 }
3521 3520 return;
3522 3521 }
3523 3522
3524 3523 /* Handle redirection for all the MAD's, except ClassPortInfo */
3525 3524 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
3526 3525 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
3527 3526 ret = ibdm_handle_redirection(msg, gid_info, &flag);
3528 3527 if (ret == IBDM_SUCCESS) {
3529 3528 return;
3530 3529 }
3531 3530 } else {
3532 3531 uint_t gl_state;
3533 3532
3534 3533 mutex_enter(&gid_info->gl_mutex);
3535 3534 gl_state = gid_info->gl_state;
3536 3535 mutex_exit(&gid_info->gl_mutex);
3537 3536
3538 3537 switch (gl_state) {
3539 3538
3540 3539 case IBDM_SET_CLASSPORTINFO:
3541 3540 ibdm_handle_setclassportinfo(
3542 3541 ibmf_hdl, msg, gid_info, &flag);
3543 3542 break;
3544 3543
3545 3544 case IBDM_GET_CLASSPORTINFO:
3546 3545 ibdm_handle_classportinfo(
3547 3546 ibmf_hdl, msg, gid_info, &flag);
3548 3547 break;
3549 3548
3550 3549 case IBDM_GET_IOUNITINFO:
3551 3550 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
3552 3551 break;
3553 3552
3554 3553 case IBDM_GET_IOC_DETAILS:
3555 3554 switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3556 3555
3557 3556 case IB_DM_ATTR_SERVICE_ENTRIES:
3558 3557 ibdm_handle_srventry_mad(msg, gid_info, &flag);
3559 3558 break;
3560 3559
3561 3560 case IB_DM_ATTR_IOC_CTRL_PROFILE:
3562 3561 ibdm_handle_ioc_profile(
3563 3562 ibmf_hdl, msg, gid_info, &flag);
3564 3563 break;
3565 3564
3566 3565 case IB_DM_ATTR_DIAG_CODE:
3567 3566 ibdm_handle_diagcode(msg, gid_info, &flag);
3568 3567 break;
3569 3568
3570 3569 default:
3571 3570 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3572 3571 "Error state, wrong attribute :-(");
3573 3572 (void) ibmf_free_msg(ibmf_hdl, &msg);
3574 3573 return;
3575 3574 }
3576 3575 break;
3577 3576 default:
3578 3577 IBTF_DPRINTF_L2("ibdm",
3579 3578 "process_incoming_mad: Dropping the packet"
3580 3579 " gl_state %x", gl_state);
3581 3580 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3582 3581 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3583 3582 "IBMF free msg failed DM request drop it");
3584 3583 }
3585 3584 return;
3586 3585 }
3587 3586 }
3588 3587
3589 3588 if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
3590 3589 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
3591 3590 IBTF_DPRINTF_L2("ibdm",
3592 3591 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
3593 3592 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3594 3593 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3595 3594 "IBMF free msg failed DM request drop it");
3596 3595 }
3597 3596 return;
3598 3597 }
3599 3598
3600 3599 mutex_enter(&gid_info->gl_mutex);
3601 3600 if (gid_info->gl_pending_cmds < 1) {
3602 3601 IBTF_DPRINTF_L2("ibdm",
3603 3602 "\tprocess_incoming_mad: pending commands negative");
3604 3603 }
3605 3604 if (--gid_info->gl_pending_cmds) {
3606 3605 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
3607 3606 "gid_info %p pending cmds %d",
3608 3607 gid_info, gid_info->gl_pending_cmds);
3609 3608 mutex_exit(&gid_info->gl_mutex);
3610 3609 } else {
3611 3610 uint_t prev_state;
3612 3611 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
3613 3612 prev_state = gid_info->gl_state;
3614 3613 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
3615 3614 if (prev_state == IBDM_SET_CLASSPORTINFO) {
3616 3615 IBTF_DPRINTF_L4("ibdm",
3617 3616 "\tprocess_incoming_mad: "
3618 3617 "Setclassportinfo for Cisco FC GW is done.");
3619 3618 gid_info->gl_flag &= ~IBDM_CISCO_PROBE;
3620 3619 gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE;
3621 3620 mutex_exit(&gid_info->gl_mutex);
3622 3621 cv_broadcast(&gid_info->gl_probe_cv);
3623 3622 } else {
3624 3623 mutex_exit(&gid_info->gl_mutex);
3625 3624 ibdm_notify_newgid_iocs(gid_info);
3626 3625 mutex_enter(&ibdm.ibdm_mutex);
3627 3626 if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3628 3627 IBTF_DPRINTF_L4("ibdm",
3629 3628 "\tprocess_incoming_mad: Wakeup");
3630 3629 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3631 3630 cv_broadcast(&ibdm.ibdm_probe_cv);
3632 3631 }
3633 3632 mutex_exit(&ibdm.ibdm_mutex);
3634 3633 }
3635 3634 }
3636 3635
3637 3636 /*
3638 3637 * Do not deallocate the IBMF packet if atleast one request
3639 3638 * is posted. IBMF packet is reused.
3640 3639 */
3641 3640 if (!(flag & IBDM_IBMF_PKT_REUSED)) {
3642 3641 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3643 3642 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3644 3643 "IBMF free msg failed DM request drop it");
3645 3644 }
3646 3645 }
3647 3646 }
3648 3647
3649 3648
3650 3649 /*
3651 3650 * ibdm_verify_mad_status()
3652 3651 * Verifies the MAD status
3653 3652 * Returns IBDM_SUCCESS if status is correct
3654 3653 * Returns IBDM_FAILURE for bogus MAD status
3655 3654 */
3656 3655 static int
3657 3656 ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
3658 3657 {
3659 3658 int ret = 0;
3660 3659
3661 3660 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
3662 3661 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
3663 3662 return (IBDM_FAILURE);
3664 3663 }
3665 3664
3666 3665 if (b2h16(hdr->Status) == 0)
3667 3666 ret = IBDM_SUCCESS;
3668 3667 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
3669 3668 ret = IBDM_SUCCESS;
3670 3669 else {
3671 3670 IBTF_DPRINTF_L2("ibdm",
3672 3671 "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status));
3673 3672 ret = IBDM_FAILURE;
3674 3673 }
3675 3674 return (ret);
3676 3675 }
3677 3676
3678 3677
3679 3678
3680 3679 /*
3681 3680 * ibdm_handle_redirection()
3682 3681 * Returns IBDM_SUCCESS/IBDM_FAILURE
3683 3682 */
3684 3683 static int
3685 3684 ibdm_handle_redirection(ibmf_msg_t *msg,
3686 3685 ibdm_dp_gidinfo_t *gid_info, int *flag)
3687 3686 {
3688 3687 int attrmod, ioc_no, start;
3689 3688 void *data;
3690 3689 timeout_id_t *timeout_id;
3691 3690 ib_mad_hdr_t *hdr;
3692 3691 ibdm_ioc_info_t *ioc = NULL;
3693 3692 ibdm_timeout_cb_args_t *cb_args;
3694 3693 ib_mad_classportinfo_t *cpi;
3695 3694
3696 3695 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
3697 3696 mutex_enter(&gid_info->gl_mutex);
3698 3697 switch (gid_info->gl_state) {
3699 3698 case IBDM_GET_IOUNITINFO:
3700 3699 cb_args = &gid_info->gl_iou_cb_args;
3701 3700 timeout_id = &gid_info->gl_timeout_id;
3702 3701 break;
3703 3702
3704 3703 case IBDM_GET_IOC_DETAILS:
3705 3704 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3706 3705 switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3707 3706
3708 3707 case IB_DM_ATTR_DIAG_CODE:
3709 3708 if (attrmod == 0) {
3710 3709 cb_args = &gid_info->gl_iou_cb_args;
3711 3710 timeout_id = &gid_info->gl_timeout_id;
3712 3711 break;
3713 3712 }
3714 3713 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3715 3714 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3716 3715 "IOC# Out of range %d", attrmod);
3717 3716 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3718 3717 mutex_exit(&gid_info->gl_mutex);
3719 3718 return (IBDM_FAILURE);
3720 3719 }
3721 3720 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3722 3721 cb_args = &ioc->ioc_dc_cb_args;
3723 3722 timeout_id = &ioc->ioc_dc_timeout_id;
3724 3723 break;
3725 3724
3726 3725 case IB_DM_ATTR_IOC_CTRL_PROFILE:
3727 3726 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3728 3727 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3729 3728 "IOC# Out of range %d", attrmod);
3730 3729 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3731 3730 mutex_exit(&gid_info->gl_mutex);
3732 3731 return (IBDM_FAILURE);
3733 3732 }
3734 3733 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3735 3734 cb_args = &ioc->ioc_cb_args;
3736 3735 timeout_id = &ioc->ioc_timeout_id;
3737 3736 break;
3738 3737
3739 3738 case IB_DM_ATTR_SERVICE_ENTRIES:
3740 3739 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK);
3741 3740 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
3742 3741 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3743 3742 "IOC# Out of range %d", ioc_no);
3744 3743 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3745 3744 mutex_exit(&gid_info->gl_mutex);
3746 3745 return (IBDM_FAILURE);
3747 3746 }
3748 3747 start = (attrmod & IBDM_8_BIT_MASK);
3749 3748 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3750 3749 if (start > ioc->ioc_profile.ioc_service_entries) {
3751 3750 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3752 3751 " SE index Out of range %d", start);
3753 3752 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3754 3753 mutex_exit(&gid_info->gl_mutex);
3755 3754 return (IBDM_FAILURE);
3756 3755 }
3757 3756 cb_args = &ioc->ioc_serv[start].se_cb_args;
3758 3757 timeout_id = &ioc->ioc_serv[start].se_timeout_id;
3759 3758 break;
3760 3759
3761 3760 default:
3762 3761 /* ERROR State */
3763 3762 IBTF_DPRINTF_L2("ibdm",
3764 3763 "\thandle_redirection: wrong attribute :-(");
3765 3764 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3766 3765 mutex_exit(&gid_info->gl_mutex);
3767 3766 return (IBDM_FAILURE);
3768 3767 }
3769 3768 break;
3770 3769 default:
3771 3770 /* ERROR State */
3772 3771 IBTF_DPRINTF_L2("ibdm",
3773 3772 "\thandle_redirection: Error state :-(");
3774 3773 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3775 3774 mutex_exit(&gid_info->gl_mutex);
3776 3775 return (IBDM_FAILURE);
3777 3776 }
3778 3777 if ((*timeout_id) != 0) {
3779 3778 mutex_exit(&gid_info->gl_mutex);
3780 3779 if (untimeout(*timeout_id) == -1) {
3781 3780 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
3782 3781 "untimeout failed %x", *timeout_id);
3783 3782 } else {
3784 3783 IBTF_DPRINTF_L5("ibdm",
3785 3784 "\thandle_redirection: timeout %x", *timeout_id);
3786 3785 }
3787 3786 mutex_enter(&gid_info->gl_mutex);
3788 3787 *timeout_id = 0;
3789 3788 }
3790 3789
3791 3790 data = msg->im_msgbufs_recv.im_bufs_cl_data;
3792 3791 cpi = (ib_mad_classportinfo_t *)data;
3793 3792
3794 3793 gid_info->gl_resp_timeout =
3795 3794 (b2h32(cpi->RespTimeValue) & 0x1F);
3796 3795
3797 3796 gid_info->gl_redirected = B_TRUE;
3798 3797 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID);
3799 3798 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff);
3800 3799 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key);
3801 3800 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key);
3802 3801 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi);
3803 3802 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo);
3804 3803 gid_info->gl_redirectSL = cpi->RedirectSL;
3805 3804
3806 3805 if (gid_info->gl_redirect_dlid != 0) {
3807 3806 msg->im_local_addr.ia_remote_lid =
3808 3807 gid_info->gl_redirect_dlid;
3809 3808 }
3810 3809 ibdm_bump_transactionID(gid_info);
3811 3810 mutex_exit(&gid_info->gl_mutex);
3812 3811
3813 3812 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3814 3813 ibdm_alloc_send_buffers(msg);
3815 3814
3816 3815 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
3817 3816 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
3818 3817 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
3819 3818 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
3820 3819 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
3821 3820 hdr->Status = 0;
3822 3821 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
3823 3822 hdr->AttributeID =
3824 3823 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
3825 3824 hdr->AttributeModifier =
3826 3825 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
3827 3826 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3828 3827
3829 3828 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3830 3829 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3831 3830 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3832 3831 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3833 3832
3834 3833 mutex_enter(&gid_info->gl_mutex);
3835 3834 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3836 3835 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3837 3836 mutex_exit(&gid_info->gl_mutex);
3838 3837
3839 3838 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
3840 3839 "timeout %x", *timeout_id);
3841 3840
3842 3841 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3843 3842 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3844 3843 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
3845 3844 "message transport failed");
3846 3845 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3847 3846 }
3848 3847 (*flag) |= IBDM_IBMF_PKT_REUSED;
3849 3848 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
3850 3849 return (IBDM_SUCCESS);
3851 3850 }
3852 3851
3853 3852
3854 3853 /*
3855 3854 * ibdm_pkt_timeout_hdlr
3856 3855 * This timeout handler is registed for every IBMF packet that is
3857 3856 * sent through the IBMF. It gets called when no response is received
3858 3857 * within the specified time for the packet. No retries for the failed
3859 3858 * commands currently. Drops the failed IBMF packet and update the
3860 3859 * pending list commands.
3861 3860 */
3862 3861 static void
3863 3862 ibdm_pkt_timeout_hdlr(void *arg)
3864 3863 {
3865 3864 ibdm_iou_info_t *iou;
3866 3865 ibdm_ioc_info_t *ioc;
3867 3866 ibdm_timeout_cb_args_t *cb_args = arg;
3868 3867 ibdm_dp_gidinfo_t *gid_info;
3869 3868 int srv_ent;
3870 3869 uint_t new_gl_state;
3871 3870
3872 3871 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
3873 3872 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3874 3873 cb_args->cb_req_type, cb_args->cb_ioc_num,
3875 3874 cb_args->cb_srvents_start);
3876 3875
3877 3876 gid_info = cb_args->cb_gid_info;
3878 3877 mutex_enter(&gid_info->gl_mutex);
3879 3878
3880 3879 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
3881 3880 (cb_args->cb_req_type == 0)) {
3882 3881
3883 3882 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
3884 3883 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
3885 3884 cb_args->cb_ioc_num, cb_args->cb_srvents_start);
3886 3885
3887 3886 if (gid_info->gl_timeout_id)
3888 3887 gid_info->gl_timeout_id = 0;
3889 3888 mutex_exit(&gid_info->gl_mutex);
3890 3889 return;
3891 3890 }
3892 3891 if (cb_args->cb_retry_count) {
3893 3892 cb_args->cb_retry_count--;
3894 3893 /*
3895 3894 * A new timeout_id is set inside ibdm_retry_command().
3896 3895 * When the function returns an error, the timeout_id
3897 3896 * is reset (to zero) in the switch statement below.
3898 3897 */
3899 3898 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
3900 3899 mutex_exit(&gid_info->gl_mutex);
3901 3900 return;
3902 3901 }
3903 3902 cb_args->cb_retry_count = 0;
3904 3903 }
3905 3904
3906 3905 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
3907 3906 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3908 3907 cb_args->cb_req_type, cb_args->cb_ioc_num,
3909 3908 cb_args->cb_srvents_start);
3910 3909
3911 3910 switch (cb_args->cb_req_type) {
3912 3911
3913 3912 case IBDM_REQ_TYPE_CLASSPORTINFO:
3914 3913 case IBDM_REQ_TYPE_IOUINFO:
3915 3914 new_gl_state = IBDM_GID_PROBING_FAILED;
3916 3915 if (gid_info->gl_timeout_id)
3917 3916 gid_info->gl_timeout_id = 0;
3918 3917 break;
3919 3918
3920 3919 case IBDM_REQ_TYPE_IOCINFO:
3921 3920 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3922 3921 iou = gid_info->gl_iou;
3923 3922 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3924 3923 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3925 3924 if (ioc->ioc_timeout_id)
3926 3925 ioc->ioc_timeout_id = 0;
3927 3926 break;
3928 3927
3929 3928 case IBDM_REQ_TYPE_SRVENTS:
3930 3929 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3931 3930 iou = gid_info->gl_iou;
3932 3931 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3933 3932 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3934 3933 srv_ent = cb_args->cb_srvents_start;
3935 3934 if (ioc->ioc_serv[srv_ent].se_timeout_id)
3936 3935 ioc->ioc_serv[srv_ent].se_timeout_id = 0;
3937 3936 break;
3938 3937
3939 3938 case IBDM_REQ_TYPE_IOU_DIAGCODE:
3940 3939 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3941 3940 iou = gid_info->gl_iou;
3942 3941 iou->iou_dc_valid = B_FALSE;
3943 3942 if (gid_info->gl_timeout_id)
3944 3943 gid_info->gl_timeout_id = 0;
3945 3944 break;
3946 3945
3947 3946 case IBDM_REQ_TYPE_IOC_DIAGCODE:
3948 3947 new_gl_state = IBDM_GID_PROBING_COMPLETE;
3949 3948 iou = gid_info->gl_iou;
3950 3949 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3951 3950 ioc->ioc_dc_valid = B_FALSE;
3952 3951 if (ioc->ioc_dc_timeout_id)
3953 3952 ioc->ioc_dc_timeout_id = 0;
3954 3953 break;
3955 3954
3956 3955 default: /* ERROR State */
3957 3956 new_gl_state = IBDM_GID_PROBING_FAILED;
3958 3957 if (gid_info->gl_timeout_id)
3959 3958 gid_info->gl_timeout_id = 0;
3960 3959 IBTF_DPRINTF_L2("ibdm",
3961 3960 "\tpkt_timeout_hdlr: wrong request type.");
3962 3961 break;
3963 3962 }
3964 3963
3965 3964 --gid_info->gl_pending_cmds; /* decrease the counter */
3966 3965
3967 3966 if (gid_info->gl_pending_cmds == 0) {
3968 3967 gid_info->gl_state = new_gl_state;
3969 3968 mutex_exit(&gid_info->gl_mutex);
3970 3969 /*
3971 3970 * Delete this gid_info if the gid probe fails.
3972 3971 */
3973 3972 if (new_gl_state == IBDM_GID_PROBING_FAILED) {
3974 3973 ibdm_delete_glhca_list(gid_info);
3975 3974 }
3976 3975 ibdm_notify_newgid_iocs(gid_info);
3977 3976 mutex_enter(&ibdm.ibdm_mutex);
3978 3977 if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3979 3978 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
3980 3979 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3981 3980 cv_broadcast(&ibdm.ibdm_probe_cv);
3982 3981 }
3983 3982 mutex_exit(&ibdm.ibdm_mutex);
3984 3983 } else {
3985 3984 /*
3986 3985 * Reset gl_pending_cmd if the extra timeout happens since
3987 3986 * gl_pending_cmd becomes negative as a result.
3988 3987 */
3989 3988 if (gid_info->gl_pending_cmds < 0) {
3990 3989 gid_info->gl_pending_cmds = 0;
3991 3990 IBTF_DPRINTF_L2("ibdm",
3992 3991 "\tpkt_timeout_hdlr: extra timeout request."
3993 3992 " reset gl_pending_cmds");
3994 3993 }
3995 3994 mutex_exit(&gid_info->gl_mutex);
3996 3995 /*
3997 3996 * Delete this gid_info if the gid probe fails.
3998 3997 */
3999 3998 if (new_gl_state == IBDM_GID_PROBING_FAILED) {
4000 3999 ibdm_delete_glhca_list(gid_info);
4001 4000 }
4002 4001 }
4003 4002 }
4004 4003
4005 4004
4006 4005 /*
4007 4006 * ibdm_retry_command()
4008 4007 * Retries the failed command.
4009 4008 * Returns IBDM_FAILURE/IBDM_SUCCESS
4010 4009 */
4011 4010 static int
4012 4011 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
4013 4012 {
4014 4013 int ret;
4015 4014 ibmf_msg_t *msg;
4016 4015 ib_mad_hdr_t *hdr;
4017 4016 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info;
4018 4017 timeout_id_t *timeout_id;
4019 4018 ibdm_ioc_info_t *ioc;
4020 4019 int ioc_no;
4021 4020 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4022 4021
4023 4022 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
4024 4023 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4025 4024 cb_args->cb_req_type, cb_args->cb_ioc_num,
4026 4025 cb_args->cb_srvents_start);
4027 4026
4028 4027 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);
4029 4028
4030 4029
4031 4030 /*
4032 4031 * Reset the gid if alloc_msg failed with BAD_HANDLE
4033 4032 * ibdm_reset_gidinfo reinits the gid_info
4034 4033 */
4035 4034 if (ret == IBMF_BAD_HANDLE) {
4036 4035 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
4037 4036 gid_info);
4038 4037
4039 4038 mutex_exit(&gid_info->gl_mutex);
4040 4039 ibdm_reset_gidinfo(gid_info);
4041 4040 mutex_enter(&gid_info->gl_mutex);
4042 4041
4043 4042 /* Retry alloc */
4044 4043 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
4045 4044 &msg);
4046 4045 }
4047 4046
4048 4047 if (ret != IBDM_SUCCESS) {
4049 4048 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
4050 4049 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4051 4050 cb_args->cb_req_type, cb_args->cb_ioc_num,
4052 4051 cb_args->cb_srvents_start);
4053 4052 return (IBDM_FAILURE);
4054 4053 }
4055 4054
4056 4055 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
4057 4056 ibdm_alloc_send_buffers(msg);
4058 4057 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
4059 4058
4060 4059 ibdm_bump_transactionID(gid_info);
4061 4060
4062 4061 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
4063 4062 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
4064 4063 if (gid_info->gl_redirected == B_TRUE) {
4065 4064 if (gid_info->gl_redirect_dlid != 0) {
4066 4065 msg->im_local_addr.ia_remote_lid =
4067 4066 gid_info->gl_redirect_dlid;
4068 4067 }
4069 4068 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
4070 4069 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
4071 4070 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
4072 4071 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
4073 4072 } else {
4074 4073 msg->im_local_addr.ia_remote_qno = 1;
4075 4074 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
4076 4075 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
4077 4076 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
4078 4077 }
4079 4078 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
4080 4079 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
4081 4080 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
4082 4081 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
4083 4082 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
4084 4083 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
4085 4084 hdr->Status = 0;
4086 4085 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
4087 4086
4088 4087 switch (cb_args->cb_req_type) {
4089 4088 case IBDM_REQ_TYPE_CLASSPORTINFO:
4090 4089 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
4091 4090 hdr->AttributeModifier = 0;
4092 4091 timeout_id = &gid_info->gl_timeout_id;
4093 4092 break;
4094 4093 case IBDM_REQ_TYPE_IOUINFO:
4095 4094 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
4096 4095 hdr->AttributeModifier = 0;
4097 4096 timeout_id = &gid_info->gl_timeout_id;
4098 4097 break;
4099 4098 case IBDM_REQ_TYPE_IOCINFO:
4100 4099 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
4101 4100 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4102 4101 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4103 4102 timeout_id = &ioc->ioc_timeout_id;
4104 4103 break;
4105 4104 case IBDM_REQ_TYPE_SRVENTS:
4106 4105 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
4107 4106 ibdm_fill_srv_attr_mod(hdr, cb_args);
4108 4107 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4109 4108 timeout_id =
4110 4109 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
4111 4110 break;
4112 4111 case IBDM_REQ_TYPE_IOU_DIAGCODE:
4113 4112 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4114 4113 hdr->AttributeModifier = 0;
4115 4114 timeout_id = &gid_info->gl_timeout_id;
4116 4115 break;
4117 4116 case IBDM_REQ_TYPE_IOC_DIAGCODE:
4118 4117 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4119 4118 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4120 4119 ioc_no = cb_args->cb_ioc_num;
4121 4120 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
4122 4121 timeout_id = &ioc->ioc_dc_timeout_id;
4123 4122 break;
4124 4123 }
4125 4124 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr))
4126 4125
4127 4126 *timeout_id = timeout(ibdm_pkt_timeout_hdlr,
4128 4127 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
4129 4128
4130 4129 mutex_exit(&gid_info->gl_mutex);
4131 4130
4132 4131 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
4133 4132 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
4134 4133 cb_args->cb_srvents_start, *timeout_id);
4135 4134
4136 4135 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl,
4137 4136 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
4138 4137 cb_args, 0) != IBMF_SUCCESS) {
4139 4138 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
4140 4139 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4141 4140 cb_args->cb_req_type, cb_args->cb_ioc_num,
4142 4141 cb_args->cb_srvents_start);
4143 4142 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
4144 4143 }
4145 4144 mutex_enter(&gid_info->gl_mutex);
4146 4145 return (IBDM_SUCCESS);
4147 4146 }
4148 4147
4149 4148
4150 4149 /*
4151 4150 * ibdm_update_ioc_port_gidlist()
4152 4151 */
4153 4152 static void
4154 4153 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
4155 4154 ibdm_dp_gidinfo_t *gid_info)
4156 4155 {
4157 4156 int ii, ngid_ents;
4158 4157 ibdm_gid_t *tmp;
4159 4158 ibdm_hca_list_t *gid_hca_head, *temp;
4160 4159 ibdm_hca_list_t *ioc_head = NULL;
4161 4160 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4162 4161
4163 4162 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");
4164 4163
4165 4164 ngid_ents = gid_info->gl_ngids;
4166 4165 dest->ioc_nportgids = ngid_ents;
4167 4166 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
4168 4167 ngid_ents, KM_SLEEP);
4169 4168 tmp = gid_info->gl_gid;
4170 4169 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
4171 4170 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
4172 4171 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
4173 4172 tmp = tmp->gid_next;
4174 4173 }
4175 4174
4176 4175 gid_hca_head = gid_info->gl_hca_list;
4177 4176 while (gid_hca_head) {
4178 4177 temp = ibdm_dup_hca_attr(gid_hca_head);
4179 4178 temp->hl_next = ioc_head;
4180 4179 ioc_head = temp;
4181 4180 gid_hca_head = gid_hca_head->hl_next;
4182 4181 }
4183 4182 dest->ioc_hca_list = ioc_head;
4184 4183 }
4185 4184
4186 4185
4187 4186 /*
4188 4187 * ibdm_alloc_send_buffers()
4189 4188 * Allocates memory for the IBMF send buffer to send and/or receive
4190 4189 * the Device Management MAD packet.
4191 4190 */
4192 4191 static void
4193 4192 ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
4194 4193 {
4195 4194 msgp->im_msgbufs_send.im_bufs_mad_hdr =
4196 4195 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);
4197 4196
4198 4197 msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *)
4199 4198 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
4200 4199 msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ;
4201 4200
4202 4201 msgp->im_msgbufs_send.im_bufs_cl_data =
4203 4202 ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ);
4204 4203 msgp->im_msgbufs_send.im_bufs_cl_data_len =
4205 4204 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ;
4206 4205 }
4207 4206
4208 4207
4209 4208 /*
4210 4209 * ibdm_alloc_send_buffers()
4211 4210 * De-allocates memory for the IBMF send buffer
4212 4211 */
4213 4212 static void
4214 4213 ibdm_free_send_buffers(ibmf_msg_t *msgp)
4215 4214 {
4216 4215 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
4217 4216 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
4218 4217 }
4219 4218
4220 4219 /*
4221 4220 * ibdm_probe_ioc()
4222 4221 * 1. Gets the node records for the port GUID. This detects all the port
4223 4222 * to the IOU.
4224 4223 * 2. Selectively probes all the IOC, given it's node GUID
4225 4224 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC
4226 4225 * Controller Profile asynchronously
4227 4226 */
4228 4227 /*ARGSUSED*/
4229 4228 static void
4230 4229 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
4231 4230 {
4232 4231 int ii, nrecords;
4233 4232 size_t nr_len = 0, pi_len = 0;
4234 4233 ib_gid_t sgid, dgid;
4235 4234 ibdm_hca_list_t *hca_list = NULL;
4236 4235 sa_node_record_t *nr, *tmp;
4237 4236 ibdm_port_attr_t *port = NULL;
4238 4237 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid;
4239 4238 ibdm_dp_gidinfo_t *temp_gidinfo;
4240 4239 ibdm_gid_t *temp_gid;
4241 4240 sa_portinfo_record_t *pi;
4242 4241
4243 4242 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin",
4244 4243 nodeguid, ioc_guid, reprobe_flag);
4245 4244
4246 4245 /* Rescan the GID list for any removed GIDs for reprobe */
4247 4246 if (reprobe_flag)
4248 4247 ibdm_rescan_gidlist(&ioc_guid);
4249 4248
4250 4249 mutex_enter(&ibdm.ibdm_hl_mutex);
4251 4250 for (ibdm_get_next_port(&hca_list, &port, 1); port;
4252 4251 ibdm_get_next_port(&hca_list, &port, 1)) {
4253 4252 reprobe_gid = new_gid = node_gid = NULL;
4254 4253
4255 4254 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
4256 4255 if (nr == NULL) {
4257 4256 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
4258 4257 continue;
4259 4258 }
4260 4259 nrecords = (nr_len / sizeof (sa_node_record_t));
4261 4260 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) {
4262 4261 if ((pi = ibdm_get_portinfo(
4263 4262 port->pa_sa_hdl, &pi_len, tmp->LID)) == NULL) {
4264 4263 IBTF_DPRINTF_L4("ibdm",
4265 4264 "\tibdm_get_portinfo: no portinfo recs");
4266 4265 continue;
4267 4266 }
4268 4267
4269 4268 /*
4270 4269 * If Device Management is not supported on
4271 4270 * this port, skip the rest.
4272 4271 */
4273 4272 if (!(pi->PortInfo.CapabilityMask &
4274 4273 SM_CAP_MASK_IS_DM_SUPPD)) {
4275 4274 kmem_free(pi, pi_len);
4276 4275 continue;
4277 4276 }
4278 4277
4279 4278 /*
4280 4279 * For reprobes: Check if GID, already in
4281 4280 * the list. If so, set the state to SKIPPED
4282 4281 */
4283 4282 if (((temp_gidinfo = ibdm_find_gid(nodeguid,
4284 4283 tmp->NodeInfo.PortGUID)) != NULL) &&
4285 4284 temp_gidinfo->gl_state ==
4286 4285 IBDM_GID_PROBING_COMPLETE) {
4287 4286 ASSERT(reprobe_gid == NULL);
4288 4287 ibdm_addto_glhcalist(temp_gidinfo,
4289 4288 hca_list);
4290 4289 reprobe_gid = temp_gidinfo;
4291 4290 kmem_free(pi, pi_len);
4292 4291 continue;
4293 4292 } else if (temp_gidinfo != NULL) {
4294 4293 kmem_free(pi, pi_len);
4295 4294 ibdm_addto_glhcalist(temp_gidinfo,
4296 4295 hca_list);
4297 4296 continue;
4298 4297 }
4299 4298
4300 4299 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
4301 4300 "create_gid : prefix %llx, guid %llx\n",
4302 4301 pi->PortInfo.GidPrefix,
4303 4302 tmp->NodeInfo.PortGUID);
4304 4303
4305 4304 sgid.gid_prefix = port->pa_sn_prefix;
4306 4305 sgid.gid_guid = port->pa_port_guid;
4307 4306 dgid.gid_prefix = pi->PortInfo.GidPrefix;
4308 4307 dgid.gid_guid = tmp->NodeInfo.PortGUID;
4309 4308 new_gid = ibdm_create_gid_info(port, sgid,
4310 4309 dgid);
4311 4310 if (new_gid == NULL) {
4312 4311 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4313 4312 "create_gid_info failed\n");
4314 4313 kmem_free(pi, pi_len);
4315 4314 continue;
4316 4315 }
4317 4316 if (node_gid == NULL) {
4318 4317 node_gid = new_gid;
4319 4318 ibdm_add_to_gl_gid(node_gid, node_gid);
4320 4319 } else {
4321 4320 IBTF_DPRINTF_L4("ibdm",
4322 4321 "\tprobe_ioc: new gid");
4323 4322 temp_gid = kmem_zalloc(
4324 4323 sizeof (ibdm_gid_t), KM_SLEEP);
4325 4324 temp_gid->gid_dgid_hi =
4326 4325 new_gid->gl_dgid_hi;
4327 4326 temp_gid->gid_dgid_lo =
4328 4327 new_gid->gl_dgid_lo;
4329 4328 temp_gid->gid_next = node_gid->gl_gid;
4330 4329 node_gid->gl_gid = temp_gid;
4331 4330 node_gid->gl_ngids++;
4332 4331 }
4333 4332 new_gid->gl_is_dm_capable = B_TRUE;
4334 4333 new_gid->gl_nodeguid = nodeguid;
4335 4334 new_gid->gl_portguid = dgid.gid_guid;
4336 4335 ibdm_addto_glhcalist(new_gid, hca_list);
4337 4336
4338 4337 /*
4339 4338 * Set the state to skipped as all these
4340 4339 * gids point to the same node.
4341 4340 * We (re)probe only one GID below and reset
4342 4341 * state appropriately
4343 4342 */
4344 4343 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
4345 4344 new_gid->gl_devid = (*tmp).NodeInfo.DeviceID;
4346 4345 kmem_free(pi, pi_len);
4347 4346 }
4348 4347 kmem_free(nr, nr_len);
4349 4348
4350 4349 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
4351 4350 "reprobe_gid %p new_gid %p node_gid %p",
4352 4351 reprobe_flag, reprobe_gid, new_gid, node_gid);
4353 4352
4354 4353 if (reprobe_flag != 0 && reprobe_gid != NULL) {
4355 4354 int niocs, jj;
4356 4355 ibdm_ioc_info_t *tmp_ioc;
4357 4356 int ioc_matched = 0;
4358 4357
4359 4358 mutex_exit(&ibdm.ibdm_hl_mutex);
4360 4359 mutex_enter(&reprobe_gid->gl_mutex);
4361 4360 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
4362 4361 niocs =
4363 4362 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
4364 4363 reprobe_gid->gl_pending_cmds++;
4365 4364 mutex_exit(&reprobe_gid->gl_mutex);
4366 4365
4367 4366 for (jj = 0; jj < niocs; jj++) {
4368 4367 tmp_ioc =
4369 4368 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
4370 4369 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
4371 4370 continue;
4372 4371
4373 4372 ioc_matched = 1;
4374 4373
4375 4374 /*
4376 4375 * Explicitly set gl_reprobe_flag to 0 so that
4377 4376 * IBnex is not notified on completion
4378 4377 */
4379 4378 mutex_enter(&reprobe_gid->gl_mutex);
4380 4379 reprobe_gid->gl_reprobe_flag = 0;
4381 4380 mutex_exit(&reprobe_gid->gl_mutex);
4382 4381
4383 4382 mutex_enter(&ibdm.ibdm_mutex);
4384 4383 ibdm.ibdm_ngid_probes_in_progress++;
4385 4384 mutex_exit(&ibdm.ibdm_mutex);
4386 4385 if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
4387 4386 IBDM_SUCCESS) {
4388 4387 IBTF_DPRINTF_L4("ibdm",
4389 4388 "\tprobe_ioc: "
4390 4389 "send_ioc_profile failed "
4391 4390 "for ioc %d", jj);
4392 4391 ibdm_gid_decr_pending(reprobe_gid);
4393 4392 break;
4394 4393 }
4395 4394 mutex_enter(&ibdm.ibdm_mutex);
4396 4395 ibdm_wait_probe_completion();
4397 4396 mutex_exit(&ibdm.ibdm_mutex);
4398 4397 break;
4399 4398 }
4400 4399 if (ioc_matched == 0)
4401 4400 ibdm_gid_decr_pending(reprobe_gid);
4402 4401 else {
4403 4402 mutex_enter(&ibdm.ibdm_hl_mutex);
4404 4403 break;
4405 4404 }
4406 4405 } else if (new_gid != NULL) {
4407 4406 mutex_exit(&ibdm.ibdm_hl_mutex);
4408 4407 node_gid = node_gid ? node_gid : new_gid;
4409 4408
4410 4409 /*
4411 4410 * New or reinserted GID : Enable notification
4412 4411 * to IBnex
4413 4412 */
4414 4413 mutex_enter(&node_gid->gl_mutex);
4415 4414 node_gid->gl_reprobe_flag = 1;
4416 4415 mutex_exit(&node_gid->gl_mutex);
4417 4416
4418 4417 ibdm_probe_gid(node_gid);
4419 4418
4420 4419 mutex_enter(&ibdm.ibdm_hl_mutex);
4421 4420 }
4422 4421 }
4423 4422 mutex_exit(&ibdm.ibdm_hl_mutex);
4424 4423 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
4425 4424 }
4426 4425
4427 4426
4428 4427 /*
4429 4428 * ibdm_probe_gid()
4430 4429 * Selectively probes the GID
4431 4430 */
4432 4431 static void
4433 4432 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
4434 4433 {
4435 4434 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");
4436 4435
4437 4436 /*
4438 4437 * A Cisco FC GW needs the special handling to get IOUnitInfo.
4439 4438 */
4440 4439 mutex_enter(&gid_info->gl_mutex);
4441 4440 if (ibdm_is_cisco_switch(gid_info)) {
4442 4441 gid_info->gl_pending_cmds++;
4443 4442 gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
4444 4443 mutex_exit(&gid_info->gl_mutex);
4445 4444
4446 4445 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
4447 4446
4448 4447 mutex_enter(&gid_info->gl_mutex);
4449 4448 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4450 4449 --gid_info->gl_pending_cmds;
4451 4450 mutex_exit(&gid_info->gl_mutex);
4452 4451
4453 4452 /* free the hca_list on this gid_info */
4454 4453 ibdm_delete_glhca_list(gid_info);
4455 4454 gid_info = gid_info->gl_next;
4456 4455 return;
4457 4456 }
4458 4457
4459 4458 mutex_enter(&gid_info->gl_mutex);
4460 4459 ibdm_wait_cisco_probe_completion(gid_info);
4461 4460
4462 4461 IBTF_DPRINTF_L4("ibdm",
4463 4462 "\tprobe_gid: CISCO Wakeup signal received");
4464 4463 }
4465 4464
4466 4465 /* move on to the 'GET_CLASSPORTINFO' stage */
4467 4466 gid_info->gl_pending_cmds++;
4468 4467 gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
4469 4468 mutex_exit(&gid_info->gl_mutex);
4470 4469
4471 4470 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
4472 4471
4473 4472 mutex_enter(&gid_info->gl_mutex);
4474 4473 gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4475 4474 --gid_info->gl_pending_cmds;
4476 4475 mutex_exit(&gid_info->gl_mutex);
4477 4476
4478 4477 /* free the hca_list on this gid_info */
4479 4478 ibdm_delete_glhca_list(gid_info);
4480 4479 gid_info = gid_info->gl_next;
4481 4480 return;
4482 4481 }
4483 4482
4484 4483 mutex_enter(&ibdm.ibdm_mutex);
4485 4484 ibdm.ibdm_ngid_probes_in_progress++;
4486 4485 gid_info = gid_info->gl_next;
4487 4486 ibdm_wait_probe_completion();
4488 4487 mutex_exit(&ibdm.ibdm_mutex);
4489 4488
4490 4489 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
4491 4490 }
4492 4491
4493 4492
4494 4493 /*
4495 4494 * ibdm_create_gid_info()
4496 4495 * Allocates a gid_info structure and initializes
4497 4496 * Returns pointer to the structure on success
4498 4497 * and NULL on failure
4499 4498 */
4500 4499 static ibdm_dp_gidinfo_t *
4501 4500 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
4502 4501 {
4503 4502 uint8_t ii, npaths;
4504 4503 sa_path_record_t *path;
4505 4504 size_t len;
4506 4505 ibdm_pkey_tbl_t *pkey_tbl;
4507 4506 ibdm_dp_gidinfo_t *gid_info = NULL;
4508 4507 int ret;
4509 4508
4510 4509 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
4511 4510 npaths = 1;
4512 4511
4513 4512 /* query for reversible paths */
4514 4513 if (port->pa_sa_hdl)
4515 4514 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
4516 4515 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
4517 4516 &len, &path);
4518 4517 else
4519 4518 return (NULL);
4520 4519
4521 4520 if (ret == IBMF_SUCCESS && path) {
4522 4521 ibdm_dump_path_info(path);
4523 4522
4524 4523 gid_info = kmem_zalloc(
4525 4524 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
4526 4525 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
4527 4526 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
4528 4527 gid_info->gl_dgid_hi = path->DGID.gid_prefix;
4529 4528 gid_info->gl_dgid_lo = path->DGID.gid_guid;
4530 4529 gid_info->gl_sgid_hi = path->SGID.gid_prefix;
4531 4530 gid_info->gl_sgid_lo = path->SGID.gid_guid;
4532 4531 gid_info->gl_p_key = path->P_Key;
4533 4532 gid_info->gl_sa_hdl = port->pa_sa_hdl;
4534 4533 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl;
4535 4534 gid_info->gl_slid = path->SLID;
4536 4535 gid_info->gl_dlid = path->DLID;
4537 4536 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID)
4538 4537 << IBDM_GID_TRANSACTIONID_SHIFT;
4539 4538 gid_info->gl_min_transactionID = gid_info->gl_transactionID;
4540 4539 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1)
4541 4540 << IBDM_GID_TRANSACTIONID_SHIFT;
4542 4541 gid_info->gl_SL = path->SL;
4543 4542
4544 4543 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
4545 4544 for (ii = 0; ii < port->pa_npkeys; ii++) {
4546 4545 if (port->pa_pkey_tbl == NULL)
4547 4546 break;
4548 4547
4549 4548 pkey_tbl = &port->pa_pkey_tbl[ii];
4550 4549 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
4551 4550 (pkey_tbl->pt_qp_hdl != NULL)) {
4552 4551 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
4553 4552 break;
4554 4553 }
4555 4554 }
4556 4555 kmem_free(path, len);
4557 4556
4558 4557 /*
4559 4558 * QP handle for GID not initialized. No matching Pkey
4560 4559 * was found!! ibdm should *not* hit this case. Flag an
4561 4560 * error and drop the GID if ibdm does encounter this.
4562 4561 */
4563 4562 if (gid_info->gl_qp_hdl == NULL) {
4564 4563 IBTF_DPRINTF_L2(ibdm_string,
4565 4564 "\tcreate_gid_info: No matching Pkey");
4566 4565 ibdm_delete_gidinfo(gid_info);
4567 4566 return (NULL);
4568 4567 }
4569 4568
4570 4569 ibdm.ibdm_ngids++;
4571 4570 if (ibdm.ibdm_dp_gidlist_head == NULL) {
4572 4571 ibdm.ibdm_dp_gidlist_head = gid_info;
4573 4572 ibdm.ibdm_dp_gidlist_tail = gid_info;
4574 4573 } else {
4575 4574 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
4576 4575 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
4577 4576 ibdm.ibdm_dp_gidlist_tail = gid_info;
4578 4577 }
4579 4578 }
4580 4579
4581 4580 return (gid_info);
4582 4581 }
4583 4582
4584 4583
4585 4584 /*
4586 4585 * ibdm_get_node_records
4587 4586 * Sends a SA query to get the NODE record
4588 4587 * Returns pointer to the sa_node_record_t on success
4589 4588 * and NULL on failure
4590 4589 */
4591 4590 static sa_node_record_t *
4592 4591 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
4593 4592 {
4594 4593 sa_node_record_t req, *resp = NULL;
4595 4594 ibmf_saa_access_args_t args;
4596 4595 int ret;
4597 4596
4598 4597 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");
4599 4598
4600 4599 bzero(&req, sizeof (sa_node_record_t));
4601 4600 req.NodeInfo.NodeGUID = guid;
4602 4601
4603 4602 args.sq_attr_id = SA_NODERECORD_ATTRID;
4604 4603 args.sq_access_type = IBMF_SAA_RETRIEVE;
4605 4604 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID;
4606 4605 args.sq_template = &req;
4607 4606 args.sq_callback = NULL;
4608 4607 args.sq_callback_arg = NULL;
4609 4608
4610 4609 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4611 4610 if (ret != IBMF_SUCCESS) {
4612 4611 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
4613 4612 " SA Retrieve Failed: %d", ret);
4614 4613 return (NULL);
4615 4614 }
4616 4615 if ((resp == NULL) || (*length == 0)) {
4617 4616 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
4618 4617 return (NULL);
4619 4618 }
4620 4619
4621 4620 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
4622 4621 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);
4623 4622
4624 4623 return (resp);
4625 4624 }
4626 4625
4627 4626
4628 4627 /*
4629 4628 * ibdm_get_portinfo()
4630 4629 * Sends a SA query to get the PortInfo record
4631 4630 * Returns pointer to the sa_portinfo_record_t on success
4632 4631 * and NULL on failure
4633 4632 */
4634 4633 static sa_portinfo_record_t *
4635 4634 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
4636 4635 {
4637 4636 sa_portinfo_record_t req, *resp = NULL;
4638 4637 ibmf_saa_access_args_t args;
4639 4638 int ret;
4640 4639
4641 4640 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");
4642 4641
4643 4642 bzero(&req, sizeof (sa_portinfo_record_t));
4644 4643 req.EndportLID = lid;
4645 4644
4646 4645 args.sq_attr_id = SA_PORTINFORECORD_ATTRID;
4647 4646 args.sq_access_type = IBMF_SAA_RETRIEVE;
4648 4647 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
4649 4648 args.sq_template = &req;
4650 4649 args.sq_callback = NULL;
4651 4650 args.sq_callback_arg = NULL;
4652 4651
4653 4652 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4654 4653 if (ret != IBMF_SUCCESS) {
4655 4654 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
4656 4655 " SA Retrieve Failed: 0x%X", ret);
4657 4656 return (NULL);
4658 4657 }
4659 4658 if ((*length == 0) || (resp == NULL))
4660 4659 return (NULL);
4661 4660
4662 4661 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
4663 4662 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
4664 4663 return (resp);
4665 4664 }
4666 4665
4667 4666
4668 4667 /*
4669 4668 * ibdm_ibnex_register_callback
4670 4669 * IB nexus callback routine for HCA attach and detach notification
4671 4670 */
4672 4671 void
4673 4672 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
4674 4673 {
4675 4674 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
4676 4675 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4677 4676 ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
4678 4677 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4679 4678 }
4680 4679
4681 4680
4682 4681 /*
4683 4682 * ibdm_ibnex_unregister_callbacks
4684 4683 */
4685 4684 void
4686 4685 ibdm_ibnex_unregister_callback()
4687 4686 {
4688 4687 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4689 4688 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4690 4689 ibdm.ibdm_ibnex_callback = NULL;
4691 4690 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4692 4691 }
4693 4692
4694 4693 /*
4695 4694 * ibdm_get_waittime()
4696 4695 * Calculates the wait time based on the last HCA attach time
4697 4696 */
4698 4697 static clock_t
4699 4698 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait_sec)
4700 4699 {
4701 4700 const hrtime_t dft_wait = dft_wait_sec * NANOSEC;
4702 4701 hrtime_t temp, wait_time = 0;
4703 4702 clock_t usecs;
4704 4703 int i;
4705 4704 ibdm_hca_list_t *hca;
4706 4705
4707 4706 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4708 4707 "\tport settling time %d", hca_guid, dft_wait);
4709 4708
4710 4709 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4711 4710
4712 4711 hca = ibdm.ibdm_hca_list_head;
4713 4712
4714 4713 for (i = 0; i < ibdm.ibdm_hca_count; i++, hca = hca->hl_next) {
4715 4714 if (hca->hl_nports == hca->hl_nports_active)
4716 4715 continue;
4717 4716
4718 4717 if (hca_guid && (hca_guid != hca->hl_hca_guid))
4719 4718 continue;
4720 4719
4721 4720 temp = gethrtime() - hca->hl_attach_time;
4722 4721 temp = MAX(0, (dft_wait - temp));
4723 4722
4724 4723 if (hca_guid) {
4725 4724 wait_time = temp;
4726 4725 break;
4727 4726 }
4728 4727
4729 4728 wait_time = MAX(temp, wait_time);
4730 4729 }
4731 4730
4732 4731 /* convert to microseconds */
4733 4732 usecs = MIN(wait_time, dft_wait) / (NANOSEC / MICROSEC);
4734 4733
4735 4734 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld usecs",
4736 4735 (long)usecs);
4737 4736
4738 4737 return (drv_usectohz(usecs));
4739 4738 }
4740 4739
4741 4740 void
4742 4741 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4743 4742 {
4744 4743 clock_t wait_time;
4745 4744
4746 4745 mutex_enter(&ibdm.ibdm_hl_mutex);
4747 4746
4748 4747 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0)
4749 4748 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4750 4749 &ibdm.ibdm_hl_mutex, wait_time, TR_CLOCK_TICK);
4751 4750
4752 4751 mutex_exit(&ibdm.ibdm_hl_mutex);
4753 4752 }
4754 4753
4755 4754
4756 4755 /*
4757 4756 * ibdm_ibnex_probe_hcaport
4758 4757 * Probes the presence of HCA port (with HCA dip and port number)
4759 4758 * Returns port attributes structure on SUCCESS
4760 4759 */
4761 4760 ibdm_port_attr_t *
4762 4761 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4763 4762 {
4764 4763 int ii, jj;
4765 4764 ibdm_hca_list_t *hca_list;
4766 4765 ibdm_port_attr_t *port_attr;
4767 4766
4768 4767 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4769 4768
4770 4769 mutex_enter(&ibdm.ibdm_hl_mutex);
4771 4770 hca_list = ibdm.ibdm_hca_list_head;
4772 4771 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4773 4772 if (hca_list->hl_hca_guid == hca_guid) {
4774 4773 for (jj = 0; jj < hca_list->hl_nports; jj++) {
4775 4774 if (hca_list->hl_port_attr[jj].pa_port_num ==
4776 4775 port_num) {
4777 4776 break;
4778 4777 }
4779 4778 }
4780 4779 if (jj != hca_list->hl_nports)
4781 4780 break;
4782 4781 }
4783 4782 hca_list = hca_list->hl_next;
4784 4783 }
4785 4784 if (ii == ibdm.ibdm_hca_count) {
4786 4785 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
4787 4786 mutex_exit(&ibdm.ibdm_hl_mutex);
4788 4787 return (NULL);
4789 4788 }
4790 4789 port_attr = (ibdm_port_attr_t *)kmem_zalloc(
4791 4790 sizeof (ibdm_port_attr_t), KM_SLEEP);
4792 4791 bcopy((char *)&hca_list->hl_port_attr[jj],
4793 4792 port_attr, sizeof (ibdm_port_attr_t));
4794 4793 ibdm_update_port_attr(port_attr);
4795 4794
4796 4795 mutex_exit(&ibdm.ibdm_hl_mutex);
4797 4796 return (port_attr);
4798 4797 }
4799 4798
4800 4799
4801 4800 /*
4802 4801 * ibdm_ibnex_get_port_attrs
4803 4802 * Scan all HCAs for a matching port_guid.
4804 4803 * Returns "port attributes" structure on success.
4805 4804 */
4806 4805 ibdm_port_attr_t *
4807 4806 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
4808 4807 {
4809 4808 int ii, jj;
4810 4809 ibdm_hca_list_t *hca_list;
4811 4810 ibdm_port_attr_t *port_attr;
4812 4811
4813 4812 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");
4814 4813
4815 4814 mutex_enter(&ibdm.ibdm_hl_mutex);
4816 4815 hca_list = ibdm.ibdm_hca_list_head;
4817 4816
4818 4817 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4819 4818 for (jj = 0; jj < hca_list->hl_nports; jj++) {
4820 4819 if (hca_list->hl_port_attr[jj].pa_port_guid ==
4821 4820 port_guid) {
4822 4821 break;
4823 4822 }
4824 4823 }
4825 4824 if (jj != hca_list->hl_nports)
4826 4825 break;
4827 4826 hca_list = hca_list->hl_next;
4828 4827 }
4829 4828
4830 4829 if (ii == ibdm.ibdm_hca_count) {
4831 4830 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
4832 4831 mutex_exit(&ibdm.ibdm_hl_mutex);
4833 4832 return (NULL);
4834 4833 }
4835 4834
4836 4835 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
4837 4836 KM_SLEEP);
4838 4837 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
4839 4838 sizeof (ibdm_port_attr_t));
4840 4839 ibdm_update_port_attr(port_attr);
4841 4840
4842 4841 mutex_exit(&ibdm.ibdm_hl_mutex);
4843 4842 return (port_attr);
4844 4843 }
4845 4844
4846 4845
4847 4846 /*
4848 4847 * ibdm_ibnex_free_port_attr()
4849 4848 */
4850 4849 void
4851 4850 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
4852 4851 {
4853 4852 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
4854 4853 if (port_attr) {
4855 4854 if (port_attr->pa_pkey_tbl != NULL) {
4856 4855 kmem_free(port_attr->pa_pkey_tbl,
4857 4856 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
4858 4857 }
4859 4858 kmem_free(port_attr, sizeof (ibdm_port_attr_t));
4860 4859 }
4861 4860 }
4862 4861
4863 4862
4864 4863 /*
4865 4864 * ibdm_ibnex_get_hca_list()
4866 4865 * Returns portinfo for all the port for all the HCA's
4867 4866 */
4868 4867 void
4869 4868 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
4870 4869 {
4871 4870 ibdm_hca_list_t *head = NULL, *temp, *temp1;
4872 4871 int ii;
4873 4872
4874 4873 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");
4875 4874
4876 4875 mutex_enter(&ibdm.ibdm_hl_mutex);
4877 4876 temp = ibdm.ibdm_hca_list_head;
4878 4877 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4879 4878 temp1 = ibdm_dup_hca_attr(temp);
4880 4879 temp1->hl_next = head;
4881 4880 head = temp1;
4882 4881 temp = temp->hl_next;
4883 4882 }
4884 4883 *count = ibdm.ibdm_hca_count;
4885 4884 *hca = head;
4886 4885 mutex_exit(&ibdm.ibdm_hl_mutex);
4887 4886 }
4888 4887
4889 4888
4890 4889 /*
4891 4890 * ibdm_ibnex_get_hca_info_by_guid()
4892 4891 */
4893 4892 ibdm_hca_list_t *
4894 4893 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
4895 4894 {
4896 4895 ibdm_hca_list_t *head = NULL, *hca = NULL;
4897 4896
4898 4897 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");
4899 4898
4900 4899 mutex_enter(&ibdm.ibdm_hl_mutex);
4901 4900 head = ibdm.ibdm_hca_list_head;
4902 4901 while (head) {
4903 4902 if (head->hl_hca_guid == hca_guid) {
4904 4903 hca = ibdm_dup_hca_attr(head);
4905 4904 hca->hl_next = NULL;
4906 4905 break;
4907 4906 }
4908 4907 head = head->hl_next;
4909 4908 }
4910 4909 mutex_exit(&ibdm.ibdm_hl_mutex);
4911 4910 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
4912 4911 return (hca);
4913 4912 }
4914 4913
4915 4914
4916 4915 /*
4917 4916 * ibdm_dup_hca_attr()
4918 4917 * Allocate a new HCA attribute strucuture and initialize
4919 4918 * hca attribute structure with the incoming HCA attributes
4920 4919 * returned the allocated hca attributes.
4921 4920 */
4922 4921 static ibdm_hca_list_t *
4923 4922 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
4924 4923 {
4925 4924 int len;
4926 4925 ibdm_hca_list_t *out_hca;
4927 4926
4928 4927 len = sizeof (ibdm_hca_list_t) +
4929 4928 (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
4930 4929 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
4931 4930 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
4932 4931 bcopy((char *)in_hca,
4933 4932 (char *)out_hca, sizeof (ibdm_hca_list_t));
4934 4933 if (in_hca->hl_nports) {
4935 4934 out_hca->hl_port_attr = (ibdm_port_attr_t *)
4936 4935 ((char *)out_hca + sizeof (ibdm_hca_list_t));
4937 4936 bcopy((char *)in_hca->hl_port_attr,
4938 4937 (char *)out_hca->hl_port_attr,
4939 4938 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
4940 4939 for (len = 0; len < out_hca->hl_nports; len++)
4941 4940 ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
4942 4941 }
4943 4942 return (out_hca);
4944 4943 }
4945 4944
4946 4945
4947 4946 /*
4948 4947 * ibdm_ibnex_free_hca_list()
4949 4948 * Free one/more HCA lists
4950 4949 */
4951 4950 void
4952 4951 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
4953 4952 {
4954 4953 int ii;
4955 4954 size_t len;
4956 4955 ibdm_hca_list_t *temp;
4957 4956 ibdm_port_attr_t *port;
4958 4957
4959 4958 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
4960 4959 ASSERT(hca_list);
4961 4960 while (hca_list) {
4962 4961 temp = hca_list;
4963 4962 hca_list = hca_list->hl_next;
4964 4963 for (ii = 0; ii < temp->hl_nports; ii++) {
4965 4964 port = &temp->hl_port_attr[ii];
4966 4965 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
4967 4966 if (len != 0)
4968 4967 kmem_free(port->pa_pkey_tbl, len);
4969 4968 }
4970 4969 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
4971 4970 sizeof (ibdm_port_attr_t));
4972 4971 kmem_free(temp, len);
4973 4972 }
4974 4973 }
4975 4974
4976 4975
4977 4976 /*
4978 4977 * ibdm_ibnex_probe_iocguid()
4979 4978 * Probes the IOC on the fabric and returns the IOC information
4980 4979 * if present. Otherwise, NULL is returned
4981 4980 */
4982 4981 /* ARGSUSED */
4983 4982 ibdm_ioc_info_t *
4984 4983 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
4985 4984 {
4986 4985 int k;
4987 4986 ibdm_ioc_info_t *ioc_info;
4988 4987 ibdm_dp_gidinfo_t *gid_info; /* used as index and arg */
4989 4988 timeout_id_t *timeout_id;
4990 4989
4991 4990 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
4992 4991 iou, ioc_guid, reprobe_flag);
4993 4992
4994 4993 if (ibdm_enumerate_iocs == 0)
4995 4994 return (NULL);
4996 4995
4997 4996 /* Check whether we know this already */
4998 4997 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
4999 4998 if (ioc_info == NULL) {
5000 4999 mutex_enter(&ibdm.ibdm_mutex);
5001 5000 while (ibdm.ibdm_busy & IBDM_BUSY)
5002 5001 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5003 5002 ibdm.ibdm_busy |= IBDM_BUSY;
5004 5003 mutex_exit(&ibdm.ibdm_mutex);
5005 5004 ibdm_probe_ioc(iou, ioc_guid, 0);
5006 5005 mutex_enter(&ibdm.ibdm_mutex);
5007 5006 ibdm.ibdm_busy &= ~IBDM_BUSY;
5008 5007 cv_broadcast(&ibdm.ibdm_busy_cv);
5009 5008 mutex_exit(&ibdm.ibdm_mutex);
5010 5009 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5011 5010 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */
5012 5011 ASSERT(gid_info != NULL);
5013 5012 /* Free the ioc_list before reprobe; and cancel any timers */
5014 5013 mutex_enter(&ibdm.ibdm_mutex);
5015 5014 mutex_enter(&gid_info->gl_mutex);
5016 5015 if (ioc_info->ioc_timeout_id) {
5017 5016 timeout_id = ioc_info->ioc_timeout_id;
5018 5017 ioc_info->ioc_timeout_id = 0;
5019 5018 mutex_exit(&gid_info->gl_mutex);
5020 5019 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5021 5020 "ioc_timeout_id = 0x%x", timeout_id);
5022 5021 if (untimeout(timeout_id) == -1) {
5023 5022 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5024 5023 "untimeout ioc_timeout_id failed");
5025 5024 }
5026 5025 mutex_enter(&gid_info->gl_mutex);
5027 5026 }
5028 5027 if (ioc_info->ioc_dc_timeout_id) {
5029 5028 timeout_id = ioc_info->ioc_dc_timeout_id;
5030 5029 ioc_info->ioc_dc_timeout_id = 0;
5031 5030 mutex_exit(&gid_info->gl_mutex);
5032 5031 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5033 5032 "ioc_dc_timeout_id = 0x%x", timeout_id);
5034 5033 if (untimeout(timeout_id) == -1) {
5035 5034 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5036 5035 "untimeout ioc_dc_timeout_id failed");
5037 5036 }
5038 5037 mutex_enter(&gid_info->gl_mutex);
5039 5038 }
5040 5039 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
5041 5040 if (ioc_info->ioc_serv[k].se_timeout_id) {
5042 5041 timeout_id = ioc_info->ioc_serv[k].
5043 5042 se_timeout_id;
5044 5043 ioc_info->ioc_serv[k].se_timeout_id = 0;
5045 5044 mutex_exit(&gid_info->gl_mutex);
5046 5045 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5047 5046 "ioc_info->ioc_serv[k].se_timeout_id = %x",
5048 5047 k, timeout_id);
5049 5048 if (untimeout(timeout_id) == -1) {
5050 5049 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5051 5050 "untimeout se_timeout_id %d "
5052 5051 "failed", k);
5053 5052 }
5054 5053 mutex_enter(&gid_info->gl_mutex);
5055 5054 }
5056 5055 mutex_exit(&gid_info->gl_mutex);
5057 5056 mutex_exit(&ibdm.ibdm_mutex);
5058 5057 ibdm_ibnex_free_ioc_list(ioc_info);
5059 5058
5060 5059 mutex_enter(&ibdm.ibdm_mutex);
5061 5060 while (ibdm.ibdm_busy & IBDM_BUSY)
5062 5061 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5063 5062 ibdm.ibdm_busy |= IBDM_BUSY;
5064 5063 mutex_exit(&ibdm.ibdm_mutex);
5065 5064
5066 5065 ibdm_probe_ioc(iou, ioc_guid, 1);
5067 5066
5068 5067 /*
5069 5068 * Skip if gl_reprobe_flag is set, this will be
5070 5069 * a re-inserted / new GID, for which notifications
5071 5070 * have already been send.
5072 5071 */
5073 5072 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
5074 5073 gid_info = gid_info->gl_next) {
5075 5074 uint8_t ii, niocs;
5076 5075 ibdm_ioc_info_t *ioc;
5077 5076
5078 5077 if (gid_info->gl_iou == NULL)
5079 5078 continue;
5080 5079
5081 5080 if (gid_info->gl_reprobe_flag) {
5082 5081 gid_info->gl_reprobe_flag = 0;
5083 5082 continue;
5084 5083 }
5085 5084
5086 5085 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
5087 5086 for (ii = 0; ii < niocs; ii++) {
5088 5087 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
5089 5088 if (ioc->ioc_profile.ioc_guid == ioc_guid) {
5090 5089 mutex_enter(&ibdm.ibdm_mutex);
5091 5090 ibdm_reprobe_update_port_srv(ioc,
5092 5091 gid_info);
5093 5092 mutex_exit(&ibdm.ibdm_mutex);
5094 5093 }
5095 5094 }
5096 5095 }
5097 5096 mutex_enter(&ibdm.ibdm_mutex);
5098 5097 ibdm.ibdm_busy &= ~IBDM_BUSY;
5099 5098 cv_broadcast(&ibdm.ibdm_busy_cv);
5100 5099 mutex_exit(&ibdm.ibdm_mutex);
5101 5100
5102 5101 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5103 5102 }
5104 5103 return (ioc_info);
5105 5104 }
5106 5105
5107 5106
5108 5107 /*
5109 5108 * ibdm_get_ioc_info_with_gid()
5110 5109 * Returns pointer to ibdm_ioc_info_t if it finds
5111 5110 * matching record for the ioc_guid. Otherwise NULL is returned.
5112 5111 * The pointer to gid_info is set to the second argument in case that
5113 5112 * the non-NULL value returns (and the second argument is not NULL).
5114 5113 *
5115 5114 * Note. use the same strings as "ibnex_get_ioc_info" in
5116 5115 * IBTF_DPRINTF() to keep compatibility.
5117 5116 */
5118 5117 static ibdm_ioc_info_t *
5119 5118 ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid,
5120 5119 ibdm_dp_gidinfo_t **gid_info)
5121 5120 {
5122 5121 int ii;
5123 5122 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL;
5124 5123 ibdm_dp_gidinfo_t *gid_list;
5125 5124 ib_dm_io_unitinfo_t *iou;
5126 5125
5127 5126 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);
5128 5127
5129 5128 mutex_enter(&ibdm.ibdm_mutex);
5130 5129 while (ibdm.ibdm_busy & IBDM_BUSY)
5131 5130 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5132 5131 ibdm.ibdm_busy |= IBDM_BUSY;
5133 5132
5134 5133 if (gid_info)
5135 5134 *gid_info = NULL; /* clear the value of gid_info */
5136 5135
5137 5136 gid_list = ibdm.ibdm_dp_gidlist_head;
5138 5137 while (gid_list) {
5139 5138 mutex_enter(&gid_list->gl_mutex);
5140 5139 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5141 5140 mutex_exit(&gid_list->gl_mutex);
5142 5141 gid_list = gid_list->gl_next;
5143 5142 continue;
5144 5143 }
5145 5144 if (gid_list->gl_iou == NULL) {
5146 5145 IBTF_DPRINTF_L2("ibdm",
5147 5146 "\tget_ioc_info: No IOU info");
5148 5147 mutex_exit(&gid_list->gl_mutex);
5149 5148 gid_list = gid_list->gl_next;
5150 5149 continue;
5151 5150 }
5152 5151 iou = &gid_list->gl_iou->iou_info;
5153 5152 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5154 5153 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5155 5154 if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
5156 5155 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
5157 5156 ioc = ibdm_dup_ioc_info(tmp, gid_list);
5158 5157 if (gid_info)
5159 5158 *gid_info = gid_list; /* set this ptr */
5160 5159 mutex_exit(&gid_list->gl_mutex);
5161 5160 ibdm.ibdm_busy &= ~IBDM_BUSY;
5162 5161 cv_broadcast(&ibdm.ibdm_busy_cv);
5163 5162 mutex_exit(&ibdm.ibdm_mutex);
5164 5163 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
5165 5164 return (ioc);
5166 5165 }
5167 5166 }
5168 5167 if (ii == iou->iou_num_ctrl_slots)
5169 5168 ioc = NULL;
5170 5169
5171 5170 mutex_exit(&gid_list->gl_mutex);
5172 5171 gid_list = gid_list->gl_next;
5173 5172 }
5174 5173
5175 5174 ibdm.ibdm_busy &= ~IBDM_BUSY;
5176 5175 cv_broadcast(&ibdm.ibdm_busy_cv);
5177 5176 mutex_exit(&ibdm.ibdm_mutex);
5178 5177 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
5179 5178 return (ioc);
5180 5179 }
5181 5180
5182 5181 /*
5183 5182 * ibdm_ibnex_get_ioc_info()
5184 5183 * Returns pointer to ibdm_ioc_info_t if it finds
5185 5184 * matching record for the ioc_guid, otherwise NULL
5186 5185 * is returned
5187 5186 *
5188 5187 * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now.
5189 5188 */
5190 5189 ibdm_ioc_info_t *
5191 5190 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
5192 5191 {
5193 5192 if (ibdm_enumerate_iocs == 0)
5194 5193 return (NULL);
5195 5194
5196 5195 /* will not use the gid_info pointer, so the second arg is NULL */
5197 5196 return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL));
5198 5197 }
5199 5198
5200 5199 /*
5201 5200 * ibdm_ibnex_get_ioc_count()
5202 5201 * Returns number of ibdm_ioc_info_t it finds
5203 5202 */
5204 5203 int
5205 5204 ibdm_ibnex_get_ioc_count(void)
5206 5205 {
5207 5206 int count = 0, k;
5208 5207 ibdm_ioc_info_t *ioc;
5209 5208 ibdm_dp_gidinfo_t *gid_list;
5210 5209
5211 5210 if (ibdm_enumerate_iocs == 0)
5212 5211 return (0);
5213 5212
5214 5213 mutex_enter(&ibdm.ibdm_mutex);
5215 5214 ibdm_sweep_fabric(0);
5216 5215
5217 5216 while (ibdm.ibdm_busy & IBDM_BUSY)
5218 5217 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5219 5218 ibdm.ibdm_busy |= IBDM_BUSY;
5220 5219
5221 5220 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5222 5221 gid_list = gid_list->gl_next) {
5223 5222 mutex_enter(&gid_list->gl_mutex);
5224 5223 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
5225 5224 (gid_list->gl_iou == NULL)) {
5226 5225 mutex_exit(&gid_list->gl_mutex);
5227 5226 continue;
5228 5227 }
5229 5228 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
5230 5229 k++) {
5231 5230 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
5232 5231 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
5233 5232 ++count;
5234 5233 }
5235 5234 mutex_exit(&gid_list->gl_mutex);
5236 5235 }
5237 5236 ibdm.ibdm_busy &= ~IBDM_BUSY;
5238 5237 cv_broadcast(&ibdm.ibdm_busy_cv);
5239 5238 mutex_exit(&ibdm.ibdm_mutex);
5240 5239
5241 5240 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
5242 5241 return (count);
5243 5242 }
5244 5243
5245 5244
5246 5245 /*
5247 5246 * ibdm_ibnex_get_ioc_list()
5248 5247 * Returns information about all the IOCs present on the fabric.
5249 5248 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
5250 5249 * Does not sweep fabric if DONOT_PROBE is set
5251 5250 */
5252 5251 ibdm_ioc_info_t *
5253 5252 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
5254 5253 {
5255 5254 int ii;
5256 5255 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc;
5257 5256 ibdm_dp_gidinfo_t *gid_list;
5258 5257 ib_dm_io_unitinfo_t *iou;
5259 5258
5260 5259 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");
5261 5260
5262 5261 if (ibdm_enumerate_iocs == 0)
5263 5262 return (NULL);
5264 5263
5265 5264 mutex_enter(&ibdm.ibdm_mutex);
5266 5265 if (list_flag != IBDM_IBNEX_DONOT_PROBE)
5267 5266 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);
5268 5267
5269 5268 while (ibdm.ibdm_busy & IBDM_BUSY)
5270 5269 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5271 5270 ibdm.ibdm_busy |= IBDM_BUSY;
5272 5271
5273 5272 gid_list = ibdm.ibdm_dp_gidlist_head;
5274 5273 while (gid_list) {
5275 5274 mutex_enter(&gid_list->gl_mutex);
5276 5275 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5277 5276 mutex_exit(&gid_list->gl_mutex);
5278 5277 gid_list = gid_list->gl_next;
5279 5278 continue;
5280 5279 }
5281 5280 if (gid_list->gl_iou == NULL) {
5282 5281 IBTF_DPRINTF_L2("ibdm",
5283 5282 "\tget_ioc_list: No IOU info");
5284 5283 mutex_exit(&gid_list->gl_mutex);
5285 5284 gid_list = gid_list->gl_next;
5286 5285 continue;
5287 5286 }
5288 5287 iou = &gid_list->gl_iou->iou_info;
5289 5288 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5290 5289 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5291 5290 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5292 5291 tmp = ibdm_dup_ioc_info(ioc, gid_list);
5293 5292 tmp->ioc_next = ioc_list;
5294 5293 ioc_list = tmp;
5295 5294 }
5296 5295 }
5297 5296 mutex_exit(&gid_list->gl_mutex);
5298 5297 gid_list = gid_list->gl_next;
5299 5298 }
5300 5299 ibdm.ibdm_busy &= ~IBDM_BUSY;
5301 5300 cv_broadcast(&ibdm.ibdm_busy_cv);
5302 5301 mutex_exit(&ibdm.ibdm_mutex);
5303 5302
5304 5303 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
5305 5304 return (ioc_list);
5306 5305 }
5307 5306
5308 5307 /*
5309 5308 * ibdm_dup_ioc_info()
5310 5309 * Duplicate the IOC information and return the IOC
5311 5310 * information.
5312 5311 */
5313 5312 static ibdm_ioc_info_t *
5314 5313 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
5315 5314 {
5316 5315 ibdm_ioc_info_t *out_ioc;
5317 5316 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
5318 5317 ASSERT(MUTEX_HELD(&gid_list->gl_mutex));
5319 5318
5320 5319 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
5321 5320 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
5322 5321 ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
5323 5322 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
5324 5323 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;
5325 5324
5326 5325 return (out_ioc);
5327 5326 }
5328 5327
5329 5328
5330 5329 /*
5331 5330 * ibdm_free_ioc_list()
5332 5331 * Deallocate memory for IOC list structure
5333 5332 */
5334 5333 void
5335 5334 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
5336 5335 {
5337 5336 ibdm_ioc_info_t *temp;
5338 5337
5339 5338 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
5340 5339 while (ioc) {
5341 5340 temp = ioc;
5342 5341 ioc = ioc->ioc_next;
5343 5342 kmem_free(temp->ioc_gid_list,
5344 5343 (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
5345 5344 if (temp->ioc_hca_list)
5346 5345 ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
5347 5346 kmem_free(temp, sizeof (ibdm_ioc_info_t));
5348 5347 }
5349 5348 }
5350 5349
5351 5350
5352 5351 /*
5353 5352 * ibdm_ibnex_update_pkey_tbls
5354 5353 * Updates the DM P_Key database.
5355 5354 * NOTE: Two cases are handled here: P_Key being added or removed.
5356 5355 *
5357 5356 * Arguments : NONE
5358 5357 * Return Values : NONE
5359 5358 */
5360 5359 void
5361 5360 ibdm_ibnex_update_pkey_tbls(void)
5362 5361 {
5363 5362 int h, pp, pidx;
5364 5363 uint_t nports;
5365 5364 uint_t size;
5366 5365 ib_pkey_t new_pkey;
5367 5366 ib_pkey_t *orig_pkey;
5368 5367 ibdm_hca_list_t *hca_list;
5369 5368 ibdm_port_attr_t *port;
5370 5369 ibt_hca_portinfo_t *pinfop;
5371 5370
5372 5371 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");
5373 5372
5374 5373 mutex_enter(&ibdm.ibdm_hl_mutex);
5375 5374 hca_list = ibdm.ibdm_hca_list_head;
5376 5375
5377 5376 for (h = 0; h < ibdm.ibdm_hca_count; h++) {
5378 5377
5379 5378 /* This updates P_Key Tables for all ports of this HCA */
5380 5379 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
5381 5380 &nports, &size);
5382 5381
5383 5382 /* number of ports shouldn't have changed */
5384 5383 ASSERT(nports == hca_list->hl_nports);
5385 5384
5386 5385 for (pp = 0; pp < hca_list->hl_nports; pp++) {
5387 5386 port = &hca_list->hl_port_attr[pp];
5388 5387
5389 5388 /*
5390 5389 * First figure out the P_Keys from IBTL.
5391 5390 * Three things could have happened:
5392 5391 * New P_Keys added
5393 5392 * Existing P_Keys removed
5394 5393 * Both of the above two
5395 5394 *
5396 5395 * Loop through the P_Key Indices and check if a
5397 5396 * give P_Key_Ix matches that of the one seen by
5398 5397 * IBDM. If they match no action is needed.
5399 5398 *
5400 5399 * If they don't match:
5401 5400 * 1. if orig_pkey is invalid and new_pkey is valid
5402 5401 * ---> add new_pkey to DM database
5403 5402 * 2. if orig_pkey is valid and new_pkey is invalid
5404 5403 * ---> remove orig_pkey from DM database
5405 5404 * 3. if orig_pkey and new_pkey are both valid:
5406 5405 * ---> remov orig_pkey from DM database
5407 5406 * ---> add new_pkey to DM database
5408 5407 * 4. if orig_pkey and new_pkey are both invalid:
5409 5408 * ---> do nothing. Updated DM database.
5410 5409 */
5411 5410
5412 5411 for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
5413 5412 new_pkey = pinfop[pp].p_pkey_tbl[pidx];
5414 5413 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;
5415 5414
5416 5415 /* keys match - do nothing */
5417 5416 if (*orig_pkey == new_pkey)
5418 5417 continue;
5419 5418
5420 5419 if (IBDM_INVALID_PKEY(*orig_pkey) &&
5421 5420 !IBDM_INVALID_PKEY(new_pkey)) {
5422 5421 /* P_Key was added */
5423 5422 IBTF_DPRINTF_L5("ibdm",
5424 5423 "\tibnex_update_pkey_tbls: new "
5425 5424 "P_Key added = 0x%x", new_pkey);
5426 5425 *orig_pkey = new_pkey;
5427 5426 ibdm_port_attr_ibmf_init(port,
5428 5427 new_pkey, pp);
5429 5428 } else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5430 5429 IBDM_INVALID_PKEY(new_pkey)) {
5431 5430 /* P_Key was removed */
5432 5431 IBTF_DPRINTF_L5("ibdm",
5433 5432 "\tibnex_update_pkey_tbls: P_Key "
5434 5433 "removed = 0x%x", *orig_pkey);
5435 5434 *orig_pkey = new_pkey;
5436 5435 (void) ibdm_port_attr_ibmf_fini(port,
5437 5436 pidx);
5438 5437 } else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5439 5438 !IBDM_INVALID_PKEY(new_pkey)) {
5440 5439 /* P_Key were replaced */
5441 5440 IBTF_DPRINTF_L5("ibdm",
5442 5441 "\tibnex_update_pkey_tbls: P_Key "
5443 5442 "replaced 0x%x with 0x%x",
5444 5443 *orig_pkey, new_pkey);
5445 5444 (void) ibdm_port_attr_ibmf_fini(port,
5446 5445 pidx);
5447 5446 *orig_pkey = new_pkey;
5448 5447 ibdm_port_attr_ibmf_init(port,
5449 5448 new_pkey, pp);
5450 5449 } else {
5451 5450 /*
5452 5451 * P_Keys are invalid
5453 5452 * set anyway to reflect if
5454 5453 * INVALID_FULL was changed to
5455 5454 * INVALID_LIMITED or vice-versa.
5456 5455 */
5457 5456 *orig_pkey = new_pkey;
5458 5457 } /* end of else */
5459 5458
5460 5459 } /* loop of p_key index */
5461 5460
5462 5461 } /* loop of #ports of HCA */
5463 5462
5464 5463 ibt_free_portinfo(pinfop, size);
5465 5464 hca_list = hca_list->hl_next;
5466 5465
5467 5466 } /* loop for all HCAs in the system */
5468 5467
5469 5468 mutex_exit(&ibdm.ibdm_hl_mutex);
5470 5469 }
5471 5470
5472 5471
5473 5472 /*
5474 5473 * ibdm_send_ioc_profile()
5475 5474 * Send IOC Controller Profile request. When the request is completed
5476 5475 * IBMF calls ibdm_process_incoming_mad routine to inform about
5477 5476 * the completion.
5478 5477 */
5479 5478 static int
5480 5479 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
5481 5480 {
5482 5481 ibmf_msg_t *msg;
5483 5482 ib_mad_hdr_t *hdr;
5484 5483 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
5485 5484 ibdm_timeout_cb_args_t *cb_args;
5486 5485
5487 5486 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
5488 5487 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);
5489 5488
5490 5489 /*
5491 5490 * Send command to get IOC profile.
5492 5491 * Allocate a IBMF packet and initialize the packet.
5493 5492 */
5494 5493 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
5495 5494 &msg) != IBMF_SUCCESS) {
5496 5495 IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail");
5497 5496 return (IBDM_FAILURE);
5498 5497 }
5499 5498
5500 5499 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
5501 5500 ibdm_alloc_send_buffers(msg);
5502 5501 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
5503 5502
5504 5503 mutex_enter(&gid_info->gl_mutex);
5505 5504 ibdm_bump_transactionID(gid_info);
5506 5505 mutex_exit(&gid_info->gl_mutex);
5507 5506
5508 5507 msg->im_local_addr.ia_local_lid = gid_info->gl_slid;
5509 5508 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid;
5510 5509 if (gid_info->gl_redirected == B_TRUE) {
5511 5510 if (gid_info->gl_redirect_dlid != 0) {
5512 5511 msg->im_local_addr.ia_remote_lid =
5513 5512 gid_info->gl_redirect_dlid;
5514 5513 }
5515 5514 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
5516 5515 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
5517 5516 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
5518 5517 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
5519 5518 } else {
5520 5519 msg->im_local_addr.ia_remote_qno = 1;
5521 5520 msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
5522 5521 msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
5523 5522 msg->im_local_addr.ia_service_level = gid_info->gl_SL;
5524 5523 }
5525 5524
5526 5525 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
5527 5526 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1;
5528 5527 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT;
5529 5528 hdr->ClassVersion = IB_DM_CLASS_VERSION_1;
5530 5529 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET;
5531 5530 hdr->Status = 0;
5532 5531 hdr->TransactionID = h2b64(gid_info->gl_transactionID);
5533 5532 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
5534 5533 hdr->AttributeModifier = h2b32(ioc_no + 1);
5535 5534
5536 5535 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5537 5536 cb_args = &ioc_info->ioc_cb_args;
5538 5537 cb_args->cb_gid_info = gid_info;
5539 5538 cb_args->cb_retry_count = ibdm_dft_retry_cnt;
5540 5539 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO;
5541 5540 cb_args->cb_ioc_num = ioc_no;
5542 5541
5543 5542 mutex_enter(&gid_info->gl_mutex);
5544 5543 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
5545 5544 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
5546 5545 mutex_exit(&gid_info->gl_mutex);
5547 5546
5548 5547 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
5549 5548 "timeout %x", ioc_info->ioc_timeout_id);
5550 5549
5551 5550 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
5552 5551 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
5553 5552 IBTF_DPRINTF_L2("ibdm",
5554 5553 "\tsend_ioc_profile: msg transport failed");
5555 5554 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
5556 5555 }
5557 5556 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5558 5557 return (IBDM_SUCCESS);
5559 5558 }
5560 5559
5561 5560
5562 5561 /*
5563 5562 * ibdm_port_reachable
5564 5563 * Returns B_TRUE if the port GID is reachable by sending
5565 5564 * a SA query to get the NODE record for this port GUID.
5566 5565 */
5567 5566 static boolean_t
5568 5567 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid)
5569 5568 {
5570 5569 sa_node_record_t *resp;
5571 5570 size_t length;
5572 5571
5573 5572 /*
5574 5573 * Verify if it's reachable by getting the node record.
5575 5574 */
5576 5575 if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) ==
5577 5576 IBDM_SUCCESS) {
5578 5577 kmem_free(resp, length);
5579 5578 return (B_TRUE);
5580 5579 }
5581 5580 return (B_FALSE);
5582 5581 }
5583 5582
5584 5583 /*
5585 5584 * ibdm_get_node_record_by_port
5586 5585 * Sends a SA query to get the NODE record for port GUID
5587 5586 * Returns IBDM_SUCCESS if the port GID is reachable.
5588 5587 *
5589 5588 * Note: the caller must be responsible for freeing the resource
5590 5589 * by calling kmem_free(resp, length) later.
5591 5590 */
5592 5591 static int
5593 5592 ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
5594 5593 sa_node_record_t **resp, size_t *length)
5595 5594 {
5596 5595 sa_node_record_t req;
5597 5596 ibmf_saa_access_args_t args;
5598 5597 int ret;
5599 5598 ASSERT(resp != NULL && length != NULL);
5600 5599
5601 5600 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
5602 5601 guid);
5603 5602
5604 5603 bzero(&req, sizeof (sa_node_record_t));
5605 5604 req.NodeInfo.PortGUID = guid;
5606 5605
5607 5606 args.sq_attr_id = SA_NODERECORD_ATTRID;
5608 5607 args.sq_access_type = IBMF_SAA_RETRIEVE;
5609 5608 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID;
5610 5609 args.sq_template = &req;
5611 5610 args.sq_callback = NULL;
5612 5611 args.sq_callback_arg = NULL;
5613 5612
5614 5613 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp);
5615 5614 if (ret != IBMF_SUCCESS) {
5616 5615 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
5617 5616 " SA Retrieve Failed: %d", ret);
5618 5617 return (IBDM_FAILURE);
5619 5618 }
5620 5619 if (*resp == NULL || *length == 0) {
5621 5620 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
5622 5621 return (IBDM_FAILURE);
5623 5622 }
5624 5623 /*
5625 5624 * There is one NodeRecord on each endport on a subnet.
5626 5625 */
5627 5626 ASSERT(*length == sizeof (sa_node_record_t));
5628 5627
5629 5628 return (IBDM_SUCCESS);
5630 5629 }
5631 5630
5632 5631
5633 5632 /*
5634 5633 * Update the gidlist for all affected IOCs when GID becomes
5635 5634 * available/unavailable.
5636 5635 *
5637 5636 * Parameters :
5638 5637 * gidinfo - Incoming / Outgoing GID.
5639 5638 * add_flag - 1 for GID added, 0 for GID removed.
5640 5639 * - (-1) : IOC gid list updated, ioc_list required.
5641 5640 *
5642 5641 * This function gets the GID for the node GUID corresponding to the
5643 5642 * port GID. Gets the IOU info
5644 5643 */
5645 5644 static ibdm_ioc_info_t *
5646 5645 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
5647 5646 {
5648 5647 ibdm_dp_gidinfo_t *node_gid = NULL;
5649 5648 uint8_t niocs, ii;
5650 5649 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp;
5651 5650
5652 5651 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");
5653 5652
5654 5653 switch (avail_flag) {
5655 5654 case 1 :
5656 5655 node_gid = ibdm_check_dest_nodeguid(gid_info);
5657 5656 break;
5658 5657 case 0 :
5659 5658 node_gid = ibdm_handle_gid_rm(gid_info);
5660 5659 break;
5661 5660 case -1 :
5662 5661 node_gid = gid_info;
5663 5662 break;
5664 5663 default :
5665 5664 break;
5666 5665 }
5667 5666
5668 5667 if (node_gid == NULL) {
5669 5668 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
5670 5669 "No node GID found, port gid 0x%p, avail_flag %d",
5671 5670 gid_info, avail_flag);
5672 5671 return (NULL);
5673 5672 }
5674 5673
5675 5674 mutex_enter(&node_gid->gl_mutex);
5676 5675 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
5677 5676 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
5678 5677 node_gid->gl_iou == NULL) {
5679 5678 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
5680 5679 "gl_state %x, gl_iou %p", node_gid->gl_state,
5681 5680 node_gid->gl_iou);
5682 5681 mutex_exit(&node_gid->gl_mutex);
5683 5682 return (NULL);
5684 5683 }
5685 5684
5686 5685 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
5687 5686 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
5688 5687 niocs);
5689 5688 for (ii = 0; ii < niocs; ii++) {
5690 5689 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
5691 5690 /*
5692 5691 * Skip IOCs for which probe is not complete or
5693 5692 * reprobe is progress
5694 5693 */
5695 5694 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5696 5695 tmp = ibdm_dup_ioc_info(ioc, node_gid);
5697 5696 tmp->ioc_info_updated.ib_gid_prop_updated = 1;
5698 5697 tmp->ioc_next = ioc_list;
5699 5698 ioc_list = tmp;
5700 5699 }
5701 5700 }
5702 5701 mutex_exit(&node_gid->gl_mutex);
5703 5702
5704 5703 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
5705 5704 ioc_list);
5706 5705 return (ioc_list);
5707 5706 }
5708 5707
5709 5708 /*
5710 5709 * ibdm_saa_event_cb :
5711 5710 * Event handling which does *not* require ibdm_hl_mutex to be
5712 5711 * held are executed in the same thread. This is to prevent
5713 5712 * deadlocks with HCA port down notifications which hold the
5714 5713 * ibdm_hl_mutex.
5715 5714 *
5716 5715 * GID_AVAILABLE event is handled here. A taskq is spawned to
5717 5716 * handle GID_UNAVAILABLE.
5718 5717 *
5719 5718 * A new mutex ibdm_ibnex_mutex has been introduced to protect
5720 5719 * ibnex_callback. This has been done to prevent any possible
5721 5720 * deadlock (described above) while handling GID_AVAILABLE.
5722 5721 *
5723 5722 * IBMF calls the event callback for a HCA port. The SA handle
5724 5723 * for this port would be valid, till the callback returns.
5725 5724 * IBDM calling IBDM using the above SA handle should be valid.
5726 5725 *
5727 5726 * IBDM will additionally check (SA handle != NULL), before
5728 5727 * calling IBMF.
5729 5728 */
5730 5729 /*ARGSUSED*/
5731 5730 static void
5732 5731 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
5733 5732 ibmf_saa_subnet_event_t ibmf_saa_event,
5734 5733 ibmf_saa_event_details_t *event_details, void *callback_arg)
5735 5734 {
5736 5735 ibdm_saa_event_arg_t *event_arg;
5737 5736 ib_gid_t sgid, dgid;
5738 5737 ibdm_port_attr_t *hca_port;
5739 5738 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL;
5740 5739 sa_node_record_t *nrec;
5741 5740 size_t length;
5742 5741
5743 5742 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5744 5743
5745 5744 hca_port = (ibdm_port_attr_t *)callback_arg;
5746 5745
5747 5746 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
5748 5747 ibmf_saa_handle, ibmf_saa_event, event_details,
5749 5748 callback_arg);
5750 5749
5751 5750 #ifdef DEBUG
5752 5751 if (ibdm_ignore_saa_event)
5753 5752 return;
5754 5753 #endif
5755 5754
5756 5755 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
5757 5756 /*
5758 5757 * Ensure no other probe / sweep fabric is in
5759 5758 * progress.
5760 5759 */
5761 5760 mutex_enter(&ibdm.ibdm_mutex);
5762 5761 while (ibdm.ibdm_busy & IBDM_BUSY)
5763 5762 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5764 5763 ibdm.ibdm_busy |= IBDM_BUSY;
5765 5764 mutex_exit(&ibdm.ibdm_mutex);
5766 5765
5767 5766 /*
5768 5767 * If we already know about this GID, return.
5769 5768 * GID_AVAILABLE may be reported for multiple HCA
5770 5769 * ports.
5771 5770 */
5772 5771 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
5773 5772 event_details->ie_gid.gid_prefix)) != NULL) {
5774 5773 mutex_enter(&ibdm.ibdm_mutex);
5775 5774 ibdm.ibdm_busy &= ~IBDM_BUSY;
5776 5775 cv_broadcast(&ibdm.ibdm_busy_cv);
5777 5776 mutex_exit(&ibdm.ibdm_mutex);
5778 5777 return;
5779 5778 }
5780 5779
5781 5780 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5782 5781 "Insertion notified",
5783 5782 event_details->ie_gid.gid_prefix,
5784 5783 event_details->ie_gid.gid_guid);
5785 5784
5786 5785 /* This is a new gid, insert it to GID list */
5787 5786 sgid.gid_prefix = hca_port->pa_sn_prefix;
5788 5787 sgid.gid_guid = hca_port->pa_port_guid;
5789 5788 dgid.gid_prefix = event_details->ie_gid.gid_prefix;
5790 5789 dgid.gid_guid = event_details->ie_gid.gid_guid;
5791 5790 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
5792 5791 if (gid_info == NULL) {
5793 5792 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
5794 5793 "create_gid_info returned NULL");
5795 5794 mutex_enter(&ibdm.ibdm_mutex);
5796 5795 ibdm.ibdm_busy &= ~IBDM_BUSY;
5797 5796 cv_broadcast(&ibdm.ibdm_busy_cv);
5798 5797 mutex_exit(&ibdm.ibdm_mutex);
5799 5798 return;
5800 5799 }
5801 5800 mutex_enter(&gid_info->gl_mutex);
5802 5801 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
5803 5802 mutex_exit(&gid_info->gl_mutex);
5804 5803
5805 5804 /* Get the node GUID */
5806 5805 if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid,
5807 5806 &nrec, &length) != IBDM_SUCCESS) {
5808 5807 /*
5809 5808 * Set the state to PROBE_NOT_DONE for the
5810 5809 * next sweep to probe it
5811 5810 */
5812 5811 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
5813 5812 "Skipping GID : port GUID not found");
5814 5813 mutex_enter(&gid_info->gl_mutex);
5815 5814 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5816 5815 mutex_exit(&gid_info->gl_mutex);
5817 5816 mutex_enter(&ibdm.ibdm_mutex);
5818 5817 ibdm.ibdm_busy &= ~IBDM_BUSY;
5819 5818 cv_broadcast(&ibdm.ibdm_busy_cv);
5820 5819 mutex_exit(&ibdm.ibdm_mutex);
5821 5820 return;
5822 5821 }
5823 5822 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
5824 5823 gid_info->gl_devid = nrec->NodeInfo.DeviceID;
5825 5824 kmem_free(nrec, length);
5826 5825 gid_info->gl_portguid = dgid.gid_guid;
5827 5826
5828 5827 /*
5829 5828 * Get the gid info with the same node GUID.
5830 5829 */
5831 5830 mutex_enter(&ibdm.ibdm_mutex);
5832 5831 node_gid_info = ibdm.ibdm_dp_gidlist_head;
5833 5832 while (node_gid_info) {
5834 5833 if (node_gid_info->gl_nodeguid ==
5835 5834 gid_info->gl_nodeguid &&
5836 5835 node_gid_info->gl_iou != NULL) {
5837 5836 break;
5838 5837 }
5839 5838 node_gid_info = node_gid_info->gl_next;
5840 5839 }
5841 5840 mutex_exit(&ibdm.ibdm_mutex);
5842 5841
5843 5842 /*
5844 5843 * Handling a new GID requires filling of gl_hca_list.
5845 5844 * This require ibdm hca_list to be parsed and hence
5846 5845 * holding the ibdm_hl_mutex. Spawning a new thread to
5847 5846 * handle this.
5848 5847 */
5849 5848 if (node_gid_info == NULL) {
5850 5849 if (taskq_dispatch(system_taskq,
5851 5850 ibdm_saa_handle_new_gid, (void *)gid_info,
5852 5851 TQ_NOSLEEP) == NULL) {
5853 5852 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5854 5853 "new_gid taskq_dispatch failed");
5855 5854 return;
5856 5855 }
5857 5856 }
5858 5857
5859 5858 mutex_enter(&ibdm.ibdm_mutex);
5860 5859 ibdm.ibdm_busy &= ~IBDM_BUSY;
5861 5860 cv_broadcast(&ibdm.ibdm_busy_cv);
5862 5861 mutex_exit(&ibdm.ibdm_mutex);
5863 5862 return;
5864 5863 }
5865 5864
5866 5865 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
5867 5866 return;
5868 5867
5869 5868 /*
5870 5869 * GID UNAVAIL EVENT: Try to locate the GID in the GID list.
5871 5870 * If we don't find it we just return.
5872 5871 */
5873 5872 mutex_enter(&ibdm.ibdm_mutex);
5874 5873 gid_info = ibdm.ibdm_dp_gidlist_head;
5875 5874 while (gid_info) {
5876 5875 if (gid_info->gl_portguid ==
5877 5876 event_details->ie_gid.gid_guid) {
5878 5877 break;
5879 5878 }
5880 5879 gid_info = gid_info->gl_next;
5881 5880 }
5882 5881 mutex_exit(&ibdm.ibdm_mutex);
5883 5882 if (gid_info == NULL) {
5884 5883 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5885 5884 "GID for GUID %llX not found during GID UNAVAIL event",
5886 5885 event_details->ie_gid.gid_guid);
5887 5886 return;
5888 5887 }
5889 5888
5890 5889 /*
5891 5890 * If this GID is DM capable, we'll have to check whether this DGID
5892 5891 * is reachable via another port.
5893 5892 */
5894 5893 if (gid_info->gl_is_dm_capable == B_TRUE) {
5895 5894 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
5896 5895 sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
5897 5896 event_arg->ibmf_saa_handle = ibmf_saa_handle;
5898 5897 event_arg->ibmf_saa_event = ibmf_saa_event;
5899 5898 bcopy(event_details, &event_arg->event_details,
5900 5899 sizeof (ibmf_saa_event_details_t));
5901 5900 event_arg->callback_arg = callback_arg;
5902 5901
5903 5902 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
5904 5903 (void *)event_arg, TQ_NOSLEEP) == NULL) {
5905 5904 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5906 5905 "taskq_dispatch failed");
5907 5906 ibdm_free_saa_event_arg(event_arg);
5908 5907 return;
5909 5908 }
5910 5909 }
5911 5910 }
5912 5911
5913 5912 /*
5914 5913 * Handle a new GID discovered by GID_AVAILABLE saa event.
5915 5914 */
5916 5915 void
5917 5916 ibdm_saa_handle_new_gid(void *arg)
5918 5917 {
5919 5918 ibdm_dp_gidinfo_t *gid_info;
5920 5919 ibdm_hca_list_t *hca_list = NULL;
5921 5920 ibdm_port_attr_t *port = NULL;
5922 5921 ibdm_ioc_info_t *ioc_list = NULL;
5923 5922
5924 5923 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);
5925 5924
5926 5925 gid_info = (ibdm_dp_gidinfo_t *)arg;
5927 5926
5928 5927 /*
5929 5928 * Ensure that no other sweep / probe has completed
5930 5929 * probing this gid.
5931 5930 */
5932 5931 mutex_enter(&gid_info->gl_mutex);
5933 5932 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
5934 5933 mutex_exit(&gid_info->gl_mutex);
5935 5934 return;
5936 5935 }
5937 5936 mutex_exit(&gid_info->gl_mutex);
5938 5937
5939 5938 /*
5940 5939 * Parse HCAs to fill gl_hca_list
5941 5940 */
5942 5941 mutex_enter(&ibdm.ibdm_hl_mutex);
5943 5942 for (ibdm_get_next_port(&hca_list, &port, 1); port;
5944 5943 ibdm_get_next_port(&hca_list, &port, 1)) {
5945 5944 if (ibdm_port_reachable(port->pa_sa_hdl,
5946 5945 gid_info->gl_portguid) == B_TRUE) {
5947 5946 ibdm_addto_glhcalist(gid_info, hca_list);
5948 5947 }
5949 5948 }
5950 5949 mutex_exit(&ibdm.ibdm_hl_mutex);
5951 5950
5952 5951 /*
5953 5952 * Ensure no other probe / sweep fabric is in
5954 5953 * progress.
5955 5954 */
5956 5955 mutex_enter(&ibdm.ibdm_mutex);
5957 5956 while (ibdm.ibdm_busy & IBDM_BUSY)
5958 5957 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5959 5958 ibdm.ibdm_busy |= IBDM_BUSY;
5960 5959 mutex_exit(&ibdm.ibdm_mutex);
5961 5960
5962 5961 /*
5963 5962 * New IOU probe it, to check if new IOCs
5964 5963 */
5965 5964 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
5966 5965 "new GID : probing");
5967 5966 mutex_enter(&ibdm.ibdm_mutex);
5968 5967 ibdm.ibdm_ngid_probes_in_progress++;
5969 5968 mutex_exit(&ibdm.ibdm_mutex);
5970 5969 mutex_enter(&gid_info->gl_mutex);
5971 5970 gid_info->gl_reprobe_flag = 0;
5972 5971 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5973 5972 mutex_exit(&gid_info->gl_mutex);
5974 5973 ibdm_probe_gid_thread((void *)gid_info);
5975 5974
5976 5975 mutex_enter(&ibdm.ibdm_mutex);
5977 5976 ibdm_wait_probe_completion();
5978 5977 mutex_exit(&ibdm.ibdm_mutex);
5979 5978
5980 5979 if (gid_info->gl_iou == NULL) {
5981 5980 mutex_enter(&ibdm.ibdm_mutex);
5982 5981 ibdm.ibdm_busy &= ~IBDM_BUSY;
5983 5982 cv_broadcast(&ibdm.ibdm_busy_cv);
5984 5983 mutex_exit(&ibdm.ibdm_mutex);
5985 5984 return;
5986 5985 }
5987 5986
5988 5987 /*
5989 5988 * Update GID list in all IOCs affected by this
5990 5989 */
5991 5990 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);
5992 5991
5993 5992 /*
5994 5993 * Pass on the IOCs with updated GIDs to IBnexus
5995 5994 */
5996 5995 if (ioc_list) {
5997 5996 mutex_enter(&ibdm.ibdm_ibnex_mutex);
5998 5997 if (ibdm.ibdm_ibnex_callback != NULL) {
5999 5998 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6000 5999 IBDM_EVENT_IOC_PROP_UPDATE);
6001 6000 }
6002 6001 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6003 6002 }
6004 6003
6005 6004 mutex_enter(&ibdm.ibdm_mutex);
6006 6005 ibdm.ibdm_busy &= ~IBDM_BUSY;
6007 6006 cv_broadcast(&ibdm.ibdm_busy_cv);
6008 6007 mutex_exit(&ibdm.ibdm_mutex);
6009 6008 }
6010 6009
6011 6010 /*
6012 6011 * ibdm_saa_event_taskq :
6013 6012 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
6014 6013 * held. The GID_UNAVAILABLE handling is done in a taskq to
6015 6014 * prevent deadlocks with HCA port down notifications which hold
6016 6015 * ibdm_hl_mutex.
6017 6016 */
6018 6017 void
6019 6018 ibdm_saa_event_taskq(void *arg)
6020 6019 {
6021 6020 ibdm_saa_event_arg_t *event_arg;
6022 6021 ibmf_saa_handle_t ibmf_saa_handle;
6023 6022 ibmf_saa_subnet_event_t ibmf_saa_event;
6024 6023 ibmf_saa_event_details_t *event_details;
6025 6024 void *callback_arg;
6026 6025
6027 6026 ibdm_dp_gidinfo_t *gid_info;
6028 6027 ibdm_port_attr_t *hca_port, *port = NULL;
6029 6028 ibdm_hca_list_t *hca_list = NULL;
6030 6029 int sa_handle_valid = 0;
6031 6030 ibdm_ioc_info_t *ioc_list = NULL;
6032 6031
6033 6032 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
6034 6033
6035 6034 event_arg = (ibdm_saa_event_arg_t *)arg;
6036 6035 ibmf_saa_handle = event_arg->ibmf_saa_handle;
6037 6036 ibmf_saa_event = event_arg->ibmf_saa_event;
6038 6037 event_details = &event_arg->event_details;
6039 6038 callback_arg = event_arg->callback_arg;
6040 6039
6041 6040 ASSERT(callback_arg != NULL);
6042 6041 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
6043 6042 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
6044 6043 ibmf_saa_handle, ibmf_saa_event, event_details,
6045 6044 callback_arg);
6046 6045
6047 6046 hca_port = (ibdm_port_attr_t *)callback_arg;
6048 6047
6049 6048 /* Check if the port_attr is still valid */
6050 6049 mutex_enter(&ibdm.ibdm_hl_mutex);
6051 6050 for (ibdm_get_next_port(&hca_list, &port, 0); port;
6052 6051 ibdm_get_next_port(&hca_list, &port, 0)) {
6053 6052 if (port == hca_port && port->pa_port_guid ==
6054 6053 hca_port->pa_port_guid) {
6055 6054 if (ibmf_saa_handle == hca_port->pa_sa_hdl)
6056 6055 sa_handle_valid = 1;
6057 6056 break;
6058 6057 }
6059 6058 }
6060 6059 mutex_exit(&ibdm.ibdm_hl_mutex);
6061 6060 if (sa_handle_valid == 0) {
6062 6061 ibdm_free_saa_event_arg(event_arg);
6063 6062 return;
6064 6063 }
6065 6064
6066 6065 if (hca_port && (hca_port->pa_sa_hdl == NULL ||
6067 6066 ibmf_saa_handle != hca_port->pa_sa_hdl)) {
6068 6067 ibdm_free_saa_event_arg(event_arg);
6069 6068 return;
6070 6069 }
6071 6070 hca_list = NULL;
6072 6071 port = NULL;
6073 6072
6074 6073 /*
6075 6074 * Check if the GID is visible to other HCA ports.
6076 6075 * Return if so.
6077 6076 */
6078 6077 mutex_enter(&ibdm.ibdm_hl_mutex);
6079 6078 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6080 6079 ibdm_get_next_port(&hca_list, &port, 1)) {
6081 6080 if (ibdm_port_reachable(port->pa_sa_hdl,
6082 6081 event_details->ie_gid.gid_guid) == B_TRUE) {
6083 6082 mutex_exit(&ibdm.ibdm_hl_mutex);
6084 6083 ibdm_free_saa_event_arg(event_arg);
6085 6084 return;
6086 6085 }
6087 6086 }
6088 6087 mutex_exit(&ibdm.ibdm_hl_mutex);
6089 6088
6090 6089 /*
6091 6090 * Ensure no other probe / sweep fabric is in
6092 6091 * progress.
6093 6092 */
6094 6093 mutex_enter(&ibdm.ibdm_mutex);
6095 6094 while (ibdm.ibdm_busy & IBDM_BUSY)
6096 6095 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
6097 6096 ibdm.ibdm_busy |= IBDM_BUSY;
6098 6097 mutex_exit(&ibdm.ibdm_mutex);
6099 6098
6100 6099 /*
6101 6100 * If this GID is no longer in GID list, return
6102 6101 * GID_UNAVAILABLE may be reported for multiple HCA
6103 6102 * ports.
6104 6103 */
6105 6104 mutex_enter(&ibdm.ibdm_mutex);
6106 6105 gid_info = ibdm.ibdm_dp_gidlist_head;
6107 6106 while (gid_info) {
6108 6107 if (gid_info->gl_portguid ==
6109 6108 event_details->ie_gid.gid_guid) {
6110 6109 break;
6111 6110 }
6112 6111 gid_info = gid_info->gl_next;
6113 6112 }
6114 6113 mutex_exit(&ibdm.ibdm_mutex);
6115 6114 if (gid_info == NULL) {
6116 6115 mutex_enter(&ibdm.ibdm_mutex);
6117 6116 ibdm.ibdm_busy &= ~IBDM_BUSY;
6118 6117 cv_broadcast(&ibdm.ibdm_busy_cv);
6119 6118 mutex_exit(&ibdm.ibdm_mutex);
6120 6119 ibdm_free_saa_event_arg(event_arg);
6121 6120 return;
6122 6121 }
6123 6122
6124 6123 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
6125 6124 "Unavailable notification",
6126 6125 event_details->ie_gid.gid_prefix,
6127 6126 event_details->ie_gid.gid_guid);
6128 6127
6129 6128 /*
6130 6129 * Update GID list in all IOCs affected by this
6131 6130 */
6132 6131 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
6133 6132 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
6134 6133 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6135 6134
6136 6135 /*
6137 6136 * Remove GID from the global GID list
6138 6137 * Handle the case where all port GIDs for an
6139 6138 * IOU have been hot-removed. Check both gid_info
6140 6139 * & ioc_info for checking ngids.
6141 6140 */
6142 6141 mutex_enter(&ibdm.ibdm_mutex);
6143 6142 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6144 6143 mutex_enter(&gid_info->gl_mutex);
6145 6144 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6146 6145 mutex_exit(&gid_info->gl_mutex);
6147 6146 }
6148 6147 if (gid_info->gl_prev != NULL)
6149 6148 gid_info->gl_prev->gl_next = gid_info->gl_next;
6150 6149 if (gid_info->gl_next != NULL)
6151 6150 gid_info->gl_next->gl_prev = gid_info->gl_prev;
6152 6151
6153 6152 if (gid_info == ibdm.ibdm_dp_gidlist_head)
6154 6153 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6155 6154 if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6156 6155 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6157 6156 ibdm.ibdm_ngids--;
6158 6157
6159 6158 ibdm.ibdm_busy &= ~IBDM_BUSY;
6160 6159 cv_broadcast(&ibdm.ibdm_busy_cv);
6161 6160 mutex_exit(&ibdm.ibdm_mutex);
6162 6161
6163 6162 /* free the hca_list on this gid_info */
6164 6163 ibdm_delete_glhca_list(gid_info);
6165 6164
6166 6165 mutex_destroy(&gid_info->gl_mutex);
6167 6166 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6168 6167
6169 6168 /*
6170 6169 * Pass on the IOCs with updated GIDs to IBnexus
6171 6170 */
6172 6171 if (ioc_list) {
6173 6172 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
6174 6173 "IOC_PROP_UPDATE for %p\n", ioc_list);
6175 6174 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6176 6175 if (ibdm.ibdm_ibnex_callback != NULL) {
6177 6176 (*ibdm.ibdm_ibnex_callback)((void *)
6178 6177 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6179 6178 }
6180 6179 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6181 6180 }
6182 6181
6183 6182 ibdm_free_saa_event_arg(event_arg);
6184 6183 }
6185 6184
6186 6185
6187 6186 static int
6188 6187 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
6189 6188 {
6190 6189 ibdm_gid_t *scan_new, *scan_prev;
6191 6190 int cmp_failed = 0;
6192 6191
6193 6192 ASSERT(new != NULL);
6194 6193 ASSERT(prev != NULL);
6195 6194
6196 6195 /*
6197 6196 * Search for each new gid anywhere in the prev GID list.
6198 6197 * Note that the gid list could have been re-ordered.
6199 6198 */
6200 6199 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
6201 6200 for (scan_prev = prev, cmp_failed = 1; scan_prev;
6202 6201 scan_prev = scan_prev->gid_next) {
6203 6202 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
6204 6203 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
6205 6204 cmp_failed = 0;
6206 6205 break;
6207 6206 }
6208 6207 }
6209 6208
6210 6209 if (cmp_failed)
6211 6210 return (1);
6212 6211 }
6213 6212 return (0);
6214 6213 }
6215 6214
6216 6215 /*
6217 6216 * This is always called in a single thread
6218 6217 * This function updates the gid_list and serv_list of IOC
6219 6218 * The current gid_list is in ioc_info_t(contains only port
6220 6219 * guids for which probe is done) & gidinfo_t(other port gids)
6221 6220 * The gids in both locations are used for comparision.
6222 6221 */
6223 6222 static void
6224 6223 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
6225 6224 {
6226 6225 ibdm_gid_t *cur_gid_list;
6227 6226 uint_t cur_nportgids;
6228 6227
6229 6228 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
6230 6229
6231 6230 ioc->ioc_info_updated.ib_prop_updated = 0;
6232 6231
6233 6232
6234 6233 /* Current GID list in gid_info only */
6235 6234 cur_gid_list = gidinfo->gl_gid;
6236 6235 cur_nportgids = gidinfo->gl_ngids;
6237 6236
6238 6237 if (ioc->ioc_prev_serv_cnt !=
6239 6238 ioc->ioc_profile.ioc_service_entries ||
6240 6239 ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0],
6241 6240 ioc->ioc_prev_serv_cnt))
6242 6241 ioc->ioc_info_updated.ib_srv_prop_updated = 1;
6243 6242
6244 6243 if (ioc->ioc_prev_nportgids != cur_nportgids ||
6245 6244 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
6246 6245 ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6247 6246 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
6248 6247 ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6249 6248 }
6250 6249
6251 6250 /* Zero out previous entries */
6252 6251 ibdm_free_gid_list(ioc->ioc_prev_gid_list);
6253 6252 if (ioc->ioc_prev_serv)
6254 6253 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
6255 6254 sizeof (ibdm_srvents_info_t));
6256 6255 ioc->ioc_prev_serv_cnt = 0;
6257 6256 ioc->ioc_prev_nportgids = 0;
6258 6257 ioc->ioc_prev_serv = NULL;
6259 6258 ioc->ioc_prev_gid_list = NULL;
6260 6259 }
6261 6260
6262 6261 /*
6263 6262 * Handle GID removal. This returns gid_info of an GID for the same
6264 6263 * node GUID, if found. For an GID with IOU information, the same
6265 6264 * gid_info is returned if no gid_info with same node_guid is found.
6266 6265 */
6267 6266 static ibdm_dp_gidinfo_t *
6268 6267 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
6269 6268 {
6270 6269 ibdm_dp_gidinfo_t *gid_list;
6271 6270
6272 6271 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);
6273 6272
6274 6273 if (rm_gid->gl_iou == NULL) {
6275 6274 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
6276 6275 /*
6277 6276 * Search for a GID with same node_guid and
6278 6277 * gl_iou != NULL
6279 6278 */
6280 6279 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6281 6280 gid_list = gid_list->gl_next) {
6282 6281 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
6283 6282 == rm_gid->gl_nodeguid))
6284 6283 break;
6285 6284 }
6286 6285
6287 6286 if (gid_list)
6288 6287 ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6289 6288
6290 6289 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6291 6290 return (gid_list);
6292 6291 } else {
6293 6292 /*
6294 6293 * Search for a GID with same node_guid and
6295 6294 * gl_iou == NULL
6296 6295 */
6297 6296 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
6298 6297 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6299 6298 gid_list = gid_list->gl_next) {
6300 6299 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
6301 6300 == rm_gid->gl_nodeguid))
6302 6301 break;
6303 6302 }
6304 6303
6305 6304 if (gid_list) {
6306 6305 /*
6307 6306 * Copy the following fields from rm_gid :
6308 6307 * 1. gl_state
6309 6308 * 2. gl_iou
6310 6309 * 3. gl_gid & gl_ngids
6311 6310 *
6312 6311 * Note : Function is synchronized by
6313 6312 * ibdm_busy flag.
6314 6313 *
6315 6314 * Note : Redirect info is initialized if
6316 6315 * any MADs for the GID fail
6317 6316 */
6318 6317 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
6319 6318 "copying info to GID with gl_iou != NULl");
6320 6319 gid_list->gl_state = rm_gid->gl_state;
6321 6320 gid_list->gl_iou = rm_gid->gl_iou;
6322 6321 gid_list->gl_gid = rm_gid->gl_gid;
6323 6322 gid_list->gl_ngids = rm_gid->gl_ngids;
6324 6323
6325 6324 /* Remove the GID from gl_gid list */
6326 6325 ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6327 6326 } else {
6328 6327 /*
6329 6328 * Handle a case where all GIDs to the IOU have
6330 6329 * been removed.
6331 6330 */
6332 6331 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
6333 6332 "to IOU");
6334 6333
6335 6334 ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
6336 6335 return (rm_gid);
6337 6336 }
6338 6337 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6339 6338 return (gid_list);
6340 6339 }
6341 6340 }
6342 6341
6343 6342 static void
6344 6343 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
6345 6344 ibdm_dp_gidinfo_t *rm_gid)
6346 6345 {
6347 6346 ibdm_gid_t *tmp, *prev;
6348 6347
6349 6348 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
6350 6349 gid_info, rm_gid);
6351 6350
6352 6351 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
6353 6352 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
6354 6353 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
6355 6354 if (prev == NULL)
6356 6355 gid_info->gl_gid = tmp->gid_next;
6357 6356 else
6358 6357 prev->gid_next = tmp->gid_next;
6359 6358
6360 6359 kmem_free(tmp, sizeof (ibdm_gid_t));
6361 6360 gid_info->gl_ngids--;
6362 6361 break;
6363 6362 } else {
6364 6363 prev = tmp;
6365 6364 tmp = tmp->gid_next;
6366 6365 }
6367 6366 }
6368 6367 }
6369 6368
6370 6369 static void
6371 6370 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
6372 6371 {
6373 6372 ibdm_gid_t *head = NULL, *new, *tail;
6374 6373
6375 6374 /* First copy the destination */
6376 6375 for (; dest; dest = dest->gid_next) {
6377 6376 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6378 6377 new->gid_dgid_hi = dest->gid_dgid_hi;
6379 6378 new->gid_dgid_lo = dest->gid_dgid_lo;
6380 6379 new->gid_next = head;
6381 6380 head = new;
6382 6381 }
6383 6382
6384 6383 /* Insert this to the source */
6385 6384 if (*src_ptr == NULL)
6386 6385 *src_ptr = head;
6387 6386 else {
6388 6387 for (tail = *src_ptr; tail->gid_next != NULL;
6389 6388 tail = tail->gid_next)
6390 6389 ;
6391 6390
6392 6391 tail->gid_next = head;
6393 6392 }
6394 6393 }
6395 6394
6396 6395 static void
6397 6396 ibdm_free_gid_list(ibdm_gid_t *head)
6398 6397 {
6399 6398 ibdm_gid_t *delete;
6400 6399
6401 6400 for (delete = head; delete; ) {
6402 6401 head = delete->gid_next;
6403 6402 kmem_free(delete, sizeof (ibdm_gid_t));
6404 6403 delete = head;
6405 6404 }
6406 6405 }
6407 6406
6408 6407 /*
6409 6408 * This function rescans the DM capable GIDs (gl_state is
6410 6409 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
6411 6410 * basically checks if the DM capable GID is reachable. If
6412 6411 * not this is handled the same way as GID_UNAVAILABLE,
6413 6412 * except that notifications are not send to IBnexus.
6414 6413 *
6415 6414 * This function also initializes the ioc_prev_list for
6416 6415 * a particular IOC (when called from probe_ioc, with
6417 6416 * ioc_guidp != NULL) or all IOCs for the gid (called from
6418 6417 * sweep_fabric, ioc_guidp == NULL).
6419 6418 */
6420 6419 static void
6421 6420 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
6422 6421 {
6423 6422 ibdm_dp_gidinfo_t *gid_info, *tmp;
6424 6423 int ii, niocs, found;
6425 6424 ibdm_hca_list_t *hca_list = NULL;
6426 6425 ibdm_port_attr_t *port = NULL;
6427 6426 ibdm_ioc_info_t *ioc_list;
6428 6427
6429 6428 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6430 6429 found = 0;
6431 6430 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
6432 6431 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
6433 6432 gid_info = gid_info->gl_next;
6434 6433 continue;
6435 6434 }
6436 6435
6437 6436 /*
6438 6437 * Check if the GID is visible to any HCA ports.
6439 6438 * Return if so.
6440 6439 */
6441 6440 mutex_enter(&ibdm.ibdm_hl_mutex);
6442 6441 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6443 6442 ibdm_get_next_port(&hca_list, &port, 1)) {
6444 6443 if (ibdm_port_reachable(port->pa_sa_hdl,
6445 6444 gid_info->gl_dgid_lo) == B_TRUE) {
6446 6445 found = 1;
6447 6446 break;
6448 6447 }
6449 6448 }
6450 6449 mutex_exit(&ibdm.ibdm_hl_mutex);
6451 6450
6452 6451 if (found) {
6453 6452 if (gid_info->gl_iou == NULL) {
6454 6453 gid_info = gid_info->gl_next;
6455 6454 continue;
6456 6455 }
6457 6456
6458 6457 /* Intialize the ioc_prev_gid_list */
6459 6458 niocs =
6460 6459 gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
6461 6460 for (ii = 0; ii < niocs; ii++) {
6462 6461 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);
6463 6462
6464 6463 if (ioc_guidp == NULL || (*ioc_guidp ==
6465 6464 ioc_list->ioc_profile.ioc_guid)) {
6466 6465 /* Add info of GIDs in gid_info also */
6467 6466 ibdm_addto_gidlist(
6468 6467 &ioc_list->ioc_prev_gid_list,
6469 6468 gid_info->gl_gid);
6470 6469 ioc_list->ioc_prev_nportgids =
6471 6470 gid_info->gl_ngids;
6472 6471 }
6473 6472 }
6474 6473 gid_info = gid_info->gl_next;
6475 6474 continue;
6476 6475 }
6477 6476
6478 6477 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6479 6478 "deleted port GUID %llx",
6480 6479 gid_info->gl_dgid_lo);
6481 6480
6482 6481 /*
6483 6482 * Update GID list in all IOCs affected by this
6484 6483 */
6485 6484 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6486 6485
6487 6486 /*
6488 6487 * Remove GID from the global GID list
6489 6488 * Handle the case where all port GIDs for an
6490 6489 * IOU have been hot-removed.
6491 6490 */
6492 6491 mutex_enter(&ibdm.ibdm_mutex);
6493 6492 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6494 6493 mutex_enter(&gid_info->gl_mutex);
6495 6494 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6496 6495 mutex_exit(&gid_info->gl_mutex);
6497 6496 }
6498 6497
6499 6498 tmp = gid_info->gl_next;
6500 6499 if (gid_info->gl_prev != NULL)
6501 6500 gid_info->gl_prev->gl_next = gid_info->gl_next;
6502 6501 if (gid_info->gl_next != NULL)
6503 6502 gid_info->gl_next->gl_prev = gid_info->gl_prev;
6504 6503
6505 6504 if (gid_info == ibdm.ibdm_dp_gidlist_head)
6506 6505 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6507 6506 if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6508 6507 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6509 6508 ibdm.ibdm_ngids--;
6510 6509 mutex_exit(&ibdm.ibdm_mutex);
6511 6510
6512 6511 /* free the hca_list on this gid_info */
6513 6512 ibdm_delete_glhca_list(gid_info);
6514 6513
6515 6514 mutex_destroy(&gid_info->gl_mutex);
6516 6515 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6517 6516
6518 6517 gid_info = tmp;
6519 6518
6520 6519 /*
6521 6520 * Pass on the IOCs with updated GIDs to IBnexus
6522 6521 */
6523 6522 if (ioc_list) {
6524 6523 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6525 6524 "IOC_PROP_UPDATE for %p\n", ioc_list);
6526 6525 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6527 6526 if (ibdm.ibdm_ibnex_callback != NULL) {
6528 6527 (*ibdm.ibdm_ibnex_callback)((void *)
6529 6528 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6530 6529 }
6531 6530 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6532 6531 }
6533 6532 }
6534 6533 }
6535 6534
6536 6535 /*
6537 6536 * This function notifies IBnex of IOCs on this GID.
6538 6537 * Notification is for GIDs with gl_reprobe_flag set.
6539 6538 * The flag is set when IOC probe / fabric sweep
6540 6539 * probes a GID starting from CLASS port info.
6541 6540 *
6542 6541 * IBnexus will have information of a reconnected IOC
6543 6542 * if it had probed it before. If this is a new IOC,
6544 6543 * IBnexus ignores the notification.
6545 6544 *
6546 6545 * This function should be called with no locks held.
6547 6546 */
6548 6547 static void
6549 6548 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
6550 6549 {
6551 6550 ibdm_ioc_info_t *ioc_list;
6552 6551
6553 6552 if (gid_info->gl_reprobe_flag == 0 ||
6554 6553 gid_info->gl_iou == NULL)
6555 6554 return;
6556 6555
6557 6556 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);
6558 6557
6559 6558 /*
6560 6559 * Pass on the IOCs with updated GIDs to IBnexus
6561 6560 */
6562 6561 if (ioc_list) {
6563 6562 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6564 6563 if (ibdm.ibdm_ibnex_callback != NULL) {
6565 6564 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6566 6565 IBDM_EVENT_IOC_PROP_UPDATE);
6567 6566 }
6568 6567 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6569 6568 }
6570 6569 }
6571 6570
6572 6571
6573 6572 static void
6574 6573 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
6575 6574 {
6576 6575 if (arg != NULL)
6577 6576 kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
6578 6577 }
6579 6578
6580 6579 /*
6581 6580 * This function parses the list of HCAs and HCA ports
6582 6581 * to return the port_attr of the next HCA port. A port
6583 6582 * connected to IB fabric (port_state active) is returned,
6584 6583 * if connected_flag is set.
6585 6584 */
6586 6585 static void
6587 6586 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
6588 6587 ibdm_port_attr_t **inp_portp, int connect_flag)
6589 6588 {
6590 6589 int ii;
6591 6590 ibdm_port_attr_t *port, *next_port = NULL;
6592 6591 ibdm_port_attr_t *inp_port;
6593 6592 ibdm_hca_list_t *hca_list;
6594 6593 int found = 0;
6595 6594
6596 6595 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6597 6596 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
6598 6597 inp_hcap, inp_portp, connect_flag);
6599 6598
6600 6599 hca_list = *inp_hcap;
6601 6600 inp_port = *inp_portp;
6602 6601
6603 6602 if (hca_list == NULL)
6604 6603 hca_list = ibdm.ibdm_hca_list_head;
6605 6604
6606 6605 for (; hca_list; hca_list = hca_list->hl_next) {
6607 6606 for (ii = 0; ii < hca_list->hl_nports; ii++) {
6608 6607 port = &hca_list->hl_port_attr[ii];
6609 6608
6610 6609 /*
6611 6610 * inp_port != NULL;
6612 6611 * Skip till we find the matching port
6613 6612 */
6614 6613 if (inp_port && !found) {
6615 6614 if (inp_port == port)
6616 6615 found = 1;
6617 6616 continue;
6618 6617 }
6619 6618
6620 6619 if (!connect_flag) {
6621 6620 next_port = port;
6622 6621 break;
6623 6622 }
6624 6623
6625 6624 if (port->pa_sa_hdl == NULL)
6626 6625 ibdm_initialize_port(port);
6627 6626 if (port->pa_sa_hdl == NULL)
6628 6627 (void) ibdm_fini_port(port);
6629 6628 else if (next_port == NULL &&
6630 6629 port->pa_sa_hdl != NULL &&
6631 6630 port->pa_state == IBT_PORT_ACTIVE) {
6632 6631 next_port = port;
6633 6632 break;
6634 6633 }
6635 6634 }
6636 6635
6637 6636 if (next_port)
6638 6637 break;
6639 6638 }
6640 6639
6641 6640 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
6642 6641 "returns hca_list %p port %p", hca_list, next_port);
6643 6642 *inp_hcap = hca_list;
6644 6643 *inp_portp = next_port;
6645 6644 }
6646 6645
6647 6646 static void
6648 6647 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
6649 6648 {
6650 6649 ibdm_gid_t *tmp;
6651 6650
6652 6651 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6653 6652 tmp->gid_dgid_hi = addgid->gl_dgid_hi;
6654 6653 tmp->gid_dgid_lo = addgid->gl_dgid_lo;
6655 6654
6656 6655 mutex_enter(&nodegid->gl_mutex);
6657 6656 tmp->gid_next = nodegid->gl_gid;
6658 6657 nodegid->gl_gid = tmp;
6659 6658 nodegid->gl_ngids++;
6660 6659 mutex_exit(&nodegid->gl_mutex);
6661 6660 }
6662 6661
6663 6662 static void
6664 6663 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
6665 6664 ibdm_hca_list_t *hca)
6666 6665 {
6667 6666 ibdm_hca_list_t *head, *prev = NULL, *temp;
6668 6667
6669 6668 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
6670 6669 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
6671 6670 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6672 6671
6673 6672 mutex_enter(&gid_info->gl_mutex);
6674 6673 head = gid_info->gl_hca_list;
6675 6674 if (head == NULL) {
6676 6675 head = ibdm_dup_hca_attr(hca);
6677 6676 head->hl_next = NULL;
6678 6677 gid_info->gl_hca_list = head;
6679 6678 mutex_exit(&gid_info->gl_mutex);
6680 6679 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6681 6680 "gid %p, gl_hca_list %p", gid_info,
6682 6681 gid_info->gl_hca_list);
6683 6682 return;
6684 6683 }
6685 6684
6686 6685 /* Check if already in the list */
6687 6686 while (head) {
6688 6687 if (head->hl_hca_guid == hca->hl_hca_guid) {
6689 6688 mutex_exit(&gid_info->gl_mutex);
6690 6689 IBTF_DPRINTF_L4(ibdm_string,
6691 6690 "\taddto_glhcalist : gid %p hca %p dup",
6692 6691 gid_info, hca);
6693 6692 return;
6694 6693 }
6695 6694 prev = head;
6696 6695 head = head->hl_next;
6697 6696 }
6698 6697
6699 6698 /* Add this HCA to gl_hca_list */
6700 6699 temp = ibdm_dup_hca_attr(hca);
6701 6700 temp->hl_next = NULL;
6702 6701 prev->hl_next = temp;
6703 6702 mutex_exit(&gid_info->gl_mutex);
6704 6703
6705 6704 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6706 6705 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
6707 6706 }
6708 6707
6709 6708 static void
6710 6709 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
6711 6710 {
6712 6711 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6713 6712 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6714 6713
6715 6714 mutex_enter(&gid_info->gl_mutex);
6716 6715 if (gid_info->gl_hca_list)
6717 6716 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
6718 6717 gid_info->gl_hca_list = NULL;
6719 6718 mutex_exit(&gid_info->gl_mutex);
6720 6719 }
6721 6720
6722 6721
6723 6722 static void
6724 6723 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
6725 6724 {
6726 6725 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
6727 6726 port_sa_hdl);
6728 6727
6729 6728 if (ibdm_enumerate_iocs == 0)
6730 6729 return;
6731 6730
6732 6731 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6733 6732 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6734 6733
6735 6734 /* Check : Not busy in another probe / sweep */
6736 6735 mutex_enter(&ibdm.ibdm_mutex);
6737 6736 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
6738 6737 ibdm_dp_gidinfo_t *gid_info;
6739 6738
6740 6739 ibdm.ibdm_busy |= IBDM_BUSY;
6741 6740 mutex_exit(&ibdm.ibdm_mutex);
6742 6741
6743 6742 /*
6744 6743 * Check if any GID is using the SA & IBMF handle
6745 6744 * of HCA port going down. Reset ibdm_dp_gidinfo_t
6746 6745 * using another HCA port which can reach the GID.
6747 6746 * This is for DM capable GIDs only, no need to do
6748 6747 * this for others
6749 6748 *
6750 6749 * Delete the GID if no alternate HCA port to reach
6751 6750 * it is found.
6752 6751 */
6753 6752 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6754 6753 ibdm_dp_gidinfo_t *tmp;
6755 6754
6756 6755 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
6757 6756 "checking gidinfo %p", gid_info);
6758 6757
6759 6758 if (gid_info->gl_sa_hdl == port_sa_hdl) {
6760 6759 IBTF_DPRINTF_L3(ibdm_string,
6761 6760 "\tevent_hdlr: down HCA port hdl "
6762 6761 "matches gid %p", gid_info);
6763 6762
6764 6763 /*
6765 6764 * The non-DM GIDs can come back
6766 6765 * with a new subnet prefix, when
6767 6766 * the HCA port commes up again. To
6768 6767 * avoid issues, delete non-DM
6769 6768 * capable GIDs, if the gid was
6770 6769 * discovered using the HCA port
6771 6770 * going down. This is ensured by
6772 6771 * setting gl_disconnected to 1.
6773 6772 */
6774 6773 if (gid_info->gl_is_dm_capable == B_FALSE)
6775 6774 gid_info->gl_disconnected = 1;
6776 6775 else
6777 6776 ibdm_reset_gidinfo(gid_info);
6778 6777
6779 6778 if (gid_info->gl_disconnected) {
6780 6779 IBTF_DPRINTF_L3(ibdm_string,
6781 6780 "\tevent_hdlr: deleting"
6782 6781 " gid %p", gid_info);
6783 6782 tmp = gid_info;
6784 6783 gid_info = gid_info->gl_next;
6785 6784 ibdm_delete_gidinfo(tmp);
6786 6785 } else
6787 6786 gid_info = gid_info->gl_next;
6788 6787 } else
6789 6788 gid_info = gid_info->gl_next;
6790 6789 }
6791 6790
6792 6791 mutex_enter(&ibdm.ibdm_mutex);
6793 6792 ibdm.ibdm_busy &= ~IBDM_BUSY;
6794 6793 cv_signal(&ibdm.ibdm_busy_cv);
6795 6794 }
6796 6795 mutex_exit(&ibdm.ibdm_mutex);
6797 6796 }
6798 6797
6799 6798 static void
6800 6799 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6801 6800 {
6802 6801 ibdm_hca_list_t *hca_list = NULL;
6803 6802 ibdm_port_attr_t *port = NULL;
6804 6803 int gid_reinited = 0;
6805 6804 sa_node_record_t *nr, *tmp;
6806 6805 sa_portinfo_record_t *pi;
6807 6806 size_t nr_len = 0, pi_len = 0;
6808 6807 size_t path_len;
6809 6808 ib_gid_t sgid, dgid;
6810 6809 int ret, ii, nrecords;
6811 6810 sa_path_record_t *path;
6812 6811 uint8_t npaths = 1;
6813 6812 ibdm_pkey_tbl_t *pkey_tbl;
6814 6813
6815 6814 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);
6816 6815
6817 6816 /*
6818 6817 * Get list of all the ports reachable from the local known HCA
6819 6818 * ports which are active
6820 6819 */
6821 6820 mutex_enter(&ibdm.ibdm_hl_mutex);
6822 6821 for (ibdm_get_next_port(&hca_list, &port, 1); port;
6823 6822 ibdm_get_next_port(&hca_list, &port, 1)) {
6824 6823
6825 6824
6826 6825 /*
6827 6826 * Get the path and re-populate the gidinfo.
6828 6827 * Getting the path is the same probe_ioc
6829 6828 * Init the gid info as in ibdm_create_gidinfo()
6830 6829 */
6831 6830 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
6832 6831 gidinfo->gl_nodeguid);
6833 6832 if (nr == NULL) {
6834 6833 IBTF_DPRINTF_L4(ibdm_string,
6835 6834 "\treset_gidinfo : no records");
6836 6835 continue;
6837 6836 }
6838 6837
6839 6838 nrecords = (nr_len / sizeof (sa_node_record_t));
6840 6839 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) {
6841 6840 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
6842 6841 break;
6843 6842 }
6844 6843
6845 6844 if (ii == nrecords) {
6846 6845 IBTF_DPRINTF_L4(ibdm_string,
6847 6846 "\treset_gidinfo : no record for portguid");
6848 6847 kmem_free(nr, nr_len);
6849 6848 continue;
6850 6849 }
6851 6850
6852 6851 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
6853 6852 if (pi == NULL) {
6854 6853 IBTF_DPRINTF_L4(ibdm_string,
6855 6854 "\treset_gidinfo : no portinfo");
6856 6855 kmem_free(nr, nr_len);
6857 6856 continue;
6858 6857 }
6859 6858
6860 6859 sgid.gid_prefix = port->pa_sn_prefix;
6861 6860 sgid.gid_guid = port->pa_port_guid;
6862 6861 dgid.gid_prefix = pi->PortInfo.GidPrefix;
6863 6862 dgid.gid_guid = tmp->NodeInfo.PortGUID;
6864 6863
6865 6864 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
6866 6865 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);
6867 6866
6868 6867 if ((ret != IBMF_SUCCESS) || path == NULL) {
6869 6868 IBTF_DPRINTF_L4(ibdm_string,
6870 6869 "\treset_gidinfo : no paths");
6871 6870 kmem_free(pi, pi_len);
6872 6871 kmem_free(nr, nr_len);
6873 6872 continue;
6874 6873 }
6875 6874
6876 6875 gidinfo->gl_dgid_hi = path->DGID.gid_prefix;
6877 6876 gidinfo->gl_dgid_lo = path->DGID.gid_guid;
6878 6877 gidinfo->gl_sgid_hi = path->SGID.gid_prefix;
6879 6878 gidinfo->gl_sgid_lo = path->SGID.gid_guid;
6880 6879 gidinfo->gl_p_key = path->P_Key;
6881 6880 gidinfo->gl_sa_hdl = port->pa_sa_hdl;
6882 6881 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl;
6883 6882 gidinfo->gl_slid = path->SLID;
6884 6883 gidinfo->gl_dlid = path->DLID;
6885 6884 /* Reset redirect info, next MAD will set if redirected */
6886 6885 gidinfo->gl_redirected = 0;
6887 6886 gidinfo->gl_devid = (*tmp).NodeInfo.DeviceID;
6888 6887 gidinfo->gl_SL = path->SL;
6889 6888
6890 6889 gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
6891 6890 for (ii = 0; ii < port->pa_npkeys; ii++) {
6892 6891 if (port->pa_pkey_tbl == NULL)
6893 6892 break;
6894 6893
6895 6894 pkey_tbl = &port->pa_pkey_tbl[ii];
6896 6895 if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) &&
6897 6896 (pkey_tbl->pt_qp_hdl != NULL)) {
6898 6897 gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
6899 6898 break;
6900 6899 }
6901 6900 }
6902 6901
6903 6902 if (gidinfo->gl_qp_hdl == NULL)
6904 6903 IBTF_DPRINTF_L2(ibdm_string,
6905 6904 "\treset_gid_info: No matching Pkey");
6906 6905 else
6907 6906 gid_reinited = 1;
6908 6907
6909 6908 kmem_free(path, path_len);
6910 6909 kmem_free(pi, pi_len);
6911 6910 kmem_free(nr, nr_len);
6912 6911 break;
6913 6912 }
6914 6913 mutex_exit(&ibdm.ibdm_hl_mutex);
6915 6914
6916 6915 if (!gid_reinited)
6917 6916 gidinfo->gl_disconnected = 1;
6918 6917 }
6919 6918
6920 6919 static void
6921 6920 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6922 6921 {
6923 6922 ibdm_ioc_info_t *ioc_list;
6924 6923 int in_gidlist = 0;
6925 6924
6926 6925 /*
6927 6926 * Check if gidinfo has been inserted into the
6928 6927 * ibdm_dp_gidlist_head list. gl_next or gl_prev
6929 6928 * != NULL, if gidinfo is the list.
6930 6929 */
6931 6930 if (gidinfo->gl_prev != NULL ||
6932 6931 gidinfo->gl_next != NULL ||
6933 6932 ibdm.ibdm_dp_gidlist_head == gidinfo)
6934 6933 in_gidlist = 1;
6935 6934
6936 6935 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);
6937 6936
6938 6937 /*
6939 6938 * Remove GID from the global GID list
6940 6939 * Handle the case where all port GIDs for an
6941 6940 * IOU have been hot-removed.
6942 6941 */
6943 6942 mutex_enter(&ibdm.ibdm_mutex);
6944 6943 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
6945 6944 mutex_enter(&gidinfo->gl_mutex);
6946 6945 (void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou);
6947 6946 mutex_exit(&gidinfo->gl_mutex);
6948 6947 }
6949 6948
6950 6949 /* Delete gl_hca_list */
6951 6950 mutex_exit(&ibdm.ibdm_mutex);
6952 6951 ibdm_delete_glhca_list(gidinfo);
6953 6952 mutex_enter(&ibdm.ibdm_mutex);
6954 6953
6955 6954 if (in_gidlist) {
6956 6955 if (gidinfo->gl_prev != NULL)
6957 6956 gidinfo->gl_prev->gl_next = gidinfo->gl_next;
6958 6957 if (gidinfo->gl_next != NULL)
6959 6958 gidinfo->gl_next->gl_prev = gidinfo->gl_prev;
6960 6959
6961 6960 if (gidinfo == ibdm.ibdm_dp_gidlist_head)
6962 6961 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
6963 6962 if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
6964 6963 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
6965 6964 ibdm.ibdm_ngids--;
6966 6965 }
6967 6966 mutex_exit(&ibdm.ibdm_mutex);
6968 6967
6969 6968 mutex_destroy(&gidinfo->gl_mutex);
6970 6969 cv_destroy(&gidinfo->gl_probe_cv);
6971 6970 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));
6972 6971
6973 6972 /*
6974 6973 * Pass on the IOCs with updated GIDs to IBnexus
6975 6974 */
6976 6975 if (ioc_list) {
6977 6976 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
6978 6977 "IOC_PROP_UPDATE for %p\n", ioc_list);
6979 6978 mutex_enter(&ibdm.ibdm_ibnex_mutex);
6980 6979 if (ibdm.ibdm_ibnex_callback != NULL) {
6981 6980 (*ibdm.ibdm_ibnex_callback)((void *)
6982 6981 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6983 6982 }
6984 6983 mutex_exit(&ibdm.ibdm_ibnex_mutex);
6985 6984 }
6986 6985 }
6987 6986
6988 6987
6989 6988 static void
6990 6989 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
6991 6990 {
6992 6991 uint32_t attr_mod;
6993 6992
6994 6993 attr_mod = (cb_args->cb_ioc_num + 1) << 16;
6995 6994 attr_mod |= cb_args->cb_srvents_start;
6996 6995 attr_mod |= (cb_args->cb_srvents_end) << 8;
6997 6996 hdr->AttributeModifier = h2b32(attr_mod);
6998 6997 }
6999 6998
7000 6999 static void
7001 7000 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
7002 7001 {
7003 7002 ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
7004 7003 gid_info->gl_transactionID++;
7005 7004 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
7006 7005 IBTF_DPRINTF_L4(ibdm_string,
7007 7006 "\tbump_transactionID(%p), wrapup", gid_info);
7008 7007 gid_info->gl_transactionID = gid_info->gl_min_transactionID;
7009 7008 }
7010 7009 }
7011 7010
7012 7011 /*
7013 7012 * gl_prev_iou is set for *non-reprobe* sweeep requests, which
7014 7013 * detected that ChangeID in IOU info has changed. The service
7015 7014 * entry also may have changed. Check if service entry in IOC
7016 7015 * has changed wrt the prev iou, if so notify to IB Nexus.
7017 7016 */
7018 7017 static ibdm_ioc_info_t *
7019 7018 ibdm_handle_prev_iou()
7020 7019 {
7021 7020 ibdm_dp_gidinfo_t *gid_info;
7022 7021 ibdm_ioc_info_t *ioc_list_head = NULL, *ioc_list;
7023 7022 ibdm_ioc_info_t *prev_ioc, *ioc;
7024 7023 int ii, jj, niocs, prev_niocs;
7025 7024
7026 7025 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
7027 7026
7028 7027 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter");
7029 7028 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
7030 7029 gid_info = gid_info->gl_next) {
7031 7030 if (gid_info->gl_prev_iou == NULL)
7032 7031 continue;
7033 7032
7034 7033 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p",
7035 7034 gid_info);
7036 7035 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
7037 7036 prev_niocs =
7038 7037 gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots;
7039 7038 for (ii = 0; ii < niocs; ii++) {
7040 7039 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
7041 7040
7042 7041 /* Find matching IOC */
7043 7042 for (jj = 0; jj < prev_niocs; jj++) {
7044 7043 prev_ioc = (ibdm_ioc_info_t *)
7045 7044 &gid_info->gl_prev_iou->iou_ioc_info[jj];
7046 7045 if (prev_ioc->ioc_profile.ioc_guid ==
7047 7046 ioc->ioc_profile.ioc_guid)
7048 7047 break;
7049 7048 }
7050 7049 if (jj == prev_niocs)
7051 7050 prev_ioc = NULL;
7052 7051 if (ioc == NULL || prev_ioc == NULL)
7053 7052 continue;
7054 7053 if ((ioc->ioc_profile.ioc_service_entries !=
7055 7054 prev_ioc->ioc_profile.ioc_service_entries) ||
7056 7055 ibdm_serv_cmp(&ioc->ioc_serv[0],
7057 7056 &prev_ioc->ioc_serv[0],
7058 7057 ioc->ioc_profile.ioc_service_entries) != 0) {
7059 7058 IBTF_DPRINTF_L4(ibdm_string,
7060 7059 "/thandle_prev_iou modified IOC: "
7061 7060 "current ioc %p, old ioc %p",
7062 7061 ioc, prev_ioc);
7063 7062 mutex_enter(&gid_info->gl_mutex);
7064 7063 ioc_list = ibdm_dup_ioc_info(ioc, gid_info);
7065 7064 mutex_exit(&gid_info->gl_mutex);
7066 7065 ioc_list->ioc_info_updated.ib_prop_updated
7067 7066 = 0;
7068 7067 ioc_list->ioc_info_updated.ib_srv_prop_updated
7069 7068 = 1;
7070 7069
7071 7070 if (ioc_list_head == NULL)
7072 7071 ioc_list_head = ioc_list;
7073 7072 else {
7074 7073 ioc_list_head->ioc_next = ioc_list;
7075 7074 ioc_list_head = ioc_list;
7076 7075 }
7077 7076 }
7078 7077 }
7079 7078
7080 7079 mutex_enter(&gid_info->gl_mutex);
7081 7080 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou);
7082 7081 mutex_exit(&gid_info->gl_mutex);
7083 7082 }
7084 7083 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p",
7085 7084 ioc_list_head);
7086 7085 return (ioc_list_head);
7087 7086 }
7088 7087
7089 7088 /*
7090 7089 * Compares two service entries lists, returns 0 if same, returns 1
7091 7090 * if no match.
7092 7091 */
7093 7092 static int
7094 7093 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2,
7095 7094 int nserv)
7096 7095 {
7097 7096 int ii;
7098 7097
7099 7098 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter");
7100 7099 for (ii = 0; ii < nserv; ii++, serv1++, serv2++) {
7101 7100 if (serv1->se_attr.srv_id != serv2->se_attr.srv_id ||
7102 7101 bcmp(serv1->se_attr.srv_name,
7103 7102 serv2->se_attr.srv_name,
7104 7103 IB_DM_MAX_SVC_NAME_LEN) != 0) {
7105 7104 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1");
7106 7105 return (1);
7107 7106 }
7108 7107 }
7109 7108 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0");
7110 7109 return (0);
7111 7110 }
7112 7111
7113 7112 /* For debugging purpose only */
7114 7113 #ifdef DEBUG
7115 7114 void
7116 7115 ibdm_dump_mad_hdr(ib_mad_hdr_t *mad_hdr)
7117 7116 {
7118 7117 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
7119 7118 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");
7120 7119
7121 7120 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x"
7122 7121 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
7123 7122 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
7124 7123 "\tR Method : 0x%x",
7125 7124 mad_hdr->ClassVersion, mad_hdr->R_Method);
7126 7125 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x"
7127 7126 "\tTransaction ID : 0x%llx",
7128 7127 b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID));
7129 7128 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x"
7130 7129 "\tAttribute Modified : 0x%lx",
7131 7130 b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier));
7132 7131 }
7133 7132
7134 7133
7135 7134 void
7136 7135 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
7137 7136 {
7138 7137 ib_mad_hdr_t *mad_hdr;
7139 7138
7140 7139 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
7141 7140 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------");
7142 7141
7143 7142 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x"
7144 7143 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
7145 7144 ibmf_msg->im_local_addr.ia_remote_lid,
7146 7145 ibmf_msg->im_local_addr.ia_remote_qno);
7147 7146 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x"
7148 7147 " SL : 0x%x", ibmf_msg->im_local_addr.ia_p_key,
7149 7148 ibmf_msg->im_local_addr.ia_q_key,
7150 7149 ibmf_msg->im_local_addr.ia_service_level);
7151 7150
7152 7151 if (flag)
7153 7152 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
7154 7153 else
7155 7154 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);
7156 7155
7157 7156 ibdm_dump_mad_hdr(mad_hdr);
7158 7157 }
7159 7158
7160 7159
7161 7160 void
7162 7161 ibdm_dump_path_info(sa_path_record_t *path)
7163 7162 {
7164 7163 IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
7165 7164 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");
7166 7165
7167 7166 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx",
7168 7167 path->DGID.gid_prefix, path->DGID.gid_guid);
7169 7168 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx",
7170 7169 path->SGID.gid_prefix, path->SGID.gid_guid);
7171 7170 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\t\tDlID : %x",
7172 7171 path->SLID, path->DLID);
7173 7172 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x\t\tSL : %x",
7174 7173 path->P_Key, path->SL);
7175 7174 }
7176 7175
7177 7176
7178 7177 void
7179 7178 ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo)
7180 7179 {
7181 7180 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
7182 7181 IBTF_DPRINTF_L4("ibdm", "\t\t --------------");
7183 7182
7184 7183 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
7185 7184 ((b2h32(classportinfo->RespTimeValue)) & 0x1F));
7186 7185
7187 7186 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%llx",
7188 7187 b2h64(classportinfo->RedirectGID_hi));
7189 7188 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%llx",
7190 7189 b2h64(classportinfo->RedirectGID_lo));
7191 7190 IBTF_DPRINTF_L4("ibdm", "\t Redirected TC : 0x%x",
7192 7191 classportinfo->RedirectTC);
7193 7192 IBTF_DPRINTF_L4("ibdm", "\t Redirected SL : 0x%x",
7194 7193 classportinfo->RedirectSL);
7195 7194 IBTF_DPRINTF_L4("ibdm", "\t Redirected FL : 0x%x",
7196 7195 classportinfo->RedirectFL);
7197 7196 IBTF_DPRINTF_L4("ibdm", "\t Redirected LID : 0x%x",
7198 7197 b2h16(classportinfo->RedirectLID));
7199 7198 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x",
7200 7199 b2h16(classportinfo->RedirectP_Key));
7201 7200 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x",
7202 7201 classportinfo->RedirectQP);
7203 7202 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x",
7204 7203 b2h32(classportinfo->RedirectQ_Key));
7205 7204 IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi : 0x%llx",
7206 7205 b2h64(classportinfo->TrapGID_hi));
7207 7206 IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo : 0x%llx",
7208 7207 b2h64(classportinfo->TrapGID_lo));
7209 7208 IBTF_DPRINTF_L4("ibdm", "\t Trap TC : 0x%x",
7210 7209 classportinfo->TrapTC);
7211 7210 IBTF_DPRINTF_L4("ibdm", "\t Trap SL : 0x%x",
7212 7211 classportinfo->TrapSL);
7213 7212 IBTF_DPRINTF_L4("ibdm", "\t Trap FL : 0x%x",
7214 7213 classportinfo->TrapFL);
7215 7214 IBTF_DPRINTF_L4("ibdm", "\t Trap LID : 0x%x",
7216 7215 b2h16(classportinfo->TrapLID));
7217 7216 IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key : 0x%x",
7218 7217 b2h16(classportinfo->TrapP_Key));
7219 7218 IBTF_DPRINTF_L4("ibdm", "\t Trap HL : 0x%x",
7220 7219 classportinfo->TrapHL);
7221 7220 IBTF_DPRINTF_L4("ibdm", "\t Trap QP : 0x%x",
7222 7221 classportinfo->TrapQP);
7223 7222 IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key : 0x%x",
7224 7223 b2h32(classportinfo->TrapQ_Key));
7225 7224 }
7226 7225
7227 7226
7228 7227 void
7229 7228 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
7230 7229 {
7231 7230 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
7232 7231 IBTF_DPRINTF_L4("ibdm", "\t\t ------------");
7233 7232
7234 7233 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x",
7235 7234 b2h16(iou_info->iou_changeid));
7236 7235 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d",
7237 7236 iou_info->iou_num_ctrl_slots);
7238 7237 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x",
7239 7238 iou_info->iou_flag);
7240 7239 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x",
7241 7240 iou_info->iou_ctrl_list[0]);
7242 7241 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x",
7243 7242 iou_info->iou_ctrl_list[1]);
7244 7243 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x",
7245 7244 iou_info->iou_ctrl_list[2]);
7246 7245 }
7247 7246
7248 7247
7249 7248 void
7250 7249 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
7251 7250 {
7252 7251 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
7253 7252 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");
7254 7253
7255 7254 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid);
7256 7255 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid);
7257 7256 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid);
7258 7257 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver);
7259 7258 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id);
7260 7259 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class);
7261 7260 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
7262 7261 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol);
7263 7262 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver);
7264 7263 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth);
7265 7264 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
7266 7265 ioc->ioc_rdma_read_qdepth);
7267 7266 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz);
7268 7267 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
7269 7268 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x",
7270 7269 ioc->ioc_ctrl_opcap_mask);
7271 7270 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries);
7272 7271 }
7273 7272
7274 7273
7275 7274 void
7276 7275 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
7277 7276 {
7278 7277 IBTF_DPRINTF_L4("ibdm",
7279 7278 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);
7280 7279
7281 7280 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
7282 7281 "Service Name : %s", srv_ents->srv_name);
7283 7282 }
7284 7283
7285 7284 int ibdm_allow_sweep_fabric_timestamp = 1;
7286 7285
7287 7286 void
7288 7287 ibdm_dump_sweep_fabric_timestamp(int flag)
7289 7288 {
7290 7289 static hrtime_t x;
7291 7290 if (flag) {
7292 7291 if (ibdm_allow_sweep_fabric_timestamp) {
7293 7292 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
7294 7293 "sweep %lld ms", ((gethrtime() - x)/ 1000000));
7295 7294 }
7296 7295 x = 0;
7297 7296 } else
7298 7297 x = gethrtime();
7299 7298 }
7300 7299 #endif
|
↓ open down ↓ |
7113 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX