1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * hermon_agents.c
29 * Hermon InfiniBand Management Agent (SMA, PMA, BMA) routines
30 *
31 * Implements all the routines necessary for initializing, handling,
32 * and (later) tearing down all the infrastructure necessary for Hermon
33 * MAD processing.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41
42 #include <sys/ib/adapters/hermon/hermon.h>
43 #include <sys/ib/mgt/ibmf/ibmf.h>
44 #include <sys/disp.h>
45
46 static void hermon_agent_request_cb(ibmf_handle_t ibmf_handle,
47 ibmf_msg_t *msgp, void *args);
48 static void hermon_agent_handle_req(void *cb_args);
49 static void hermon_agent_response_cb(ibmf_handle_t ibmf_handle,
50 ibmf_msg_t *msgp, void *args);
51 static int hermon_agent_list_init(hermon_state_t *state);
52 static void hermon_agent_list_fini(hermon_state_t *state);
53 static int hermon_agent_register_all(hermon_state_t *state);
54 static int hermon_agent_unregister_all(hermon_state_t *state, int num_reg);
55 static void hermon_agent_mad_resp_handling(hermon_state_t *state,
56 ibmf_msg_t *msgp, uint_t port);
57
58 /*
59 * hermon_agent_handlers_init()
60 * Context: Only called from attach() and/or detach() path contexts
61 */
62 int
63 hermon_agent_handlers_init(hermon_state_t *state)
64 {
65 int status;
66 char *rsrc_name;
67
68 /* Determine if we need to register any agents with the IBMF */
69 if ((state->hs_cfg_profile->cp_qp0_agents_in_fw) &&
70 (state->hs_cfg_profile->cp_qp1_agents_in_fw)) {
71 return (DDI_SUCCESS);
72 }
73
74 /*
75 * Build a unique name for the Hermon task queue from the Hermon driver
76 * instance number and HERMON_TASKQ_NAME
77 */
78 rsrc_name = (char *)kmem_zalloc(HERMON_RSRC_NAME_MAXLEN, KM_SLEEP);
79 HERMON_RSRC_NAME(rsrc_name, HERMON_TASKQ_NAME);
80
81 /* Initialize the Hermon IB management agent list */
82 status = hermon_agent_list_init(state);
83 if (status != DDI_SUCCESS) {
84 goto agentsinit_fail;
85 }
86
87 /*
88 * Initialize the agent handling task queue. Note: We set the task
89 * queue priority to the minimum system priority. At this point this
90 * is considered acceptable because MADs are unreliable datagrams
91 * and could get lost (in general) anyway.
92 */
93 state->hs_taskq_agents = ddi_taskq_create(state->hs_dip,
94 rsrc_name, HERMON_TASKQ_NTHREADS, TASKQ_DEFAULTPRI, 0);
95 if (state->hs_taskq_agents == NULL) {
96 hermon_agent_list_fini(state);
97 goto agentsinit_fail;
98 }
99
100 /* Now attempt to register all of the agents with the IBMF */
101 status = hermon_agent_register_all(state);
102 if (status != DDI_SUCCESS) {
103 ddi_taskq_destroy(state->hs_taskq_agents);
104 hermon_agent_list_fini(state);
105 goto agentsinit_fail;
106 }
107
108 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
109 return (DDI_SUCCESS);
110
111 agentsinit_fail:
112 kmem_free(rsrc_name, HERMON_RSRC_NAME_MAXLEN);
113 return (status);
114 }
115
116
117 /*
118 * hermon_agent_handlers_fini()
119 * Context: Only called from detach() path context
120 */
121 int
122 hermon_agent_handlers_fini(hermon_state_t *state)
123 {
124 int status;
125
126 /* Determine if we need to unregister any agents from the IBMF */
127 if ((state->hs_cfg_profile->cp_qp0_agents_in_fw) &&
128 (state->hs_cfg_profile->cp_qp1_agents_in_fw)) {
129 return (DDI_SUCCESS);
130 }
131
132 /* Now attempt to unregister all of the agents from the IBMF */
133 status = hermon_agent_unregister_all(state, state->hs_num_agents);
134 if (status != DDI_SUCCESS) {
135 return (DDI_FAILURE);
136 }
137
138 /*
139 * Destroy the task queue. The task queue destroy is guaranteed to
140 * wait until any scheduled tasks have completed. We are able to
141 * guarantee that no _new_ tasks will be added the task queue while
142 * we are in the ddi_taskq_destroy() call because we have
143 * (at this point) successfully unregistered from IBMF (in
144 * hermon_agent_unregister_all() above).
145 */
146 ddi_taskq_destroy(state->hs_taskq_agents);
147
148 /* Teardown the Hermon IB management agent list */
149 hermon_agent_list_fini(state);
150
151 return (DDI_SUCCESS);
152 }
153
154
155 /*
156 * hermon_agent_request_cb()
157 * Context: Called from the IBMF context
158 */
159 static void
160 hermon_agent_request_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
161 void *args)
162 {
163 hermon_agent_handler_arg_t *cb_args;
164 hermon_agent_list_t *curr;
165 hermon_state_t *state;
166 int status;
167
168 curr = (hermon_agent_list_t *)args;
169 state = curr->agl_state;
170
171 /*
172 * Allocate space to hold the callback args (for passing to the
173 * task queue). Note: If we are unable to allocate space for the
174 * the callback args here, then we just return. But we must ensure
175 * that we call ibmf_free_msg() to free up the message.
176 */
177 cb_args = (hermon_agent_handler_arg_t *)kmem_zalloc(
178 sizeof (hermon_agent_handler_arg_t), KM_NOSLEEP);
179 if (cb_args == NULL) {
180 (void) ibmf_free_msg(ibmf_handle, &msgp);
181 return;
182 }
183
184 /* Fill in the callback args */
185 cb_args->ahd_ibmfhdl = ibmf_handle;
186 cb_args->ahd_ibmfmsg = msgp;
187 cb_args->ahd_agentlist = args;
188
189 /*
190 * Dispatch the message to the task queue. Note: Just like above,
191 * if this request fails for any reason then make sure to free up
192 * the IBMF message and then return
193 */
194 status = ddi_taskq_dispatch(state->hs_taskq_agents,
195 hermon_agent_handle_req, cb_args, DDI_NOSLEEP);
196 if (status == DDI_FAILURE) {
197 kmem_free(cb_args, sizeof (hermon_agent_handler_arg_t));
198 (void) ibmf_free_msg(ibmf_handle, &msgp);
199 }
200 }
201
202 /*
203 * hermon_get_smlid()
204 * Simple helper function for hermon_agent_handle_req() below.
205 * Get the portinfo and extract the smlid.
206 */
207 static ib_lid_t
208 hermon_get_smlid(hermon_state_t *state, uint_t port)
209 {
210 sm_portinfo_t portinfo;
211 int status;
212
213 status = hermon_getportinfo_cmd_post(state, port,
214 HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
215 if (status != HERMON_CMD_SUCCESS) {
216 cmn_err(CE_CONT, "Hermon: GetPortInfo (port %02d) command "
217 "failed: %08x\n", port, status);
218 return (0);
219 }
220 return (portinfo.MasterSMLID);
221 }
222
223 /*
224 * hermon_get_port_change_flags()
225 * Helper function to determine the changes in the incoming MAD's portinfo
226 * for the Port Change event.
227 */
228 static ibt_port_change_t
229 hermon_port_change_flags(sm_portinfo_t *curpinfo, sm_portinfo_t *madpinfo)
230 {
231 int SMDisabled, ReregSuppd;
232 ibt_port_change_t flags = 0;
233
234 SMDisabled = curpinfo->CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED;
235 ReregSuppd = curpinfo->CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
236
237 if (curpinfo->MasterSMLID != madpinfo->MasterSMLID) {
238 flags |= IBT_PORT_CHANGE_SM_LID;
239 }
240 if (curpinfo->MasterSMSL != madpinfo->MasterSMSL) {
241 flags |= IBT_PORT_CHANGE_SM_SL;
242 }
243 if (curpinfo->SubnetTimeOut != madpinfo->SubnetTimeOut) {
244 flags |= IBT_PORT_CHANGE_SUB_TIMEOUT;
245 }
246 if ((madpinfo->CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED)
247 ^ SMDisabled) {
248 flags |= IBT_PORT_CHANGE_SM_FLAG;
249 }
250 if ((madpinfo->CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD)
251 ^ ReregSuppd) {
252 flags |= IBT_PORT_CHANGE_REREG;
253 }
254 return (flags);
255 }
256
257 int
258 hermon_set_port_capability(hermon_state_t *state, uint8_t port,
259 sm_portinfo_t *portinfo, ibt_port_change_t flags)
260 {
261 uint32_t capmask;
262 int status;
263 hermon_hw_set_port_t set_port;
264
265 bzero(&set_port, sizeof (set_port));
266
267 /* Validate that specified port number is legal */
268 if (!hermon_portnum_is_valid(state, port)) {
269 return (IBT_HCA_PORT_INVALID);
270 }
271
272 /*
273 * Convert InfiniBand-defined port capability flags to the format
274 * specified by the IBTF. Specifically, we modify the capability
275 * mask based on the specified values.
276 */
277 capmask = portinfo->CapabilityMask;
278
279 if (flags & IBT_PORT_CHANGE_SM_FLAG)
280 capmask ^= SM_CAP_MASK_IS_SM;
281
282 if (flags & IBT_PORT_CHANGE_REREG)
283 capmask ^= SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
284 set_port.cap_mask = capmask;
285
286 /*
287 * Use the Hermon SET_PORT command to update the capability mask and
288 * (possibly) reset the QKey violation counter for the specified port.
289 * Note: In general, this operation shouldn't fail. If it does, then
290 * it is an indication that something (probably in HW, but maybe in
291 * SW) has gone seriously wrong.
292 */
293 status = hermon_set_port_cmd_post(state, &set_port, port,
294 HERMON_SLEEPFLAG_FOR_CONTEXT());
295 if (status != HERMON_CMD_SUCCESS) {
296 HERMON_WARNING(state, "failed to modify port capabilities");
297 cmn_err(CE_CONT, "Hermon: SET_IB (port %02d) command failed: "
298 "%08x\n", port, status);
299 return (DDI_FAILURE);
300 }
301
302 return (DDI_SUCCESS);
303 }
304
305 /*
306 * hermon_agent_handle_req()
307 * Context: Called with priority of taskQ thread
308 */
309 static void
310 hermon_agent_handle_req(void *cb_args)
311 {
312 hermon_agent_handler_arg_t *agent_args;
313 hermon_agent_list_t *curr;
314 ibc_async_event_t event;
315 ibt_async_code_t type, code;
316 sm_portinfo_t curpinfo, tmadpinfo;
317 sm_portinfo_t *madpinfop;
318 hermon_state_t *state;
319 ibmf_handle_t ibmf_handle;
320 ibmf_msg_t *msgp;
321 ibmf_msg_bufs_t *recv_msgbufp;
322 ibmf_msg_bufs_t *send_msgbufp;
323 ib_mad_hdr_t *madhdrp;
324 ibmf_retrans_t retrans;
325 uint_t port;
326 int status;
327
328 /* Extract the necessary info from the callback args parameter */
329 agent_args = (hermon_agent_handler_arg_t *)cb_args;
330 ibmf_handle = agent_args->ahd_ibmfhdl;
331 msgp = agent_args->ahd_ibmfmsg;
332 curr = agent_args->ahd_agentlist;
333 state = curr->agl_state;
334 port = curr->agl_port;
335
336 /*
337 * Set the message send buffer pointers to the message receive buffer
338 * pointers to reuse the IBMF provided buffers for the sender
339 * information.
340 */
341 recv_msgbufp = &msgp->im_msgbufs_recv;
342 send_msgbufp = &msgp->im_msgbufs_send;
343 bcopy(recv_msgbufp, send_msgbufp, sizeof (ibmf_msg_bufs_t));
344
345 /*
346 * Check if the incoming packet is a special "Hermon Trap" MAD. If it
347 * is, then do the special handling. If it isn't, then simply pass it
348 * on to the firmware and forward the response back to the IBMF.
349 *
350 * Note: Hermon has a unique method for handling internally generated
351 * Traps. All internally detected/generated Trap messages are
352 * automatically received by the IBMF (as receive completions on QP0),
353 * which (because all Hermon Trap MADs have SLID == 0) detects it as a
354 * special "Hermon Trap" and forwards it here to the driver's SMA.
355 * It is then our responsibility here to fill in the Trap MAD's DLID
356 * for forwarding to the real Master SM (as programmed in the port's
357 * PortInfo.MasterSMLID field.)
358 */
359 if (HERMON_IS_SPECIAL_TRAP_MAD(msgp)) {
360 msgp->im_local_addr.ia_remote_lid =
361 hermon_get_smlid(state, port);
362 } else {
363 int isSMSet, isReregSuppd;
364 uint_t attr_id, method, mgmt_class;
365
366 madhdrp = recv_msgbufp->im_bufs_mad_hdr;
367 method = madhdrp->R_Method;
368 attr_id = b2h16(madhdrp->AttributeID);
369 mgmt_class = madhdrp->MgmtClass;
370
371 /*
372 * Is this a Subnet Manager MAD with SET method ? If so
373 * we will have to get the current portinfo to generate
374 * events based on what has changed in portinfo.
375 */
376 isSMSet = (((mgmt_class == MAD_MGMT_CLASS_SUBN_LID_ROUTED)||
377 (mgmt_class == MAD_MGMT_CLASS_SUBN_DIRECT_ROUTE)) &&
378 (method == MAD_METHOD_SET));
379
380 /*
381 * Get the current portinfo to compare with the portinfo
382 * received in the MAD for PortChange event.
383 */
384 if (isSMSet && (attr_id == SM_PORTINFO_ATTRID) ||
385 (attr_id == SM_PKEY_TABLE_ATTRID) ||
386 (attr_id == SM_GUIDINFO_ATTRID)) {
387 madpinfop = recv_msgbufp->im_bufs_cl_data;
388 tmadpinfo = *madpinfop;
389 HERMON_GETPORTINFO_SWAP(&tmadpinfo);
390 status = hermon_getportinfo_cmd_post(state, port,
391 HERMON_SLEEPFLAG_FOR_CONTEXT(), &curpinfo);
392 if (status != HERMON_CMD_SUCCESS) {
393 cmn_err(CE_CONT, "Hermon: GetPortInfo "
394 "(port %02d) command failed: %08x\n", port,
395 status);
396 goto hermon_agent_handle_req_skip_response;
397 }
398 }
399
400 /*
401 * Post the command to the firmware (using the MAD_IFC
402 * command). Note: We also reuse the command that was passed
403 * in. We pass the pointer to the original MAD payload as if
404 * it were both the source of the incoming MAD as well as the
405 * destination for the response. This is acceptable and saves
406 * us the step of one additional copy. Note: If this command
407 * fails for any reason other than HERMON_CMD_BAD_PKT, it
408 * probably indicates a serious problem.
409 */
410 status = hermon_mad_ifc_cmd_post(state, port,
411 HERMON_CMD_SLEEP_NOSPIN,
412 (uint32_t *)recv_msgbufp->im_bufs_mad_hdr,
413 (uint32_t *)send_msgbufp->im_bufs_mad_hdr);
414 if (status != HERMON_CMD_SUCCESS) {
415 if ((status != HERMON_CMD_BAD_PKT) &&
416 (status != HERMON_CMD_INSUFF_RSRC)) {
417 cmn_err(CE_CONT, "Hermon: MAD_IFC (port %02d) "
418 "command failed: %08x\n", port, status);
419 }
420
421 /* finish cleanup */
422 goto hermon_agent_handle_req_skip_response;
423 }
424
425 if (isSMSet) {
426 event.ev_port_flags = 0;
427 type = 0;
428 event.ev_port = (uint8_t)port;
429
430 switch (attr_id) {
431 case SM_PORTINFO_ATTRID:
432 /*
433 * This is a SM SET method with portinfo
434 * attribute. If ClientRereg bit was set in
435 * the MADs portinfo this is a REREG event
436 * (see section 14.4.11 in IB Spec 1.2.1). Else
437 * compare the current (before MAD_IFC command)
438 * portinfo with the portinfo in the MAD and
439 * signal PORT_CHANGE event with the proper
440 * ev_port_flags.
441 *
442 */
443 isReregSuppd = curpinfo.CapabilityMask &
444 SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
445
446 madpinfop = recv_msgbufp->im_bufs_cl_data;
447 if (tmadpinfo.ClientRereg && isReregSuppd) {
448 type |= IBT_CLNT_REREG_EVENT;
449 }
450
451 type |= IBT_PORT_CHANGE_EVENT;
452 event.ev_port_flags = hermon_port_change_flags(
453 &curpinfo, &tmadpinfo);
454 if (event.ev_port_flags &
455 (IBT_PORT_CHANGE_REREG |
456 IBT_PORT_CHANGE_SM_FLAG)) {
457 if (hermon_set_port_capability(state,
458 port, &curpinfo,
459 event.ev_port_flags)
460 != DDI_SUCCESS) {
461 cmn_err(CE_CONT, "HERMON: Port "
462 "%d capability reset "
463 "failed\n", port);
464 }
465 }
466
467 /*
468 * If we have a SMLID change event but
469 * capability mask doesn't have Rereg support
470 * bit set, we have to do the Rereg part too.
471 */
472 if ((event.ev_port_flags &
473 IBT_PORT_CHANGE_SM_LID) && !isReregSuppd)
474 type |= IBT_CLNT_REREG_EVENT;
475 break;
476 case SM_PKEY_TABLE_ATTRID:
477 type |= IBT_PORT_CHANGE_EVENT;
478 event.ev_port_flags = IBT_PORT_CHANGE_PKEY;
479 break;
480 case SM_GUIDINFO_ATTRID:
481 type |= IBT_PORT_CHANGE_EVENT;
482 event.ev_port_flags = IBT_PORT_CHANGE_SGID;
483 break;
484 default:
485 break;
486
487 }
488
489 /*
490 * NOTE: here we call ibc_async_handler directly without
491 * using the HERMON_DO_IBTF_ASYNC_CALLB, since hermon
492 * can not be unloaded till ibmf_unregiter is done and
493 * this thread (hs_taskq_agents) will be destroyed
494 * before ibmf_uregister is called.
495 *
496 * The hermon event queue based hs_in_evcallb flag
497 * assumes that we will pick one event after another
498 * and dispatch them sequentially. If we use
499 * HERMON_DO_IBTF_ASYNC_CALLB, we will break this
500 * assumption make hs_in_evcallb inconsistent.
501 */
502 while (type != 0) {
503 if (type & IBT_PORT_CHANGE_EVENT) {
504 code = IBT_PORT_CHANGE_EVENT;
505 type &= ~IBT_PORT_CHANGE_EVENT;
506 } else {
507 code = IBT_CLNT_REREG_EVENT;
508 type = 0;
509 }
510 ibc_async_handler(state->hs_ibtfpriv, code,
511 &event);
512 }
513 }
514 }
515
516 /*
517 * If incoming MAD was "TrapRepress", then no response is necessary.
518 * Free the IBMF message and return.
519 */
520 if (HERMON_IS_TRAP_REPRESS_MAD(msgp)) {
521 goto hermon_agent_handle_req_skip_response;
522 }
523
524 /*
525 * Modify the response MAD as necessary (for any special cases).
526 * Specifically, if this MAD was a directed route MAD, then some
527 * additional packet manipulation may be necessary because the Hermon
528 * firmware does not do all the required steps to respond to the
529 * MAD.
530 */
531 hermon_agent_mad_resp_handling(state, msgp, port);
532
533 /*
534 * Send response (or forwarded "Trap" MAD) back to IBMF. We use the
535 * "response callback" to indicate when it is appropriate (later) to
536 * free the IBMF msg.
537 */
538 status = ibmf_msg_transport(ibmf_handle, IBMF_QP_HANDLE_DEFAULT,
539 msgp, &retrans, hermon_agent_response_cb, state, 0);
540 if (status != IBMF_SUCCESS) {
541 goto hermon_agent_handle_req_skip_response;
542 }
543
544 /* Free up the callback args parameter */
545 kmem_free(agent_args, sizeof (hermon_agent_handler_arg_t));
546 return;
547
548 hermon_agent_handle_req_skip_response:
549 /* Free up the ibmf message */
550 (void) ibmf_free_msg(ibmf_handle, &msgp);
551
552 /* Free up the callback args parameter */
553 kmem_free(agent_args, sizeof (hermon_agent_handler_arg_t));
554 }
555
556
557 /*
558 * hermon_agent_response_cb()
559 * Context: Called from the IBMF context
560 */
561 /* ARGSUSED */
562 static void
563 hermon_agent_response_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
564 void *args)
565 {
566 /*
567 * It is the responsibility of each IBMF callback recipient to free
568 * the packets that it has been given. Now that we are in the
569 * response callback, we can be assured that it is safe to do so.
570 */
571 (void) ibmf_free_msg(ibmf_handle, &msgp);
572 }
573
574
575 /*
576 * hermon_agent_list_init()
577 * Context: Only called from attach() path context
578 */
579 static int
580 hermon_agent_list_init(hermon_state_t *state)
581 {
582 hermon_agent_list_t *curr;
583 uint_t num_ports, num_agents, num_agents_per_port;
584 uint_t num_sma_agents = 0;
585 uint_t num_pma_agents = 0;
586 uint_t num_bma_agents = 0;
587 uint_t do_qp0, do_qp1;
588 int i, j, indx;
589
590 /*
591 * Calculate the number of registered agents for each port
592 * (SMA, PMA, and BMA) and determine whether or not to register
593 * a given agent with the IBMF (or whether to let the Hermon firmware
594 * handle it)
595 */
596 num_ports = state->hs_cfg_profile->cp_num_ports;
597 num_agents = 0;
598 num_agents_per_port = 0;
599 do_qp0 = state->hs_cfg_profile->cp_qp0_agents_in_fw;
600 do_qp1 = state->hs_cfg_profile->cp_qp1_agents_in_fw;
601 if (do_qp0 == 0) {
602 num_agents += (num_ports * HERMON_NUM_QP0_AGENTS_PER_PORT);
603 num_agents_per_port += HERMON_NUM_QP0_AGENTS_PER_PORT;
604 num_sma_agents = num_ports;
605 }
606 if (do_qp1 == 0) {
607 num_agents += (num_ports * HERMON_NUM_QP1_AGENTS_PER_PORT);
608 num_agents_per_port += HERMON_NUM_QP1_AGENTS_PER_PORT;
609 num_pma_agents = num_ports;
610 /*
611 * The following line is commented out because the Hermon
612 * firmware does not currently support a BMA. If it did,
613 * then we would want to register the agent with the IBMF.
614 * (We would also need to have HERMON_NUM_QP1_AGENTS_PER_PORT
615 * set to 2, instead of 1.)
616 *
617 * num_bma_agents = num_ports;
618 */
619 }
620
621 state->hs_num_agents = num_agents;
622
623 /*
624 * Allocate the memory for all of the agent list entries
625 */
626 state->hs_agents = (hermon_agent_list_t *)kmem_zalloc(num_agents *
627 sizeof (hermon_agent_list_t), KM_SLEEP);
628 if (state->hs_agents == NULL) {
629 return (DDI_FAILURE);
630 }
631
632 /*
633 * Fill in each of the agent list entries with the agent's
634 * MgmtClass, port number, and Hermon softstate pointer
635 */
636 indx = 0;
637 for (i = 0; i < num_agents_per_port; i++) {
638 for (j = 0; j < num_ports; j++) {
639 curr = &state->hs_agents[indx];
640 curr->agl_state = state;
641 curr->agl_port = j + 1;
642
643 if ((do_qp0 == 0) && num_sma_agents) {
644 curr->agl_mgmtclass = SUBN_AGENT;
645 num_sma_agents--;
646 indx++;
647 } else if ((do_qp1 == 0) && (num_pma_agents)) {
648 curr->agl_mgmtclass = PERF_AGENT;
649 num_pma_agents--;
650 indx++;
651 } else if ((do_qp1 == 0) && (num_bma_agents)) {
652 curr->agl_mgmtclass = BM_AGENT;
653 num_bma_agents--;
654 indx++;
655 }
656 }
657 }
658
659 return (DDI_SUCCESS);
660 }
661
662
663 /*
664 * hermon_agent_list_fini()
665 * Context: Only called from attach() and/or detach() path contexts
666 */
667 static void
668 hermon_agent_list_fini(hermon_state_t *state)
669 {
670 /* Free up the memory for the agent list entries */
671 kmem_free(state->hs_agents,
672 state->hs_num_agents * sizeof (hermon_agent_list_t));
673 }
674
675
676 /*
677 * hermon_agent_register_all()
678 * Context: Only called from attach() path context
679 */
680 static int
681 hermon_agent_register_all(hermon_state_t *state)
682 {
683 hermon_agent_list_t *curr;
684 ibmf_register_info_t ibmf_reg;
685 ibmf_impl_caps_t impl_caps;
686 ib_guid_t nodeguid;
687 int i, status, num_registered;
688
689 /* Get the Hermon NodeGUID from the softstate */
690 nodeguid = state->hs_ibtfinfo.hca_attr->hca_node_guid;
691
692 /*
693 * Register each of the agents with the IBMF (and add callbacks for
694 * each to the hermon_agent_request_cb() routine). Note: If we
695 * fail somewhere along the line here, we attempt to cleanup as much
696 * of the mess as we can and then jump to hermon_agent_unregister_all()
697 * to cleanup the rest.
698 */
699 num_registered = 0;
700
701 if (state->hs_num_agents == 0) {
702 return (DDI_SUCCESS);
703 }
704
705 for (i = 0; i < state->hs_num_agents; i++) {
706 /* Register each agent with the IBMF */
707 curr = &state->hs_agents[i];
708 ibmf_reg.ir_ci_guid = nodeguid;
709 ibmf_reg.ir_port_num = curr->agl_port;
710 ibmf_reg.ir_client_class = curr->agl_mgmtclass;
711
712 status = ibmf_register(&ibmf_reg, IBMF_VERSION, 0,
713 NULL, NULL, &curr->agl_ibmfhdl, &impl_caps);
714 if (status != IBMF_SUCCESS) {
715 goto agents_reg_fail;
716 }
717
718 /* Setup callbacks with the IBMF */
719 status = ibmf_setup_async_cb(curr->agl_ibmfhdl,
720 IBMF_QP_HANDLE_DEFAULT, hermon_agent_request_cb, curr, 0);
721 if (status != IBMF_SUCCESS) {
722 (void) ibmf_unregister(&curr->agl_ibmfhdl, 0);
723 goto agents_reg_fail;
724 }
725 num_registered++;
726 }
727
728 return (DDI_SUCCESS);
729
730 agents_reg_fail:
731 (void) hermon_agent_unregister_all(state, num_registered);
732 return (DDI_FAILURE);
733 }
734
735
736 /*
737 * hermon_agent_unregister_all()
738 * Context: Only called from detach() path context
739 */
740 static int
741 hermon_agent_unregister_all(hermon_state_t *state, int num_reg)
742 {
743 hermon_agent_list_t *curr;
744 int i, status;
745
746 if (num_reg == 0) {
747 return (DDI_SUCCESS);
748 }
749
750 /*
751 * For each registered agent in the agent list, teardown the
752 * callbacks from the IBMF and unregister.
753 */
754 for (i = 0; i < num_reg; i++) {
755 curr = &state->hs_agents[i];
756
757 /* Teardown the IBMF callback */
758 status = ibmf_tear_down_async_cb(curr->agl_ibmfhdl,
759 IBMF_QP_HANDLE_DEFAULT, 0);
760 if (status != IBMF_SUCCESS) {
761 return (DDI_FAILURE);
762 }
763
764 /* Unregister the agent from the IBMF */
765 status = ibmf_unregister(&curr->agl_ibmfhdl, 0);
766 if (status != IBMF_SUCCESS) {
767 return (DDI_FAILURE);
768 }
769 }
770
771 return (DDI_SUCCESS);
772 }
773
774
775 /*
776 * hermon_agent_mad_resp_handling()
777 * Context: Called with priority of taskQ thread
778 */
779 /* ARGSUSED */
780 static void
781 hermon_agent_mad_resp_handling(hermon_state_t *state, ibmf_msg_t *msgp,
782 uint_t port)
783 {
784 ib_mad_hdr_t *rmadhdrp = msgp->im_msgbufs_recv.im_bufs_mad_hdr;
785 ib_mad_hdr_t *smadhdrp = msgp->im_msgbufs_send.im_bufs_mad_hdr;
786 uint_t hop_count, hop_point;
787 uchar_t *resp, *ret_path;
788
789 resp = (uchar_t *)msgp->im_msgbufs_send.im_bufs_cl_data;
790
791 /*
792 * Handle directed route MADs as a special case. Hermon firmware
793 * does not update the "direction" bit, "hop pointer", "Return
794 * Path" or, in fact, any of the "directed route" parameters. So
795 * the responsibility falls on Hermon driver software to inspect the
796 * MADs and update those fields as appropriate (see section 14.2.2
797 * of the IBA specification, rev 1.1)
798 */
799 if (HERMON_MAD_IS_DR(rmadhdrp)) {
800 /*
801 * Set the "Direction" bit to one. This indicates that this
802 * is now directed route response
803 */
804 HERMON_DRMAD_SET_DIRECTION(rmadhdrp);
805
806 /* Extract the "hop pointer" and "hop count" from the MAD */
807 hop_count = HERMON_DRMAD_GET_HOPCOUNT(rmadhdrp);
808 hop_point = HERMON_DRMAD_GET_HOPPOINTER(rmadhdrp);
809
810 /* Append the port we came in on to the "Return Path" */
811 if ((hop_count != 0) && ((hop_point == hop_count) ||
812 (hop_point == hop_count + 1))) {
813 ret_path = &resp[HERMON_DRMAD_RETURN_PATH_OFFSET];
814 ret_path[hop_point] = (uchar_t)port;
815 }
816
817 /* Then increment the "hop pointer" in the MAD */
818 hop_point++;
819 HERMON_DRMAD_SET_HOPPOINTER(smadhdrp, (uint8_t)hop_point);
820 }
821 }