1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * Copyright 2019, Joyent, Inc.
28 */
29
30 /*
31 * IB CM handlers for s Solaris SCSI RDMA Protocol Target (SRP)
32 * transport port provider module for the COMSTAR framework.
33 */
34
35 #include <sys/cpuvar.h>
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/stat.h>
39 #include <sys/file.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 #include <sys/modctl.h>
43 #include <sys/sysmacros.h>
44 #include <sys/sdt.h>
45 #include <sys/taskq.h>
46 #include <sys/ib/ibtl/ibti.h>
47
48 #include <sys/stmf.h>
49 #include <sys/stmf_ioctl.h>
50 #include <sys/portif.h>
51
52 #include "srp.h"
53 #include "srpt_impl.h"
54 #include "srpt_cm.h"
55 #include "srpt_stp.h"
56 #include "srpt_ch.h"
57
58 extern uint16_t srpt_send_msg_depth;
59 extern srpt_ctxt_t *srpt_ctxt;
60
61 /*
62 * srpt_cm_req_hdlr() - Login request
63 *
64 * CM has called back with a CM REQ message associated with an
65 * SRP initiator login request.
66 */
67 static ibt_cm_status_t
68 srpt_cm_req_hdlr(srpt_target_port_t *tgt, ibt_cm_event_t *event,
69 ibt_cm_return_args_t *ret_args, void *ret_priv_data,
70 ibt_priv_data_len_t ret_priv_data_len)
71 {
72 ibt_cm_status_t status;
73 ibt_cm_req_rcv_t *req;
74 srp_login_req_t login;
75 srp_login_rej_t login_rej;
76 srp_login_rsp_t login_rsp;
77 srpt_channel_t *ch = NULL;
78 char remote_gid[SRPT_ALIAS_LEN];
79 char local_gid[SRPT_ALIAS_LEN];
80
81 ASSERT(tgt != NULL);
82 req = &event->cm_event.req;
83
84 if (event->cm_priv_data_len < sizeof (srp_login_req_t)) {
85 SRPT_DPRINTF_L2("cm_req_hdlr, IU size expected (>= %d),"
86 " received size (%d)", (uint_t)sizeof (srp_login_req_t),
87 event->cm_priv_data_len);
88 return (IBT_CM_REJECT);
89 }
90
91 if (event->cm_priv_data == NULL) {
92 SRPT_DPRINTF_L2("cm_req_hdlr, NULL ULP private data pointer");
93 return (IBT_CM_REJECT);
94 }
95
96 if (ret_priv_data_len < sizeof (srp_login_rej_t)) {
97 SRPT_DPRINTF_L2("cm_req_hdlr, return private len too"
98 " small (%d)", ret_priv_data_len);
99 return (IBT_CM_REJECT);
100 }
101
102 if (ret_priv_data == NULL) {
103 SRPT_DPRINTF_L2("cm_req_hdlr, NULL ULP return private data"
104 " pointer");
105 return (IBT_CM_REJECT);
106 }
107
108 /*
109 * Copy to avoid potential alignment problems, process login
110 * creating a new channel and possibly session.
111 */
112 bcopy(event->cm_priv_data, &login, sizeof (login));
113
114 ALIAS_STR(local_gid,
115 req->req_prim_addr.av_sgid.gid_prefix,
116 req->req_prim_addr.av_sgid.gid_guid);
117 ALIAS_STR(remote_gid,
118 req->req_prim_addr.av_dgid.gid_prefix,
119 req->req_prim_addr.av_dgid.gid_guid);
120
121 ch = srpt_stp_login(tgt, &login, &login_rsp,
122 &login_rej, req->req_prim_hca_port, local_gid, remote_gid);
123 if (ch != NULL) {
124 bcopy(&login_rsp, ret_priv_data, SRP_LOGIN_RSP_SIZE);
125 ret_args->cm_ret_len = SRP_LOGIN_RSP_SIZE;
126
127 SRPT_DPRINTF_L3("cm_req_hdlr, rsp priv len(%d)"
128 " ch created on port(%d)"
129 ", cm_req_hdlr, req ra_out(%d), ra_in(%d)"
130 ", retry(%d)",
131 ret_args->cm_ret_len, req->req_prim_hca_port,
132 req->req_rdma_ra_out, req->req_rdma_ra_in,
133 req->req_retry_cnt);
134
135 ret_args->cm_ret.rep.cm_channel = ch->ch_chan_hdl;
136 ret_args->cm_ret.rep.cm_rdma_ra_out =
137 min(tgt->tp_ioc->ioc_attr.hca_max_rdma_out_chan,
138 req->req_rdma_ra_in);
139 ret_args->cm_ret.rep.cm_rdma_ra_in =
140 min(tgt->tp_ioc->ioc_attr.hca_max_rdma_in_chan,
141 req->req_rdma_ra_out);
142 ret_args->cm_ret.rep.cm_rnr_retry_cnt = req->req_retry_cnt;
143
144 SRPT_DPRINTF_L3("cm_req_hdlr, hca_max_rdma_in_chan (%d)"
145 ", hca_max_rdma_out_chan (%d)"
146 ", updated ra_out(%d), ra_in(%d), retry(%d)",
147 tgt->tp_ioc->ioc_attr.hca_max_rdma_in_chan,
148 tgt->tp_ioc->ioc_attr.hca_max_rdma_out_chan,
149 ret_args->cm_ret.rep.cm_rdma_ra_out,
150 ret_args->cm_ret.rep.cm_rdma_ra_in,
151 ret_args->cm_ret.rep.cm_rnr_retry_cnt);
152 status = IBT_CM_ACCEPT;
153
154 } else {
155 bcopy(&login_rej, ret_priv_data, sizeof (login_rej));
156 ret_args->cm_ret_len = sizeof (login_rej);
157 status = IBT_CM_REJECT;
158 }
159
160 return (status);
161 }
162
163 /*
164 * srpt_cm_conn_est_hdlr() - Connection established
165 *
166 * CM has called back to inform us that a connection attempt has
167 * completed (explicit or implicit) and may now be used.
168 */
169 /* ARGSUSED */
170 static ibt_cm_status_t
171 srpt_cm_conn_est_hdlr(srpt_target_port_t *tgt, ibt_cm_event_t *event)
172 {
173 srpt_channel_t *ch;
174
175 ASSERT(tgt != NULL);
176 ASSERT(event != NULL);
177
178 ch = (srpt_channel_t *)ibt_get_chan_private(event->cm_channel);
179 ASSERT(ch != NULL);
180
181 SRPT_DPRINTF_L3("cm_conn_est_hdlr, invoked for ch(%p)",
182 (void *)ch);
183
184 rw_enter(&ch->ch_rwlock, RW_WRITER);
185 if (ch->ch_state != SRPT_CHANNEL_CONNECTING &&
186 ch->ch_state != SRPT_CHANNEL_CONNECTED) {
187 SRPT_DPRINTF_L2("cm_conn_est_hdlr, invalid ch state (%d)",
188 ch->ch_state);
189 rw_exit(&ch->ch_rwlock);
190 return (IBT_CM_REJECT);
191 }
192
193 ch->ch_state = SRPT_CHANNEL_CONNECTED;
194
195 rw_exit(&ch->ch_rwlock);
196 return (IBT_CM_ACCEPT);
197 }
198
199 /*
200 * srpt_cm_conn_closed_hdlr() - Channel closed
201 *
202 * CM callback indicating a channel has been completely closed.
203 */
204 /* ARGSUSED */
205 static ibt_cm_status_t
206 srpt_cm_conn_closed_hdlr(srpt_target_port_t *tgt, ibt_cm_event_t *event)
207 {
208 ibt_cm_status_t status = IBT_CM_ACCEPT;
209 srpt_channel_t *ch;
210
211 ASSERT(tgt != NULL);
212 ASSERT(event != NULL);
213
214 ch = (srpt_channel_t *)ibt_get_chan_private(event->cm_channel);
215 ASSERT(ch != NULL);
216
217 SRPT_DPRINTF_L3("cm_conn_closed_hdlr, invoked for chan_hdl(%p),"
218 " event(%d)", (void *)ch->ch_chan_hdl,
219 event->cm_event.closed);
220
221 switch (event->cm_event.closed) {
222
223 case IBT_CM_CLOSED_DREP_RCVD:
224 case IBT_CM_CLOSED_DREQ_TIMEOUT:
225 case IBT_CM_CLOSED_DUP:
226 case IBT_CM_CLOSED_ABORT:
227 case IBT_CM_CLOSED_ALREADY:
228 /*
229 * These cases indicate the SRP target initiated
230 * the closing of the channel and it is now closed.
231 * Cleanup the channel (which will remove the targets
232 * reference) and then release CM's reference.
233 */
234 SRPT_DPRINTF_L3("cm_conn_closed_hdlr, local close call-back");
235 srpt_ch_cleanup(ch);
236 srpt_ch_release_ref(ch, 1);
237 break;
238
239 case IBT_CM_CLOSED_DREQ_RCVD:
240 case IBT_CM_CLOSED_REJ_RCVD:
241 case IBT_CM_CLOSED_STALE:
242 /*
243 * These cases indicate that the SRP initiator is closing
244 * the channel. CM will have already closed the RC channel,
245 * so simply initiate cleanup which will remove the target
246 * ports reference to the channel and then release the
247 * reference held by the CM.
248 */
249 SRPT_DPRINTF_L3("cm_conn_closed_hdlr, remote close,"
250 " free channel");
251 srpt_ch_cleanup(ch);
252 srpt_ch_release_ref(ch, 1);
253 break;
254
255 default:
256 SRPT_DPRINTF_L2("cm_conn_closed_hdlr, unknown close type (%d)",
257 event->cm_event.closed);
258 status = IBT_CM_DEFAULT;
259 break;
260 }
261 return (status);
262 }
263
264 /*
265 * srpt_cm_failure_hdlr() - Called when the channel is in error. Cleanup
266 * and release the channel.
267 */
268 static ibt_cm_status_t
269 srpt_cm_failure_hdlr(ibt_cm_event_t *event)
270 {
271 srpt_channel_t *ch;
272
273 ASSERT(event != NULL);
274
275 ch = (srpt_channel_t *)ibt_get_chan_private(event->cm_channel);
276 ASSERT(ch != NULL);
277
278 SRPT_DPRINTF_L3("cm_failure_hdlr, chan_hdl: 0x%p, code: %d"
279 "msg: %d reason: %d", (void *)event->cm_channel,
280 event->cm_event.failed.cf_code,
281 event->cm_event.failed.cf_msg,
282 event->cm_event.failed.cf_reason);
283
284 srpt_ch_cleanup(ch);
285 srpt_ch_release_ref(ch, 1);
286
287 return (IBT_CM_ACCEPT);
288 }
289
290 /*
291 * srpt_cm_hdlr() - CM call-back handler.
292 */
293 ibt_cm_status_t
294 srpt_cm_hdlr(void *cm_private, ibt_cm_event_t *event,
295 ibt_cm_return_args_t *ret_args, void *ret_priv_data,
296 ibt_priv_data_len_t ret_len_max)
297 {
298 ibt_cm_status_t status = IBT_CM_ACCEPT;
299
300 switch (event->cm_type) {
301
302 case IBT_CM_EVENT_REQ_RCV:
303 SRPT_DPRINTF_L3("cm_hdlr, REQ received");
304 status = srpt_cm_req_hdlr((srpt_target_port_t *)cm_private,
305 event, ret_args, ret_priv_data, ret_len_max);
306 break;
307
308 case IBT_CM_EVENT_REP_RCV:
309 SRPT_DPRINTF_L3("cm_hdlr, REP received");
310 break;
311
312 case IBT_CM_EVENT_MRA_RCV:
313 SRPT_DPRINTF_L3("cm_hdlr, MRA received");
314 break;
315
316 case IBT_CM_EVENT_CONN_EST:
317 SRPT_DPRINTF_L3("cm_hdlr, Connection established");
318 status = srpt_cm_conn_est_hdlr(
319 (srpt_target_port_t *)cm_private, event);
320 break;
321
322 case IBT_CM_EVENT_CONN_CLOSED:
323 SRPT_DPRINTF_L3("cm_hdlr, Connection closed");
324 status = srpt_cm_conn_closed_hdlr(
325 (srpt_target_port_t *)cm_private, event);
326 break;
327
328 case IBT_CM_EVENT_FAILURE:
329 SRPT_DPRINTF_L3("cm_hdlr, Event failure");
330 status = srpt_cm_failure_hdlr(event);
331 break;
332
333 case IBT_CM_EVENT_LAP_RCV:
334 SRPT_DPRINTF_L3("cm_hdlr, LAP received");
335 break;
336
337 case IBT_CM_EVENT_APR_RCV:
338 SRPT_DPRINTF_L3("cm_hdlr, APR received");
339 break;
340
341 default:
342 SRPT_DPRINTF_L3("cm_hdlr, unknown event received");
343 break;
344 }
345
346 return (status);
347 }