1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * An implementation of the IPoIB standard based on PSARC 2001/289.
28 */
29
30 #include <sys/types.h>
31 #include <sys/conf.h>
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34 #include <sys/modctl.h>
35 #include <sys/stropts.h>
36 #include <sys/stream.h>
37 #include <sys/strsun.h>
38 #include <sys/strsubr.h>
39 #include <sys/dlpi.h>
40 #include <sys/mac_provider.h>
41
42 #include <sys/pattr.h> /* for HCK_FULLCKSUM */
43 #include <sys/sysmacros.h> /* for offsetof */
44 #include <sys/disp.h> /* for async thread pri */
45 #include <sys/atomic.h> /* for atomic_add*() */
46 #include <sys/ethernet.h> /* for ETHERTYPE_IPV6 */
47 #include <netinet/in.h> /* for netinet/ip.h below */
48 #include <netinet/ip.h> /* for struct ip */
49 #include <netinet/udp.h> /* for struct udphdr */
50 #include <inet/common.h> /* for inet/ip.h below */
51 #include <inet/ip.h> /* for ipha_t */
52 #include <inet/ip6.h> /* for ip6_t */
53 #include <inet/tcp.h> /* for tcph_t */
54 #include <netinet/icmp6.h> /* for icmp6_t */
55 #include <sys/callb.h>
56 #include <sys/modhash.h>
57
58 #include <sys/ib/clients/ibd/ibd.h>
59 #include <sys/ib/mgt/sm_attr.h> /* for SM_INIT_TYPE_* */
60 #include <sys/note.h>
61 #include <sys/multidata.h>
62
63 #include <sys/ib/mgt/ibmf/ibmf.h> /* for ibd_get_portspeed */
64
65 #include <sys/priv_names.h>
66 #include <sys/dls.h>
67 #include <sys/dld_ioc.h>
68 #include <sys/policy.h>
69 #include <sys/ibpart.h>
70 #include <sys/file.h>
71
72 /*
73 * The write-up below includes details on the following:
74 * 1. The dladm administrative model.
75 * 2. Late HCA initialization feature.
76 * 3. Brussels support and its implications to the current architecture.
77 *
78 * 1. The dladm administrative model.
79 * ------------------------------------------
80 * With the dladm model, ibnex will create one ibd instance per port. These
81 * instances will be created independent of the port state.
82 *
83 * The ibd driver is two faceted: One side of it working as the port driver and
84 * the other as the partition object driver.
85 *
86 * The port instance is a child of the HCA, and will have an entry in the devfs.
87 * A DDI attach only happens for the port driver, and its attach is
88 * handled in ibd_port_attach(). Similary, a DDI detach for the port driver is
89 * handled in ibd_port_unattach().
90 *
91 * The partition object is only a registrant to the mac layer via mac_register()
92 * and does not have an entry in the device tree. There is no DDI softstate
93 * managed by the DDI framework for the partition objects. However, the state is
94 * managed inside the ibd driver, and every partition object hangs off the
95 * "ibd_objlist_head".
96 *
97 * The partition object first comes into existence when a user runs the
98 * 'create-part' subcommand of dladm. This is like invoking the attach entry
99 * point of the partition object. The partition object goes away with the
100 * 'delete-part' subcommand of dladm. This is like invoking the detach entry
101 * point of the partition object.
102 *
103 * The create-part and delete-part subcommands result in dld ioctls that end up
104 * calling ibd_create_parition() and ibd_delete_partition respectively.
105 * There ioctls are registered with the dld layer in _init() via a call to
106 * dld_ioc_register().
107 *
108 * The port instance by itself cannot be plumbed. It is only the partition
109 * objects that can be plumbed and they alone participate in I/O and not the
110 * port driver.
111 *
112 * There are some info ioctls supported in ibd which are used by dladm(1M) to
113 * display useful information. The info entry point for ibd is
114 * ibd_get_partition_info().
115 *
116 * 2. Late HCA initialization feature.
117 * ------------------------------------
118 * As mentioned in section 1, the user creates the partition objects via
119 * dladm(1M). It is possible that:
120 * a) The physical port itself is down and the SM cannot be reached.
121 * b) The PKEY specified by the used has not been created in the SM yet.
122 * c) An IPoIB broadcast group for the specified PKEY is not present.
123 *
124 * In all of the above cases, complete initialization of the partition object is
125 * not possible. However, the new model allows the creation of partition
126 * objects even in such cases but will defer the initialization for later.
127 * When such a partition object is plumbed, the link state will be displayed as
128 * "down".
129 * The driver, at this point, is listening to events that herald the
130 * availability of resources -
131 * i) LINK_UP when the link becomes available
132 * ii) PORT_CHANGE when the PKEY has been created
133 * iii) MCG_CREATED when the IPoIB broadcast group for the given pkey has been
134 * created
135 * via ibd_async_handler() for events i) and ii), and via
136 * ibd_snet_notices_handler() for iii.
137 * The driver handles these events (as and when they arrive) and completes the
138 * initialization of the partition object and transitions it to a usable state.
139 *
140 * 3. Brussels support and its implications to the current architecture.
141 * ---------------------------------------------------------------------
142 * The brussels support introduces two new interfaces to the ibd driver -
143 * ibd_m_getprop() and ibd_m_setprop().
144 * These interfaces allow setting and retrieval of certain properties.
145 * Some of them are public properties while most other are private properties
146 * meant to be used by developers. Tuning the latter kind can cause
147 * performance issues and should not be used without understanding the
148 * implications. All properties are specific to an instance of either the
149 * partition object or the port driver.
150 *
151 * The public properties are : mtu and linkmode.
152 * mtu is a read-only property.
153 * linkmode can take two values - UD and CM.
154 *
155 * Changing the linkmode requires some bookkeeping in the driver. The
156 * capabilities need to be re-reported to the mac layer. This is done by
157 * calling mac_capab_update(). The maxsdu is updated by calling
158 * mac_maxsdu_update2().
159 * The private properties retain their values across the change of linkmode.
160 * NOTE:
161 * - The port driver does not support any property apart from mtu.
162 * - All other properties are only meant for the partition object.
163 * - The properties cannot be set when an instance is plumbed. The
164 * instance has to be unplumbed to effect any setting.
165 */
166
167 /*
168 * Driver wide tunables
169 *
170 * ibd_tx_softintr
171 * ibd_rx_softintr
172 * The softintr mechanism allows ibd to avoid event queue overflows if
173 * the receive/completion handlers are to be expensive. These are enabled
174 * by default.
175 *
176 * ibd_log_sz
177 * This specifies the size of the ibd log buffer in bytes. The buffer is
178 * allocated and logging is enabled only when IBD_LOGGING is defined.
179 *
180 */
181 uint_t ibd_rx_softintr = 1;
182 uint_t ibd_tx_softintr = 1;
183
184 #ifdef IBD_LOGGING
185 uint_t ibd_log_sz = 0x20000;
186 #endif
187
188 #ifdef IBD_LOGGING
189 #define IBD_LOG_SZ ibd_log_sz
190 #endif
191
192 /* Post IBD_RX_POST_CNT receive work requests at a time. */
193 #define IBD_RX_POST_CNT 8
194
195 /* Hash into 1 << IBD_LOG_RX_POST number of rx post queues */
196 #define IBD_LOG_RX_POST 4
197
198 /* Minimum number of receive work requests driver needs to always have */
199 #define IBD_RWQE_MIN ((IBD_RX_POST_CNT << IBD_LOG_RX_POST) * 4)
200
201 /*
202 * LSO parameters
203 */
204 #define IBD_LSO_MAXLEN 65536
205 #define IBD_LSO_BUFSZ 8192
206
207 /*
208 * Async operation states
209 */
210 #define IBD_OP_NOTSTARTED 0
211 #define IBD_OP_ONGOING 1
212 #define IBD_OP_COMPLETED 2
213 #define IBD_OP_ERRORED 3
214 #define IBD_OP_ROUTERED 4
215
216 /*
217 * Start/stop in-progress flags; note that restart must always remain
218 * the OR of start and stop flag values.
219 */
220 #define IBD_DRV_START_IN_PROGRESS 0x10000000
221 #define IBD_DRV_STOP_IN_PROGRESS 0x20000000
222 #define IBD_DRV_RESTART_IN_PROGRESS 0x30000000
223 #define IBD_DRV_DELETE_IN_PROGRESS IBD_DRV_RESTART_IN_PROGRESS
224
225 /*
226 * Miscellaneous constants
227 */
228 #define IB_MGID_IPV4_LOWGRP_MASK 0xFFFFFFFF
229 #define IBD_DEF_MAX_SDU 2044
230 #define IBD_DEF_MAX_MTU (IBD_DEF_MAX_SDU + IPOIB_HDRSIZE)
231 #define IBD_DEF_RC_MAX_SDU 65520
232 #define IBD_DEF_RC_MAX_MTU (IBD_DEF_RC_MAX_SDU + IPOIB_HDRSIZE)
233 #define IBD_DEFAULT_QKEY 0xB1B
234 #ifdef IBD_LOGGING
235 #define IBD_DMAX_LINE 100
236 #endif
237
238 /*
239 * Enumerations for link states
240 */
241 typedef enum {
242 IBD_LINK_DOWN,
243 IBD_LINK_UP,
244 IBD_LINK_UP_ABSENT
245 } ibd_link_op_t;
246
247 /*
248 * Driver State Pointer
249 */
250 void *ibd_list;
251
252 /*
253 * Driver Global Data
254 */
255 ibd_global_state_t ibd_gstate;
256
257 /*
258 * Partition object list
259 */
260 ibd_state_t *ibd_objlist_head = NULL;
261 kmutex_t ibd_objlist_lock;
262
263 int ibd_rc_conn_timeout = 60 * 10; /* 10 minutes */
264
265 /*
266 * Logging
267 */
268 #ifdef IBD_LOGGING
269 kmutex_t ibd_lbuf_lock;
270 uint8_t *ibd_lbuf;
271 uint32_t ibd_lbuf_ndx;
272 #endif
273
274 /*
275 * Required system entry points
276 */
277 static int ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
278 static int ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
279
280 /*
281 * Required driver entry points for GLDv3
282 */
283 static int ibd_m_stat(void *, uint_t, uint64_t *);
284 static int ibd_m_start(void *);
285 static void ibd_m_stop(void *);
286 static int ibd_m_promisc(void *, boolean_t);
287 static int ibd_m_multicst(void *, boolean_t, const uint8_t *);
288 static int ibd_m_unicst(void *, const uint8_t *);
289 static mblk_t *ibd_m_tx(void *, mblk_t *);
290 static boolean_t ibd_m_getcapab(void *, mac_capab_t, void *);
291
292 static int ibd_m_setprop(void *, const char *, mac_prop_id_t, uint_t,
293 const void *);
294 static int ibd_m_getprop(void *, const char *, mac_prop_id_t, uint_t, void *);
295 static void ibd_m_propinfo(void *, const char *, mac_prop_id_t,
296 mac_prop_info_handle_t);
297 static int ibd_set_priv_prop(ibd_state_t *, const char *, uint_t,
298 const void *);
299 static int ibd_get_priv_prop(ibd_state_t *, const char *, uint_t, void *);
300
301 /*
302 * Private driver entry points for GLDv3
303 */
304
305 /*
306 * Initialization
307 */
308 static int ibd_state_init(ibd_state_t *, dev_info_t *);
309 static int ibd_init_txlist(ibd_state_t *);
310 static int ibd_init_rxlist(ibd_state_t *);
311 static int ibd_acache_init(ibd_state_t *);
312 #ifdef IBD_LOGGING
313 static void ibd_log_init(void);
314 #endif
315
316 /*
317 * Termination/cleanup
318 */
319 static void ibd_state_fini(ibd_state_t *);
320 static void ibd_fini_txlist(ibd_state_t *);
321 static void ibd_fini_rxlist(ibd_state_t *);
322 static void ibd_tx_cleanup(ibd_state_t *, ibd_swqe_t *);
323 static void ibd_tx_cleanup_list(ibd_state_t *, ibd_swqe_t *, ibd_swqe_t *);
324 static void ibd_acache_fini(ibd_state_t *);
325 #ifdef IBD_LOGGING
326 static void ibd_log_fini(void);
327 #endif
328
329 /*
330 * Allocation/acquire/map routines
331 */
332 static int ibd_alloc_tx_copybufs(ibd_state_t *);
333 static int ibd_alloc_rx_copybufs(ibd_state_t *);
334 static int ibd_alloc_tx_lsobufs(ibd_state_t *);
335 static ibd_swqe_t *ibd_acquire_swqe(ibd_state_t *);
336 static int ibd_acquire_lsobufs(ibd_state_t *, uint_t, ibt_wr_ds_t *,
337 uint32_t *);
338
339 /*
340 * Free/release/unmap routines
341 */
342 static void ibd_free_rwqe(ibd_state_t *, ibd_rwqe_t *);
343 static void ibd_free_tx_copybufs(ibd_state_t *);
344 static void ibd_free_rx_copybufs(ibd_state_t *);
345 static void ibd_free_rx_rsrcs(ibd_state_t *);
346 static void ibd_free_tx_lsobufs(ibd_state_t *);
347 static void ibd_release_swqe(ibd_state_t *, ibd_swqe_t *, ibd_swqe_t *, int);
348 static void ibd_release_lsobufs(ibd_state_t *, ibt_wr_ds_t *, uint32_t);
349 static void ibd_free_lsohdr(ibd_swqe_t *, mblk_t *);
350
351 /*
352 * Handlers/callback routines
353 */
354 static uint_t ibd_intr(caddr_t);
355 static uint_t ibd_tx_recycle(caddr_t);
356 static void ibd_rcq_handler(ibt_cq_hdl_t, void *);
357 static void ibd_scq_handler(ibt_cq_hdl_t, void *);
358 static void ibd_poll_rcq(ibd_state_t *, ibt_cq_hdl_t);
359 static void ibd_poll_scq(ibd_state_t *, ibt_cq_hdl_t);
360 static void ibd_drain_rcq(ibd_state_t *, ibt_cq_hdl_t);
361 static void ibd_drain_scq(ibd_state_t *, ibt_cq_hdl_t);
362 static void ibd_freemsg_cb(char *);
363 static void ibd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
364 ibt_async_event_t *);
365 static void ibdpd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
366 ibt_async_event_t *);
367 static void ibd_snet_notices_handler(void *, ib_gid_t,
368 ibt_subnet_event_code_t, ibt_subnet_event_t *);
369
370 /*
371 * Send/receive routines
372 */
373 static boolean_t ibd_send(ibd_state_t *, mblk_t *);
374 static void ibd_post_send(ibd_state_t *, ibd_swqe_t *);
375 static void ibd_post_recv(ibd_state_t *, ibd_rwqe_t *);
376 static mblk_t *ibd_process_rx(ibd_state_t *, ibd_rwqe_t *, ibt_wc_t *);
377
378 /*
379 * Threads
380 */
381 static void ibd_async_work(ibd_state_t *);
382
383 /*
384 * Async tasks
385 */
386 static void ibd_async_acache(ibd_state_t *, ipoib_mac_t *);
387 static void ibd_async_multicast(ibd_state_t *, ib_gid_t, int);
388 static void ibd_async_setprom(ibd_state_t *);
389 static void ibd_async_unsetprom(ibd_state_t *);
390 static void ibd_async_reap_group(ibd_state_t *, ibd_mce_t *, ib_gid_t, uint8_t);
391 static void ibd_async_trap(ibd_state_t *, ibd_req_t *);
392 static void ibd_async_txsched(ibd_state_t *);
393 static void ibd_async_link(ibd_state_t *, ibd_req_t *);
394
395 /*
396 * Async task helpers
397 */
398 static ibd_mce_t *ibd_async_mcache(ibd_state_t *, ipoib_mac_t *, boolean_t *);
399 static ibd_mce_t *ibd_join_group(ibd_state_t *, ib_gid_t, uint8_t);
400 static ibd_mce_t *ibd_mcache_find(ib_gid_t, struct list *);
401 static boolean_t ibd_get_allroutergroup(ibd_state_t *,
402 ipoib_mac_t *, ipoib_mac_t *);
403 static void ibd_leave_group(ibd_state_t *, ib_gid_t, uint8_t);
404 static void ibd_reacquire_group(ibd_state_t *, ibd_mce_t *);
405 static ibt_status_t ibd_iba_join(ibd_state_t *, ib_gid_t, ibd_mce_t *);
406 static ibt_status_t ibd_find_bgroup(ibd_state_t *);
407 static void ibd_n2h_gid(ipoib_mac_t *, ib_gid_t *);
408 static void ibd_h2n_mac(ipoib_mac_t *, ib_qpn_t, ib_sn_prefix_t, ib_guid_t);
409 static uint64_t ibd_get_portspeed(ibd_state_t *);
410 static boolean_t ibd_async_safe(ibd_state_t *);
411 static void ibd_async_done(ibd_state_t *);
412 static ibd_ace_t *ibd_acache_lookup(ibd_state_t *, ipoib_mac_t *, int *, int);
413 static ibd_ace_t *ibd_acache_get_unref(ibd_state_t *);
414 static void ibd_link_mod(ibd_state_t *, ibt_async_code_t);
415 static int ibd_locate_pkey(ib_pkey_t *, uint16_t, ib_pkey_t, uint16_t *);
416
417 /*
418 * Helpers for attach/start routines
419 */
420 static int ibd_register_mac(ibd_state_t *, dev_info_t *);
421 static int ibd_record_capab(ibd_state_t *);
422 static int ibd_get_port_details(ibd_state_t *);
423 static int ibd_alloc_cqs(ibd_state_t *);
424 static int ibd_setup_ud_channel(ibd_state_t *);
425 static int ibd_start(ibd_state_t *);
426 static int ibd_undo_start(ibd_state_t *, link_state_t);
427 static void ibd_set_mac_progress(ibd_state_t *, uint_t);
428 static void ibd_clr_mac_progress(ibd_state_t *, uint_t);
429 static int ibd_part_attach(ibd_state_t *state, dev_info_t *dip);
430 static void ibd_part_unattach(ibd_state_t *state);
431 static int ibd_port_attach(dev_info_t *);
432 static int ibd_port_unattach(ibd_state_t *state, dev_info_t *dip);
433 static int ibd_get_port_state(ibd_state_t *, link_state_t *);
434 static int ibd_part_busy(ibd_state_t *);
435
436 /*
437 * Miscellaneous helpers
438 */
439 static int ibd_sched_poll(ibd_state_t *, int, int);
440 static void ibd_resume_transmission(ibd_state_t *);
441 static int ibd_setup_lso(ibd_swqe_t *, mblk_t *, uint32_t, ibt_ud_dest_hdl_t);
442 static int ibd_prepare_sgl(ibd_state_t *, mblk_t *, ibd_swqe_t *, uint_t);
443 static void *list_get_head(list_t *);
444 static int ibd_hash_key_cmp(mod_hash_key_t, mod_hash_key_t);
445 static uint_t ibd_hash_by_id(void *, mod_hash_key_t);
446
447 ibt_status_t ibd_get_part_attr(datalink_id_t, ibt_part_attr_t *);
448 ibt_status_t ibd_get_all_part_attr(ibt_part_attr_t **, int *);
449
450 #ifdef IBD_LOGGING
451 static void ibd_log(const char *, ...);
452 #endif
453
454 DDI_DEFINE_STREAM_OPS(ibd_dev_ops, nulldev, nulldev, ibd_attach, ibd_detach,
455 nodev, NULL, D_MP, NULL, ddi_quiesce_not_needed);
456
457 /* Module Driver Info */
458 static struct modldrv ibd_modldrv = {
459 &mod_driverops, /* This one is a driver */
460 "InfiniBand GLDv3 Driver", /* short description */
461 &ibd_dev_ops /* driver specific ops */
462 };
463
464 /* Module Linkage */
465 static struct modlinkage ibd_modlinkage = {
466 MODREV_1, { (void *)&ibd_modldrv, NULL }
467 };
468
469 /*
470 * Module (static) info passed to IBTL during ibt_attach
471 */
472 static struct ibt_clnt_modinfo_s ibd_clnt_modinfo = {
473 IBTI_V_CURR,
474 IBT_NETWORK,
475 ibd_async_handler,
476 NULL,
477 "IBPART"
478 };
479
480 static struct ibt_clnt_modinfo_s ibdpd_clnt_modinfo = {
481 IBTI_V_CURR,
482 IBT_NETWORK,
483 ibdpd_async_handler,
484 NULL,
485 "IPIB"
486 };
487
488 /*
489 * GLDv3 entry points
490 */
491 #define IBD_M_CALLBACK_FLAGS \
492 (MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
493
494 static mac_callbacks_t ibd_m_callbacks = {
495 IBD_M_CALLBACK_FLAGS,
496 ibd_m_stat,
497 ibd_m_start,
498 ibd_m_stop,
499 ibd_m_promisc,
500 ibd_m_multicst,
501 ibd_m_unicst,
502 ibd_m_tx,
503 NULL,
504 NULL,
505 ibd_m_getcapab,
506 NULL,
507 NULL,
508 ibd_m_setprop,
509 ibd_m_getprop,
510 ibd_m_propinfo
511 };
512
513 /* Private properties */
514 char *ibd_priv_props[] = {
515 "_ibd_broadcast_group",
516 "_ibd_coalesce_completions",
517 "_ibd_create_broadcast_group",
518 "_ibd_hash_size",
519 "_ibd_lso_enable",
520 "_ibd_num_ah",
521 "_ibd_num_lso_bufs",
522 "_ibd_rc_enable_srq",
523 "_ibd_rc_num_rwqe",
524 "_ibd_rc_num_srq",
525 "_ibd_rc_num_swqe",
526 "_ibd_rc_rx_comp_count",
527 "_ibd_rc_rx_comp_usec",
528 "_ibd_rc_rx_copy_thresh",
529 "_ibd_rc_rx_rwqe_thresh",
530 "_ibd_rc_tx_comp_count",
531 "_ibd_rc_tx_comp_usec",
532 "_ibd_rc_tx_copy_thresh",
533 "_ibd_ud_num_rwqe",
534 "_ibd_ud_num_swqe",
535 "_ibd_ud_rx_comp_count",
536 "_ibd_ud_rx_comp_usec",
537 "_ibd_ud_tx_comp_count",
538 "_ibd_ud_tx_comp_usec",
539 "_ibd_ud_tx_copy_thresh",
540 NULL
541 };
542
543 static int ibd_create_partition(void *, intptr_t, int, cred_t *, int *);
544 static int ibd_delete_partition(void *, intptr_t, int, cred_t *, int *);
545 static int ibd_get_partition_info(void *, intptr_t, int, cred_t *, int *);
546
547 static dld_ioc_info_t ibd_dld_ioctl_list[] = {
548 {IBD_CREATE_IBPART, DLDCOPYINOUT, sizeof (ibpart_ioctl_t),
549 ibd_create_partition, secpolicy_dl_config},
550 {IBD_DELETE_IBPART, DLDCOPYIN, sizeof (ibpart_ioctl_t),
551 ibd_delete_partition, secpolicy_dl_config},
552 {IBD_INFO_IBPART, DLDCOPYIN, sizeof (ibd_ioctl_t),
553 ibd_get_partition_info, NULL}
554 };
555
556 /*
557 * Fill/clear <scope> and <p_key> in multicast/broadcast address
558 */
559 #define IBD_FILL_SCOPE_PKEY(maddr, scope, pkey) \
560 { \
561 *(uint32_t *)((char *)(maddr) + 4) |= \
562 htonl((uint32_t)(scope) << 16); \
563 *(uint32_t *)((char *)(maddr) + 8) |= \
564 htonl((uint32_t)(pkey) << 16); \
565 }
566
567 #define IBD_CLEAR_SCOPE_PKEY(maddr) \
568 { \
569 *(uint32_t *)((char *)(maddr) + 4) &= \
570 htonl(~((uint32_t)0xF << 16)); \
571 *(uint32_t *)((char *)(maddr) + 8) &= \
572 htonl(~((uint32_t)0xFFFF << 16)); \
573 }
574
575 /*
576 * Rudimentary debugging support
577 */
578 #ifdef DEBUG
579 int ibd_debuglevel = 100;
580 void
581 debug_print(int l, char *fmt, ...)
582 {
583 va_list ap;
584
585 if (l < ibd_debuglevel)
586 return;
587 va_start(ap, fmt);
588 vcmn_err(CE_CONT, fmt, ap);
589 va_end(ap);
590 }
591 #endif
592
593 /*
594 * Common routine to print warning messages; adds in hca guid, port number
595 * and pkey to be able to identify the IBA interface.
596 */
597 void
598 ibd_print_warn(ibd_state_t *state, char *fmt, ...)
599 {
600 ib_guid_t hca_guid;
601 char ibd_print_buf[MAXNAMELEN + 256];
602 int len;
603 va_list ap;
604 char part_name[MAXNAMELEN];
605 datalink_id_t linkid = state->id_plinkid;
606
607 hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip,
608 0, "hca-guid", 0);
609 (void) dls_mgmt_get_linkinfo(linkid, part_name, NULL, NULL, NULL);
610 len = snprintf(ibd_print_buf, sizeof (ibd_print_buf),
611 "%s%d: HCA GUID %016llx port %d PKEY %02x link %s ",
612 ddi_driver_name(state->id_dip), ddi_get_instance(state->id_dip),
613 (u_longlong_t)hca_guid, state->id_port, state->id_pkey,
614 part_name);
615 va_start(ap, fmt);
616 (void) vsnprintf(ibd_print_buf + len, sizeof (ibd_print_buf) - len,
617 fmt, ap);
618 cmn_err(CE_NOTE, "!%s", ibd_print_buf);
619 va_end(ap);
620 }
621
622 /*
623 * Warlock directives
624 */
625
626 /*
627 * id_lso_lock
628 *
629 * state->id_lso->bkt_nfree may be accessed without a lock to
630 * determine the threshold at which we have to ask the nw layer
631 * to resume transmission (see ibd_resume_transmission()).
632 */
633 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_lso_lock,
634 ibd_state_t::id_lso))
635 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_lso))
636 _NOTE(SCHEME_PROTECTS_DATA("init", ibd_state_t::id_lso_policy))
637 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_lsobkt_t::bkt_nfree))
638
639 /*
640 * id_scq_poll_lock
641 */
642 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_scq_poll_lock,
643 ibd_state_t::id_scq_poll_busy))
644
645 /*
646 * id_txpost_lock
647 */
648 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
649 ibd_state_t::id_tx_head))
650 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
651 ibd_state_t::id_tx_busy))
652
653 /*
654 * id_acache_req_lock
655 */
656 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock,
657 ibd_state_t::id_acache_req_cv))
658 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock,
659 ibd_state_t::id_req_list))
660 _NOTE(SCHEME_PROTECTS_DATA("atomic",
661 ibd_acache_s::ac_ref))
662
663 /*
664 * id_ac_mutex
665 *
666 * This mutex is actually supposed to protect id_ah_op as well,
667 * but this path of the code isn't clean (see update of id_ah_op
668 * in ibd_async_acache(), immediately after the call to
669 * ibd_async_mcache()). For now, we'll skip this check by
670 * declaring that id_ah_op is protected by some internal scheme
671 * that warlock isn't aware of.
672 */
673 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
674 ibd_state_t::id_ah_active))
675 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
676 ibd_state_t::id_ah_free))
677 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
678 ibd_state_t::id_ah_addr))
679 _NOTE(SCHEME_PROTECTS_DATA("ac mutex should protect this",
680 ibd_state_t::id_ah_op))
681 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
682 ibd_state_t::id_ah_error))
683 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
684 ibd_state_t::id_ac_hot_ace))
685 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_ah_error))
686
687 /*
688 * id_mc_mutex
689 */
690 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex,
691 ibd_state_t::id_mc_full))
692 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex,
693 ibd_state_t::id_mc_non))
694
695 /*
696 * id_trap_lock
697 */
698 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
699 ibd_state_t::id_trap_cv))
700 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
701 ibd_state_t::id_trap_stop))
702 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
703 ibd_state_t::id_trap_inprog))
704
705 /*
706 * id_prom_op
707 */
708 _NOTE(SCHEME_PROTECTS_DATA("only by async thread",
709 ibd_state_t::id_prom_op))
710
711 /*
712 * id_sched_lock
713 */
714 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_sched_lock,
715 ibd_state_t::id_sched_needed))
716
717 /*
718 * id_link_mutex
719 */
720 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_link_mutex,
721 ibd_state_t::id_link_state))
722 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_link_state))
723 _NOTE(SCHEME_PROTECTS_DATA("only async thr and ibd_m_start",
724 ibd_state_t::id_link_speed))
725 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_sgid))
726
727 /*
728 * id_tx_list.dl_mutex
729 */
730 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
731 ibd_state_t::id_tx_list.dl_head))
732 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
733 ibd_state_t::id_tx_list.dl_pending_sends))
734 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
735 ibd_state_t::id_tx_list.dl_cnt))
736
737 /*
738 * id_rx_list.dl_mutex
739 */
740 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
741 ibd_state_t::id_rx_list.dl_bufs_outstanding))
742 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
743 ibd_state_t::id_rx_list.dl_cnt))
744
745 /*
746 * rc_timeout_lock
747 */
748 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::rc_timeout_lock,
749 ibd_state_t::rc_timeout_start))
750 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::rc_timeout_lock,
751 ibd_state_t::rc_timeout))
752
753
754 /*
755 * Items protected by atomic updates
756 */
757 _NOTE(SCHEME_PROTECTS_DATA("atomic update only",
758 ibd_state_s::id_brd_rcv
759 ibd_state_s::id_brd_xmt
760 ibd_state_s::id_multi_rcv
761 ibd_state_s::id_multi_xmt
762 ibd_state_s::id_num_intrs
763 ibd_state_s::id_rcv_bytes
764 ibd_state_s::id_rcv_pkt
765 ibd_state_s::id_rx_post_queue_index
766 ibd_state_s::id_tx_short
767 ibd_state_s::id_xmt_bytes
768 ibd_state_s::id_xmt_pkt
769 ibd_state_s::rc_rcv_trans_byte
770 ibd_state_s::rc_rcv_trans_pkt
771 ibd_state_s::rc_rcv_copy_byte
772 ibd_state_s::rc_rcv_copy_pkt
773 ibd_state_s::rc_xmt_bytes
774 ibd_state_s::rc_xmt_small_pkt
775 ibd_state_s::rc_xmt_fragmented_pkt
776 ibd_state_s::rc_xmt_map_fail_pkt
777 ibd_state_s::rc_xmt_map_succ_pkt
778 ibd_rc_chan_s::rcq_invoking))
779
780 /*
781 * Non-mutex protection schemes for data elements. Almost all of
782 * these are non-shared items.
783 */
784 _NOTE(SCHEME_PROTECTS_DATA("unshared or single-threaded",
785 callb_cpr
786 ib_gid_s
787 ib_header_info
788 ibd_acache_rq
789 ibd_acache_s::ac_mce
790 ibd_acache_s::ac_chan
791 ibd_mcache::mc_fullreap
792 ibd_mcache::mc_jstate
793 ibd_mcache::mc_req
794 ibd_rwqe_s
795 ibd_swqe_s
796 ibd_wqe_s
797 ibt_wr_ds_s::ds_va
798 ibt_wr_lso_s
799 ipoib_mac::ipoib_qpn
800 mac_capab_lso_s
801 msgb::b_next
802 msgb::b_cont
803 msgb::b_rptr
804 msgb::b_wptr
805 ibd_state_s::id_bgroup_created
806 ibd_state_s::id_mac_state
807 ibd_state_s::id_mtu
808 ibd_state_s::id_ud_num_rwqe
809 ibd_state_s::id_ud_num_swqe
810 ibd_state_s::id_qpnum
811 ibd_state_s::id_rcq_hdl
812 ibd_state_s::id_rx_buf_sz
813 ibd_state_s::id_rx_bufs
814 ibd_state_s::id_rx_mr_hdl
815 ibd_state_s::id_rx_wqes
816 ibd_state_s::id_rxwcs
817 ibd_state_s::id_rxwcs_size
818 ibd_state_s::id_rx_nqueues
819 ibd_state_s::id_rx_queues
820 ibd_state_s::id_scope
821 ibd_state_s::id_scq_hdl
822 ibd_state_s::id_tx_buf_sz
823 ibd_state_s::id_tx_bufs
824 ibd_state_s::id_tx_mr_hdl
825 ibd_state_s::id_tx_rel_list.dl_cnt
826 ibd_state_s::id_tx_wqes
827 ibd_state_s::id_txwcs
828 ibd_state_s::id_txwcs_size
829 ibd_state_s::rc_listen_hdl
830 ibd_state_s::rc_listen_hdl_OFED_interop
831 ibd_state_s::rc_srq_size
832 ibd_state_s::rc_srq_rwqes
833 ibd_state_s::rc_srq_rx_bufs
834 ibd_state_s::rc_srq_rx_mr_hdl
835 ibd_state_s::rc_tx_largebuf_desc_base
836 ibd_state_s::rc_tx_mr_bufs
837 ibd_state_s::rc_tx_mr_hdl
838 ipha_s
839 icmph_s
840 ibt_path_info_s::pi_sid
841 ibd_rc_chan_s::ace
842 ibd_rc_chan_s::chan_hdl
843 ibd_rc_chan_s::state
844 ibd_rc_chan_s::chan_state
845 ibd_rc_chan_s::is_tx_chan
846 ibd_rc_chan_s::rcq_hdl
847 ibd_rc_chan_s::rcq_size
848 ibd_rc_chan_s::scq_hdl
849 ibd_rc_chan_s::scq_size
850 ibd_rc_chan_s::rx_bufs
851 ibd_rc_chan_s::rx_mr_hdl
852 ibd_rc_chan_s::rx_rwqes
853 ibd_rc_chan_s::tx_wqes
854 ibd_rc_chan_s::tx_mr_bufs
855 ibd_rc_chan_s::tx_mr_hdl
856 ibd_rc_chan_s::tx_rel_list.dl_cnt
857 ibd_rc_chan_s::is_used
858 ibd_rc_tx_largebuf_s::lb_buf
859 ibd_rc_msg_hello_s
860 ibt_cm_return_args_s))
861
862 /*
863 * ibd_rc_chan_s::next is protected by two mutexes:
864 * 1) ibd_state_s::rc_pass_chan_list.chan_list_mutex
865 * 2) ibd_state_s::rc_obs_act_chan_list.chan_list_mutex.
866 */
867 _NOTE(SCHEME_PROTECTS_DATA("protected by two mutexes",
868 ibd_rc_chan_s::next))
869
870 /*
871 * ibd_state_s.rc_tx_large_bufs_lock
872 */
873 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock,
874 ibd_state_s::rc_tx_largebuf_free_head))
875 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock,
876 ibd_state_s::rc_tx_largebuf_nfree))
877 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock,
878 ibd_rc_tx_largebuf_s::lb_next))
879
880 /*
881 * ibd_acache_s.tx_too_big_mutex
882 */
883 _NOTE(MUTEX_PROTECTS_DATA(ibd_acache_s::tx_too_big_mutex,
884 ibd_acache_s::tx_too_big_ongoing))
885
886 /*
887 * tx_wqe_list.dl_mutex
888 */
889 _NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex,
890 ibd_rc_chan_s::tx_wqe_list.dl_head))
891 _NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex,
892 ibd_rc_chan_s::tx_wqe_list.dl_pending_sends))
893 _NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex,
894 ibd_rc_chan_s::tx_wqe_list.dl_cnt))
895
896 /*
897 * ibd_state_s.rc_ace_recycle_lock
898 */
899 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_ace_recycle_lock,
900 ibd_state_s::rc_ace_recycle))
901
902 /*
903 * rc_srq_rwqe_list.dl_mutex
904 */
905 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
906 ibd_state_t::rc_srq_rwqe_list.dl_bufs_outstanding))
907 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
908 ibd_state_t::rc_srq_rwqe_list.dl_cnt))
909
910 /*
911 * Non-mutex protection schemes for data elements. They are counters
912 * for problem diagnosis. Don't need be protected.
913 */
914 _NOTE(SCHEME_PROTECTS_DATA("counters for problem diagnosis",
915 ibd_state_s::rc_rcv_alloc_fail
916 ibd_state_s::rc_rcq_err
917 ibd_state_s::rc_ace_not_found
918 ibd_state_s::rc_xmt_drop_too_long_pkt
919 ibd_state_s::rc_xmt_icmp_too_long_pkt
920 ibd_state_s::rc_xmt_reenter_too_long_pkt
921 ibd_state_s::rc_swqe_short
922 ibd_state_s::rc_swqe_mac_update
923 ibd_state_s::rc_xmt_buf_short
924 ibd_state_s::rc_xmt_buf_mac_update
925 ibd_state_s::rc_scq_no_swqe
926 ibd_state_s::rc_scq_no_largebuf
927 ibd_state_s::rc_conn_succ
928 ibd_state_s::rc_conn_fail
929 ibd_state_s::rc_null_conn
930 ibd_state_s::rc_no_estab_conn
931 ibd_state_s::rc_act_close
932 ibd_state_s::rc_pas_close
933 ibd_state_s::rc_delay_ace_recycle
934 ibd_state_s::rc_act_close_simultaneous
935 ibd_state_s::rc_act_close_not_clean
936 ibd_state_s::rc_pas_close_rcq_invoking
937 ibd_state_s::rc_reset_cnt
938 ibd_state_s::rc_timeout_act
939 ibd_state_s::rc_timeout_pas
940 ibd_state_s::rc_stop_connect))
941
942 #ifdef DEBUG
943 /*
944 * Non-mutex protection schemes for data elements. They are counters
945 * for problem diagnosis. Don't need be protected.
946 */
947 _NOTE(SCHEME_PROTECTS_DATA("counters for problem diagnosis",
948 ibd_state_s::rc_rwqe_short
949 ibd_rc_stat_s::rc_rcv_trans_byte
950 ibd_rc_stat_s::rc_rcv_trans_pkt
951 ibd_rc_stat_s::rc_rcv_copy_byte
952 ibd_rc_stat_s::rc_rcv_copy_pkt
953 ibd_rc_stat_s::rc_rcv_alloc_fail
954 ibd_rc_stat_s::rc_rcq_err
955 ibd_rc_stat_s::rc_rwqe_short
956 ibd_rc_stat_s::rc_xmt_bytes
957 ibd_rc_stat_s::rc_xmt_small_pkt
958 ibd_rc_stat_s::rc_xmt_fragmented_pkt
959 ibd_rc_stat_s::rc_xmt_map_fail_pkt
960 ibd_rc_stat_s::rc_xmt_map_succ_pkt
961 ibd_rc_stat_s::rc_ace_not_found
962 ibd_rc_stat_s::rc_scq_no_swqe
963 ibd_rc_stat_s::rc_scq_no_largebuf
964 ibd_rc_stat_s::rc_swqe_short
965 ibd_rc_stat_s::rc_swqe_mac_update
966 ibd_rc_stat_s::rc_xmt_buf_short
967 ibd_rc_stat_s::rc_xmt_buf_mac_update
968 ibd_rc_stat_s::rc_conn_succ
969 ibd_rc_stat_s::rc_conn_fail
970 ibd_rc_stat_s::rc_null_conn
971 ibd_rc_stat_s::rc_no_estab_conn
972 ibd_rc_stat_s::rc_act_close
973 ibd_rc_stat_s::rc_pas_close
974 ibd_rc_stat_s::rc_delay_ace_recycle
975 ibd_rc_stat_s::rc_act_close_simultaneous
976 ibd_rc_stat_s::rc_reset_cnt
977 ibd_rc_stat_s::rc_timeout_act
978 ibd_rc_stat_s::rc_timeout_pas))
979 #endif
980
981 int
982 _init()
983 {
984 int status;
985
986 status = ddi_soft_state_init(&ibd_list, max(sizeof (ibd_state_t),
987 PAGESIZE), 0);
988 if (status != 0) {
989 DPRINT(10, "_init:failed in ddi_soft_state_init()");
990 return (status);
991 }
992
993 mutex_init(&ibd_objlist_lock, NULL, MUTEX_DRIVER, NULL);
994
995 mac_init_ops(&ibd_dev_ops, "ibp");
996 status = mod_install(&ibd_modlinkage);
997 if (status != 0) {
998 DPRINT(10, "_init:failed in mod_install()");
999 ddi_soft_state_fini(&ibd_list);
1000 mac_fini_ops(&ibd_dev_ops);
1001 return (status);
1002 }
1003
1004 mutex_init(&ibd_gstate.ig_mutex, NULL, MUTEX_DRIVER, NULL);
1005 mutex_enter(&ibd_gstate.ig_mutex);
1006 ibd_gstate.ig_ibt_hdl = NULL;
1007 ibd_gstate.ig_ibt_hdl_ref_cnt = 0;
1008 ibd_gstate.ig_service_list = NULL;
1009 mutex_exit(&ibd_gstate.ig_mutex);
1010
1011 if (dld_ioc_register(IBPART_IOC, ibd_dld_ioctl_list,
1012 DLDIOCCNT(ibd_dld_ioctl_list)) != 0) {
1013 return (EIO);
1014 }
1015
1016 ibt_register_part_attr_cb(ibd_get_part_attr, ibd_get_all_part_attr);
1017
1018 #ifdef IBD_LOGGING
1019 ibd_log_init();
1020 #endif
1021 return (0);
1022 }
1023
1024 int
1025 _info(struct modinfo *modinfop)
1026 {
1027 return (mod_info(&ibd_modlinkage, modinfop));
1028 }
1029
1030 int
1031 _fini()
1032 {
1033 int status;
1034
1035 status = mod_remove(&ibd_modlinkage);
1036 if (status != 0)
1037 return (status);
1038
1039 ibt_unregister_part_attr_cb();
1040
1041 mac_fini_ops(&ibd_dev_ops);
1042 mutex_destroy(&ibd_objlist_lock);
1043 ddi_soft_state_fini(&ibd_list);
1044 mutex_destroy(&ibd_gstate.ig_mutex);
1045 #ifdef IBD_LOGGING
1046 ibd_log_fini();
1047 #endif
1048 return (0);
1049 }
1050
1051 /*
1052 * Convert the GID part of the mac address from network byte order
1053 * to host order.
1054 */
1055 static void
1056 ibd_n2h_gid(ipoib_mac_t *mac, ib_gid_t *dgid)
1057 {
1058 ib_sn_prefix_t nbopref;
1059 ib_guid_t nboguid;
1060
1061 bcopy(mac->ipoib_gidpref, &nbopref, sizeof (ib_sn_prefix_t));
1062 bcopy(mac->ipoib_gidsuff, &nboguid, sizeof (ib_guid_t));
1063 dgid->gid_prefix = b2h64(nbopref);
1064 dgid->gid_guid = b2h64(nboguid);
1065 }
1066
1067 /*
1068 * Create the IPoIB address in network byte order from host order inputs.
1069 */
1070 static void
1071 ibd_h2n_mac(ipoib_mac_t *mac, ib_qpn_t qpn, ib_sn_prefix_t prefix,
1072 ib_guid_t guid)
1073 {
1074 ib_sn_prefix_t nbopref;
1075 ib_guid_t nboguid;
1076
1077 mac->ipoib_qpn = htonl(qpn);
1078 nbopref = h2b64(prefix);
1079 nboguid = h2b64(guid);
1080 bcopy(&nbopref, mac->ipoib_gidpref, sizeof (ib_sn_prefix_t));
1081 bcopy(&nboguid, mac->ipoib_gidsuff, sizeof (ib_guid_t));
1082 }
1083
1084 /*
1085 * Send to the appropriate all-routers group when the IBA multicast group
1086 * does not exist, based on whether the target group is v4 or v6.
1087 */
1088 static boolean_t
1089 ibd_get_allroutergroup(ibd_state_t *state, ipoib_mac_t *mcmac,
1090 ipoib_mac_t *rmac)
1091 {
1092 boolean_t retval = B_TRUE;
1093 uint32_t adjscope = state->id_scope << 16;
1094 uint32_t topword;
1095
1096 /*
1097 * Copy the first 4 bytes in without assuming any alignment of
1098 * input mac address; this will have IPoIB signature, flags and
1099 * scope bits.
1100 */
1101 bcopy(mcmac->ipoib_gidpref, &topword, sizeof (uint32_t));
1102 topword = ntohl(topword);
1103
1104 /*
1105 * Generate proper address for IPv4/v6, adding in the Pkey properly.
1106 */
1107 if ((topword == (IB_MCGID_IPV4_PREFIX | adjscope)) ||
1108 (topword == (IB_MCGID_IPV6_PREFIX | adjscope)))
1109 ibd_h2n_mac(rmac, IB_MC_QPN, (((uint64_t)topword << 32) |
1110 ((uint32_t)(state->id_pkey << 16))),
1111 (INADDR_ALLRTRS_GROUP - INADDR_UNSPEC_GROUP));
1112 else
1113 /*
1114 * Does not have proper bits in the mgid address.
1115 */
1116 retval = B_FALSE;
1117
1118 return (retval);
1119 }
1120
1121 /*
1122 * Membership states for different mcg's are tracked by two lists:
1123 * the "non" list is used for promiscuous mode, when all mcg traffic
1124 * needs to be inspected. This type of membership is never used for
1125 * transmission, so there can not be an AH in the active list
1126 * corresponding to a member in this list. This list does not need
1127 * any protection, since all operations are performed by the async
1128 * thread.
1129 *
1130 * "Full" and "SendOnly" membership is tracked using a single list,
1131 * the "full" list. This is because this single list can then be
1132 * searched during transmit to a multicast group (if an AH for the
1133 * mcg is not found in the active list), since at least one type
1134 * of membership must be present before initiating the transmit.
1135 * This list is also emptied during driver detach, since sendonly
1136 * membership acquired during transmit is dropped at detach time
1137 * along with ipv4 broadcast full membership. Insert/deletes to
1138 * this list are done only by the async thread, but it is also
1139 * searched in program context (see multicast disable case), thus
1140 * the id_mc_mutex protects the list. The driver detach path also
1141 * deconstructs the "full" list, but it ensures that the async
1142 * thread will not be accessing the list (by blocking out mcg
1143 * trap handling and making sure no more Tx reaping will happen).
1144 *
1145 * Currently, an IBA attach is done in the SendOnly case too,
1146 * although this is not required.
1147 */
1148 #define IBD_MCACHE_INSERT_FULL(state, mce) \
1149 list_insert_head(&state->id_mc_full, mce)
1150 #define IBD_MCACHE_INSERT_NON(state, mce) \
1151 list_insert_head(&state->id_mc_non, mce)
1152 #define IBD_MCACHE_FIND_FULL(state, mgid) \
1153 ibd_mcache_find(mgid, &state->id_mc_full)
1154 #define IBD_MCACHE_FIND_NON(state, mgid) \
1155 ibd_mcache_find(mgid, &state->id_mc_non)
1156 #define IBD_MCACHE_PULLOUT_FULL(state, mce) \
1157 list_remove(&state->id_mc_full, mce)
1158 #define IBD_MCACHE_PULLOUT_NON(state, mce) \
1159 list_remove(&state->id_mc_non, mce)
1160
1161 static void *
1162 list_get_head(list_t *list)
1163 {
1164 list_node_t *lhead = list_head(list);
1165
1166 if (lhead != NULL)
1167 list_remove(list, lhead);
1168 return (lhead);
1169 }
1170
1171 /*
1172 * This is always guaranteed to be able to queue the work.
1173 */
1174 void
1175 ibd_queue_work_slot(ibd_state_t *state, ibd_req_t *ptr, int op)
1176 {
1177 /* Initialize request */
1178 DPRINT(1, "ibd_queue_work_slot : op: %d \n", op);
1179 ptr->rq_op = op;
1180
1181 /*
1182 * Queue provided slot onto request pool.
1183 */
1184 mutex_enter(&state->id_acache_req_lock);
1185 list_insert_tail(&state->id_req_list, ptr);
1186
1187 /* Go, fetch, async thread */
1188 cv_signal(&state->id_acache_req_cv);
1189 mutex_exit(&state->id_acache_req_lock);
1190 }
1191
1192 /*
1193 * Main body of the per interface async thread.
1194 */
1195 static void
1196 ibd_async_work(ibd_state_t *state)
1197 {
1198 ibd_req_t *ptr;
1199 callb_cpr_t cprinfo;
1200
1201 mutex_enter(&state->id_acache_req_lock);
1202 CALLB_CPR_INIT(&cprinfo, &state->id_acache_req_lock,
1203 callb_generic_cpr, "ibd_async_work");
1204
1205 for (;;) {
1206 ptr = list_get_head(&state->id_req_list);
1207 if (ptr != NULL) {
1208 mutex_exit(&state->id_acache_req_lock);
1209
1210 /*
1211 * If we are in late hca initialization mode, do not
1212 * process any other async request other than TRAP. TRAP
1213 * is used for indicating creation of a broadcast group;
1214 * in which case, we need to join/create the group.
1215 */
1216 if ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) &&
1217 (ptr->rq_op != IBD_ASYNC_TRAP)) {
1218 goto free_req_and_continue;
1219 }
1220
1221 /*
1222 * Once we have done the operation, there is no
1223 * guarantee the request slot is going to be valid,
1224 * it might be freed up (as in IBD_ASYNC_LEAVE, REAP,
1225 * TRAP).
1226 *
1227 * Perform the request.
1228 */
1229 switch (ptr->rq_op) {
1230 case IBD_ASYNC_GETAH:
1231 ibd_async_acache(state, &ptr->rq_mac);
1232 break;
1233 case IBD_ASYNC_JOIN:
1234 case IBD_ASYNC_LEAVE:
1235 ibd_async_multicast(state,
1236 ptr->rq_gid, ptr->rq_op);
1237 break;
1238 case IBD_ASYNC_PROMON:
1239 ibd_async_setprom(state);
1240 break;
1241 case IBD_ASYNC_PROMOFF:
1242 ibd_async_unsetprom(state);
1243 break;
1244 case IBD_ASYNC_REAP:
1245 ibd_async_reap_group(state,
1246 ptr->rq_ptr, ptr->rq_gid,
1247 IB_MC_JSTATE_FULL);
1248 /*
1249 * the req buf contains in mce
1250 * structure, so we do not need
1251 * to free it here.
1252 */
1253 ptr = NULL;
1254 break;
1255 case IBD_ASYNC_TRAP:
1256 ibd_async_trap(state, ptr);
1257 break;
1258 case IBD_ASYNC_SCHED:
1259 ibd_async_txsched(state);
1260 break;
1261 case IBD_ASYNC_LINK:
1262 ibd_async_link(state, ptr);
1263 break;
1264 case IBD_ASYNC_EXIT:
1265 mutex_enter(&state->id_acache_req_lock);
1266 #ifndef __lock_lint
1267 CALLB_CPR_EXIT(&cprinfo);
1268 #else
1269 mutex_exit(&state->id_acache_req_lock);
1270 #endif
1271 return;
1272 case IBD_ASYNC_RC_TOO_BIG:
1273 ibd_async_rc_process_too_big(state,
1274 ptr);
1275 break;
1276 case IBD_ASYNC_RC_CLOSE_ACT_CHAN:
1277 ibd_async_rc_close_act_chan(state, ptr);
1278 break;
1279 case IBD_ASYNC_RC_RECYCLE_ACE:
1280 ibd_async_rc_recycle_ace(state, ptr);
1281 break;
1282 case IBD_ASYNC_RC_CLOSE_PAS_CHAN:
1283 (void) ibd_rc_pas_close(ptr->rq_ptr,
1284 B_TRUE, B_TRUE);
1285 break;
1286 }
1287 free_req_and_continue:
1288 if (ptr != NULL)
1289 kmem_cache_free(state->id_req_kmc, ptr);
1290
1291 mutex_enter(&state->id_acache_req_lock);
1292 } else {
1293 #ifndef __lock_lint
1294 /*
1295 * Nothing to do: wait till new request arrives.
1296 */
1297 CALLB_CPR_SAFE_BEGIN(&cprinfo);
1298 cv_wait(&state->id_acache_req_cv,
1299 &state->id_acache_req_lock);
1300 CALLB_CPR_SAFE_END(&cprinfo,
1301 &state->id_acache_req_lock);
1302 #endif
1303 }
1304 }
1305
1306 /*NOTREACHED*/
1307 _NOTE(NOT_REACHED)
1308 }
1309
1310 /*
1311 * Return when it is safe to queue requests to the async daemon; primarily
1312 * for subnet trap and async event handling. Disallow requests before the
1313 * daemon is created, and when interface deinitilization starts.
1314 */
1315 static boolean_t
1316 ibd_async_safe(ibd_state_t *state)
1317 {
1318 mutex_enter(&state->id_trap_lock);
1319 if (state->id_trap_stop) {
1320 mutex_exit(&state->id_trap_lock);
1321 return (B_FALSE);
1322 }
1323 state->id_trap_inprog++;
1324 mutex_exit(&state->id_trap_lock);
1325 return (B_TRUE);
1326 }
1327
1328 /*
1329 * Wake up ibd_m_stop() if the unplumb code is waiting for pending subnet
1330 * trap or event handling to complete to kill the async thread and deconstruct
1331 * the mcg/ace list.
1332 */
1333 static void
1334 ibd_async_done(ibd_state_t *state)
1335 {
1336 mutex_enter(&state->id_trap_lock);
1337 if (--state->id_trap_inprog == 0)
1338 cv_signal(&state->id_trap_cv);
1339 mutex_exit(&state->id_trap_lock);
1340 }
1341
1342 /*
1343 * Hash functions:
1344 * ibd_hash_by_id: Returns the qpn as the hash entry into bucket.
1345 * ibd_hash_key_cmp: Compares two keys, return 0 on success or else 1.
1346 * These operate on mac addresses input into ibd_send, but there is no
1347 * guarantee on the alignment of the ipoib_mac_t structure.
1348 */
1349 /*ARGSUSED*/
1350 static uint_t
1351 ibd_hash_by_id(void *hash_data, mod_hash_key_t key)
1352 {
1353 ulong_t ptraddr = (ulong_t)key;
1354 uint_t hval;
1355
1356 /*
1357 * If the input address is 4 byte aligned, we can just dereference
1358 * it. This is most common, since IP will send in a 4 byte aligned
1359 * IP header, which implies the 24 byte IPoIB psuedo header will be
1360 * 4 byte aligned too.
1361 */
1362 if ((ptraddr & 3) == 0)
1363 return ((uint_t)((ipoib_mac_t *)key)->ipoib_qpn);
1364
1365 bcopy(&(((ipoib_mac_t *)key)->ipoib_qpn), &hval, sizeof (uint_t));
1366 return (hval);
1367 }
1368
1369 static int
1370 ibd_hash_key_cmp(mod_hash_key_t key1, mod_hash_key_t key2)
1371 {
1372 if (bcmp((char *)key1, (char *)key2, IPOIB_ADDRL) == 0)
1373 return (0);
1374 else
1375 return (1);
1376 }
1377
1378 /*
1379 * Initialize all the per interface caches and lists; AH cache,
1380 * MCG list etc.
1381 */
1382 static int
1383 ibd_acache_init(ibd_state_t *state)
1384 {
1385 ibd_ace_t *ce;
1386 int i;
1387
1388 mutex_init(&state->id_ac_mutex, NULL, MUTEX_DRIVER, NULL);
1389 mutex_init(&state->id_mc_mutex, NULL, MUTEX_DRIVER, NULL);
1390 mutex_enter(&state->id_ac_mutex);
1391 list_create(&state->id_ah_free, sizeof (ibd_ace_t),
1392 offsetof(ibd_ace_t, ac_list));
1393 list_create(&state->id_ah_active, sizeof (ibd_ace_t),
1394 offsetof(ibd_ace_t, ac_list));
1395 state->id_ah_active_hash = mod_hash_create_extended("IBD AH hash",
1396 state->id_hash_size, mod_hash_null_keydtor, mod_hash_null_valdtor,
1397 ibd_hash_by_id, NULL, ibd_hash_key_cmp, KM_SLEEP);
1398 list_create(&state->id_mc_full, sizeof (ibd_mce_t),
1399 offsetof(ibd_mce_t, mc_list));
1400 list_create(&state->id_mc_non, sizeof (ibd_mce_t),
1401 offsetof(ibd_mce_t, mc_list));
1402 state->id_ac_hot_ace = NULL;
1403
1404 state->id_ac_list = ce = (ibd_ace_t *)kmem_zalloc(sizeof (ibd_ace_t) *
1405 state->id_num_ah, KM_SLEEP);
1406 for (i = 0; i < state->id_num_ah; i++, ce++) {
1407 if (ibt_alloc_ud_dest(state->id_hca_hdl, IBT_UD_DEST_NO_FLAGS,
1408 state->id_pd_hdl, &ce->ac_dest) != IBT_SUCCESS) {
1409 mutex_exit(&state->id_ac_mutex);
1410 ibd_acache_fini(state);
1411 return (DDI_FAILURE);
1412 } else {
1413 CLEAR_REFCYCLE(ce);
1414 ce->ac_mce = NULL;
1415 mutex_init(&ce->tx_too_big_mutex, NULL,
1416 MUTEX_DRIVER, NULL);
1417 IBD_ACACHE_INSERT_FREE(state, ce);
1418 }
1419 }
1420 mutex_exit(&state->id_ac_mutex);
1421 return (DDI_SUCCESS);
1422 }
1423
1424 static void
1425 ibd_acache_fini(ibd_state_t *state)
1426 {
1427 ibd_ace_t *ptr;
1428
1429 mutex_enter(&state->id_ac_mutex);
1430
1431 while ((ptr = IBD_ACACHE_GET_ACTIVE(state)) != NULL) {
1432 ASSERT(GET_REF(ptr) == 0);
1433 mutex_destroy(&ptr->tx_too_big_mutex);
1434 (void) ibt_free_ud_dest(ptr->ac_dest);
1435 }
1436
1437 while ((ptr = IBD_ACACHE_GET_FREE(state)) != NULL) {
1438 ASSERT(GET_REF(ptr) == 0);
1439 mutex_destroy(&ptr->tx_too_big_mutex);
1440 (void) ibt_free_ud_dest(ptr->ac_dest);
1441 }
1442
1443 list_destroy(&state->id_ah_free);
1444 list_destroy(&state->id_ah_active);
1445 list_destroy(&state->id_mc_full);
1446 list_destroy(&state->id_mc_non);
1447 kmem_free(state->id_ac_list, sizeof (ibd_ace_t) * state->id_num_ah);
1448 mutex_exit(&state->id_ac_mutex);
1449 mutex_destroy(&state->id_ac_mutex);
1450 mutex_destroy(&state->id_mc_mutex);
1451 }
1452
1453 /*
1454 * Search AH active hash list for a cached path to input destination.
1455 * If we are "just looking", hold == F. When we are in the Tx path,
1456 * we set hold == T to grab a reference on the AH so that it can not
1457 * be recycled to a new destination while the Tx request is posted.
1458 */
1459 ibd_ace_t *
1460 ibd_acache_find(ibd_state_t *state, ipoib_mac_t *mac, boolean_t hold, int num)
1461 {
1462 ibd_ace_t *ptr;
1463
1464 ASSERT(mutex_owned(&state->id_ac_mutex));
1465
1466 /*
1467 * Do hash search.
1468 */
1469 if (mod_hash_find(state->id_ah_active_hash,
1470 (mod_hash_key_t)mac, (mod_hash_val_t)&ptr) == 0) {
1471 if (hold)
1472 INC_REF(ptr, num);
1473 return (ptr);
1474 }
1475 return (NULL);
1476 }
1477
1478 /*
1479 * This is called by the tx side; if an initialized AH is found in
1480 * the active list, it is locked down and can be used; if no entry
1481 * is found, an async request is queued to do path resolution.
1482 */
1483 static ibd_ace_t *
1484 ibd_acache_lookup(ibd_state_t *state, ipoib_mac_t *mac, int *err, int numwqe)
1485 {
1486 ibd_ace_t *ptr;
1487 ibd_req_t *req;
1488
1489 /*
1490 * Only attempt to print when we can; in the mdt pattr case, the
1491 * address is not aligned properly.
1492 */
1493 if (((ulong_t)mac & 3) == 0) {
1494 DPRINT(4,
1495 "ibd_acache_lookup : lookup for %08X:%08X:%08X:%08X:%08X",
1496 htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
1497 htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]),
1498 htonl(mac->ipoib_gidsuff[1]));
1499 }
1500
1501 mutex_enter(&state->id_ac_mutex);
1502
1503 if (((ptr = state->id_ac_hot_ace) != NULL) &&
1504 (memcmp(&ptr->ac_mac, mac, sizeof (*mac)) == 0)) {
1505 INC_REF(ptr, numwqe);
1506 mutex_exit(&state->id_ac_mutex);
1507 return (ptr);
1508 }
1509 if (((ptr = ibd_acache_find(state, mac, B_TRUE, numwqe)) != NULL)) {
1510 state->id_ac_hot_ace = ptr;
1511 mutex_exit(&state->id_ac_mutex);
1512 return (ptr);
1513 }
1514
1515 /*
1516 * Implementation of a single outstanding async request; if
1517 * the operation is not started yet, queue a request and move
1518 * to ongoing state. Remember in id_ah_addr for which address
1519 * we are queueing the request, in case we need to flag an error;
1520 * Any further requests, for the same or different address, until
1521 * the operation completes, is sent back to GLDv3 to be retried.
1522 * The async thread will update id_ah_op with an error indication
1523 * or will set it to indicate the next look up can start; either
1524 * way, it will mac_tx_update() so that all blocked requests come
1525 * back here.
1526 */
1527 *err = EAGAIN;
1528 if (state->id_ah_op == IBD_OP_NOTSTARTED) {
1529 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
1530 if (req != NULL) {
1531 /*
1532 * We did not even find the entry; queue a request
1533 * for it.
1534 */
1535 bcopy(mac, &(req->rq_mac), IPOIB_ADDRL);
1536 state->id_ah_op = IBD_OP_ONGOING;
1537 ibd_queue_work_slot(state, req, IBD_ASYNC_GETAH);
1538 bcopy(mac, &state->id_ah_addr, IPOIB_ADDRL);
1539 }
1540 } else if ((state->id_ah_op != IBD_OP_ONGOING) &&
1541 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) == 0)) {
1542 /*
1543 * Check the status of the pathrecord lookup request
1544 * we had queued before.
1545 */
1546 if (state->id_ah_op == IBD_OP_ERRORED) {
1547 *err = EFAULT;
1548 state->id_ah_error++;
1549 } else {
1550 /*
1551 * IBD_OP_ROUTERED case: We need to send to the
1552 * all-router MCG. If we can find the AH for
1553 * the mcg, the Tx will be attempted. If we
1554 * do not find the AH, we return NORESOURCES
1555 * to retry.
1556 */
1557 ipoib_mac_t routermac;
1558
1559 (void) ibd_get_allroutergroup(state, mac, &routermac);
1560 ptr = ibd_acache_find(state, &routermac, B_TRUE,
1561 numwqe);
1562 }
1563 state->id_ah_op = IBD_OP_NOTSTARTED;
1564 } else if ((state->id_ah_op != IBD_OP_ONGOING) &&
1565 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) != 0)) {
1566 /*
1567 * This case can happen when we get a higher band
1568 * packet. The easiest way is to reset the state machine
1569 * to accommodate the higher priority packet.
1570 */
1571 state->id_ah_op = IBD_OP_NOTSTARTED;
1572 }
1573 mutex_exit(&state->id_ac_mutex);
1574
1575 return (ptr);
1576 }
1577
1578 /*
1579 * Grab a not-currently-in-use AH/PathRecord from the active
1580 * list to recycle to a new destination. Only the async thread
1581 * executes this code.
1582 */
1583 static ibd_ace_t *
1584 ibd_acache_get_unref(ibd_state_t *state)
1585 {
1586 ibd_ace_t *ptr = list_tail(&state->id_ah_active);
1587 boolean_t try_rc_chan_recycle = B_FALSE;
1588
1589 ASSERT(mutex_owned(&state->id_ac_mutex));
1590
1591 /*
1592 * Do plain linear search.
1593 */
1594 while (ptr != NULL) {
1595 /*
1596 * Note that it is possible that the "cycle" bit
1597 * is set on the AH w/o any reference count. The
1598 * mcg must have been deleted, and the tx cleanup
1599 * just decremented the reference count to 0, but
1600 * hasn't gotten around to grabbing the id_ac_mutex
1601 * to move the AH into the free list.
1602 */
1603 if (GET_REF(ptr) == 0) {
1604 if (ptr->ac_chan != NULL) {
1605 ASSERT(state->id_enable_rc == B_TRUE);
1606 if (!try_rc_chan_recycle) {
1607 try_rc_chan_recycle = B_TRUE;
1608 ibd_rc_signal_ace_recycle(state, ptr);
1609 }
1610 } else {
1611 IBD_ACACHE_PULLOUT_ACTIVE(state, ptr);
1612 break;
1613 }
1614 }
1615 ptr = list_prev(&state->id_ah_active, ptr);
1616 }
1617 return (ptr);
1618 }
1619
1620 /*
1621 * Invoked to clean up AH from active list in case of multicast
1622 * disable and to handle sendonly memberships during mcg traps.
1623 * And for port up processing for multicast and unicast AHs.
1624 * Normally, the AH is taken off the active list, and put into
1625 * the free list to be recycled for a new destination. In case
1626 * Tx requests on the AH have not completed yet, the AH is marked
1627 * for reaping (which will put the AH on the free list) once the Tx's
1628 * complete; in this case, depending on the "force" input, we take
1629 * out the AH from the active list right now, or leave it also for
1630 * the reap operation. Returns TRUE if the AH is taken off the active
1631 * list (and either put into the free list right now, or arranged for
1632 * later), FALSE otherwise.
1633 */
1634 boolean_t
1635 ibd_acache_recycle(ibd_state_t *state, ipoib_mac_t *acmac, boolean_t force)
1636 {
1637 ibd_ace_t *acactive;
1638 boolean_t ret = B_TRUE;
1639
1640 ASSERT(mutex_owned(&state->id_ac_mutex));
1641
1642 if ((acactive = ibd_acache_find(state, acmac, B_FALSE, 0)) != NULL) {
1643
1644 /*
1645 * Note that the AH might already have the cycle bit set
1646 * on it; this might happen if sequences of multicast
1647 * enables and disables are coming so fast, that posted
1648 * Tx's to the mcg have not completed yet, and the cycle
1649 * bit is set successively by each multicast disable.
1650 */
1651 if (SET_CYCLE_IF_REF(acactive)) {
1652 if (!force) {
1653 /*
1654 * The ace is kept on the active list, further
1655 * Tx's can still grab a reference on it; the
1656 * ace is reaped when all pending Tx's
1657 * referencing the AH complete.
1658 */
1659 ret = B_FALSE;
1660 } else {
1661 /*
1662 * In the mcg trap case, we always pull the
1663 * AH from the active list. And also the port
1664 * up multi/unicast case.
1665 */
1666 ASSERT(acactive->ac_chan == NULL);
1667 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
1668 acactive->ac_mce = NULL;
1669 }
1670 } else {
1671 /*
1672 * Determined the ref count is 0, thus reclaim
1673 * immediately after pulling out the ace from
1674 * the active list.
1675 */
1676 ASSERT(acactive->ac_chan == NULL);
1677 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
1678 acactive->ac_mce = NULL;
1679 IBD_ACACHE_INSERT_FREE(state, acactive);
1680 }
1681
1682 }
1683 return (ret);
1684 }
1685
1686 /*
1687 * Helper function for async path record lookup. If we are trying to
1688 * Tx to a MCG, check our membership, possibly trying to join the
1689 * group if required. If that fails, try to send the packet to the
1690 * all router group (indicated by the redirect output), pointing
1691 * the input mac address to the router mcg address.
1692 */
1693 static ibd_mce_t *
1694 ibd_async_mcache(ibd_state_t *state, ipoib_mac_t *mac, boolean_t *redirect)
1695 {
1696 ib_gid_t mgid;
1697 ibd_mce_t *mce;
1698 ipoib_mac_t routermac;
1699
1700 *redirect = B_FALSE;
1701 ibd_n2h_gid(mac, &mgid);
1702
1703 /*
1704 * Check the FullMember+SendOnlyNonMember list.
1705 * Since we are the only one who manipulates the
1706 * id_mc_full list, no locks are needed.
1707 */
1708 mce = IBD_MCACHE_FIND_FULL(state, mgid);
1709 if (mce != NULL) {
1710 DPRINT(4, "ibd_async_mcache : already joined to group");
1711 return (mce);
1712 }
1713
1714 /*
1715 * Not found; try to join(SendOnlyNonMember) and attach.
1716 */
1717 DPRINT(4, "ibd_async_mcache : not joined to group");
1718 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
1719 NULL) {
1720 DPRINT(4, "ibd_async_mcache : nonmem joined to group");
1721 return (mce);
1722 }
1723
1724 /*
1725 * MCGroup not present; try to join the all-router group. If
1726 * any of the following steps succeed, we will be redirecting
1727 * to the all router group.
1728 */
1729 DPRINT(4, "ibd_async_mcache : nonmem join failed");
1730 if (!ibd_get_allroutergroup(state, mac, &routermac))
1731 return (NULL);
1732 *redirect = B_TRUE;
1733 ibd_n2h_gid(&routermac, &mgid);
1734 bcopy(&routermac, mac, IPOIB_ADDRL);
1735 DPRINT(4, "ibd_async_mcache : router mgid : %016llx:%016llx\n",
1736 mgid.gid_prefix, mgid.gid_guid);
1737
1738 /*
1739 * Are we already joined to the router group?
1740 */
1741 if ((mce = IBD_MCACHE_FIND_FULL(state, mgid)) != NULL) {
1742 DPRINT(4, "ibd_async_mcache : using already joined router"
1743 "group\n");
1744 return (mce);
1745 }
1746
1747 /*
1748 * Can we join(SendOnlyNonMember) the router group?
1749 */
1750 DPRINT(4, "ibd_async_mcache : attempting join to router grp");
1751 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
1752 NULL) {
1753 DPRINT(4, "ibd_async_mcache : joined to router grp");
1754 return (mce);
1755 }
1756
1757 return (NULL);
1758 }
1759
1760 /*
1761 * Async path record lookup code.
1762 */
1763 static void
1764 ibd_async_acache(ibd_state_t *state, ipoib_mac_t *mac)
1765 {
1766 ibd_ace_t *ce;
1767 ibd_mce_t *mce = NULL;
1768 ibt_path_attr_t path_attr;
1769 ibt_path_info_t path_info;
1770 ib_gid_t destgid;
1771 char ret = IBD_OP_NOTSTARTED;
1772
1773 DPRINT(4, "ibd_async_acache : %08X:%08X:%08X:%08X:%08X",
1774 htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
1775 htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]),
1776 htonl(mac->ipoib_gidsuff[1]));
1777
1778 /*
1779 * Check whether we are trying to transmit to a MCG.
1780 * In that case, we need to make sure we are a member of
1781 * the MCG.
1782 */
1783 if (mac->ipoib_qpn == htonl(IB_MC_QPN)) {
1784 boolean_t redirected;
1785
1786 /*
1787 * If we can not find or join the group or even
1788 * redirect, error out.
1789 */
1790 if ((mce = ibd_async_mcache(state, mac, &redirected)) ==
1791 NULL) {
1792 state->id_ah_op = IBD_OP_ERRORED;
1793 return;
1794 }
1795
1796 /*
1797 * If we got redirected, we need to determine whether
1798 * the AH for the new mcg is in the cache already, and
1799 * not pull it in then; otherwise proceed to get the
1800 * path for the new mcg. There is no guarantee that
1801 * if the AH is currently in the cache, it will still be
1802 * there when we look in ibd_acache_lookup(), but that's
1803 * okay, we will come back here.
1804 */
1805 if (redirected) {
1806 ret = IBD_OP_ROUTERED;
1807 DPRINT(4, "ibd_async_acache : redirected to "
1808 "%08X:%08X:%08X:%08X:%08X",
1809 htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
1810 htonl(mac->ipoib_gidpref[1]),
1811 htonl(mac->ipoib_gidsuff[0]),
1812 htonl(mac->ipoib_gidsuff[1]));
1813
1814 mutex_enter(&state->id_ac_mutex);
1815 if (ibd_acache_find(state, mac, B_FALSE, 0) != NULL) {
1816 state->id_ah_op = IBD_OP_ROUTERED;
1817 mutex_exit(&state->id_ac_mutex);
1818 DPRINT(4, "ibd_async_acache : router AH found");
1819 return;
1820 }
1821 mutex_exit(&state->id_ac_mutex);
1822 }
1823 }
1824
1825 /*
1826 * Get an AH from the free list.
1827 */
1828 mutex_enter(&state->id_ac_mutex);
1829 if ((ce = IBD_ACACHE_GET_FREE(state)) == NULL) {
1830 /*
1831 * No free ones; try to grab an unreferenced active
1832 * one. Maybe we need to make the active list LRU,
1833 * but that will create more work for Tx callbacks.
1834 * Is there a way of not having to pull out the
1835 * entry from the active list, but just indicate it
1836 * is being recycled? Yes, but that creates one more
1837 * check in the fast lookup path.
1838 */
1839 if ((ce = ibd_acache_get_unref(state)) == NULL) {
1840 /*
1841 * Pretty serious shortage now.
1842 */
1843 state->id_ah_op = IBD_OP_NOTSTARTED;
1844 mutex_exit(&state->id_ac_mutex);
1845 DPRINT(10, "ibd_async_acache : failed to find AH "
1846 "slot\n");
1847 return;
1848 }
1849 /*
1850 * We could check whether ac_mce points to a SendOnly
1851 * member and drop that membership now. Or do it lazily
1852 * at detach time.
1853 */
1854 ce->ac_mce = NULL;
1855 }
1856 mutex_exit(&state->id_ac_mutex);
1857 ASSERT(ce->ac_mce == NULL);
1858
1859 /*
1860 * Update the entry.
1861 */
1862 bcopy((char *)mac, &ce->ac_mac, IPOIB_ADDRL);
1863
1864 bzero(&path_info, sizeof (path_info));
1865 bzero(&path_attr, sizeof (ibt_path_attr_t));
1866 path_attr.pa_sgid = state->id_sgid;
1867 path_attr.pa_num_dgids = 1;
1868 ibd_n2h_gid(&ce->ac_mac, &destgid);
1869 path_attr.pa_dgids = &destgid;
1870 path_attr.pa_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
1871 path_attr.pa_pkey = state->id_pkey;
1872 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_PKEY, &path_attr, 1,
1873 &path_info, NULL) != IBT_SUCCESS) {
1874 DPRINT(10, "ibd_async_acache : failed in ibt_get_paths");
1875 goto error;
1876 }
1877 if (ibt_modify_ud_dest(ce->ac_dest, state->id_mcinfo->mc_qkey,
1878 ntohl(ce->ac_mac.ipoib_qpn),
1879 &path_info.pi_prim_cep_path.cep_adds_vect) != IBT_SUCCESS) {
1880 DPRINT(10, "ibd_async_acache : failed in ibt_modify_ud_dest");
1881 goto error;
1882 }
1883
1884 /*
1885 * mce is set whenever an AH is being associated with a
1886 * MCG; this will come in handy when we leave the MCG. The
1887 * lock protects Tx fastpath from scanning the active list.
1888 */
1889 if (mce != NULL)
1890 ce->ac_mce = mce;
1891
1892 /*
1893 * initiate a RC mode connection for unicast address
1894 */
1895 if (state->id_enable_rc && (mac->ipoib_qpn != htonl(IB_MC_QPN)) &&
1896 (htonl(mac->ipoib_qpn) & IBD_MAC_ADDR_RC)) {
1897 ASSERT(ce->ac_chan == NULL);
1898 DPRINT(10, "ibd_async_acache: call "
1899 "ibd_rc_try_connect(ace=%p)", ce);
1900 ibd_rc_try_connect(state, ce, &path_info);
1901 if (ce->ac_chan == NULL) {
1902 DPRINT(10, "ibd_async_acache: fail to setup RC"
1903 " channel");
1904 state->rc_conn_fail++;
1905 goto error;
1906 }
1907 }
1908
1909 mutex_enter(&state->id_ac_mutex);
1910 IBD_ACACHE_INSERT_ACTIVE(state, ce);
1911 state->id_ah_op = ret;
1912 mutex_exit(&state->id_ac_mutex);
1913 return;
1914 error:
1915 /*
1916 * We might want to drop SendOnly membership here if we
1917 * joined above. The lock protects Tx callbacks inserting
1918 * into the free list.
1919 */
1920 mutex_enter(&state->id_ac_mutex);
1921 state->id_ah_op = IBD_OP_ERRORED;
1922 IBD_ACACHE_INSERT_FREE(state, ce);
1923 mutex_exit(&state->id_ac_mutex);
1924 }
1925
1926 /*
1927 * While restoring port's presence on the subnet on a port up, it is possible
1928 * that the port goes down again.
1929 */
1930 static void
1931 ibd_async_link(ibd_state_t *state, ibd_req_t *req)
1932 {
1933 ibd_link_op_t opcode = (ibd_link_op_t)req->rq_ptr;
1934 link_state_t lstate = (opcode == IBD_LINK_DOWN) ? LINK_STATE_DOWN :
1935 LINK_STATE_UP;
1936 ibd_mce_t *mce, *pmce;
1937 ibd_ace_t *ace, *pace;
1938
1939 DPRINT(10, "ibd_async_link(): %d", opcode);
1940
1941 /*
1942 * On a link up, revalidate the link speed/width. No point doing
1943 * this on a link down, since we will be unable to do SA operations,
1944 * defaulting to the lowest speed. Also notice that we update our
1945 * notion of speed before calling mac_link_update(), which will do
1946 * necessary higher level notifications for speed changes.
1947 */
1948 if ((opcode == IBD_LINK_UP_ABSENT) || (opcode == IBD_LINK_UP)) {
1949 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
1950 state->id_link_speed = ibd_get_portspeed(state);
1951 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))
1952 }
1953
1954 /*
1955 * Do all the work required to establish our presence on
1956 * the subnet.
1957 */
1958 if (opcode == IBD_LINK_UP_ABSENT) {
1959 /*
1960 * If in promiscuous mode ...
1961 */
1962 if (state->id_prom_op == IBD_OP_COMPLETED) {
1963 /*
1964 * Drop all nonmembership.
1965 */
1966 ibd_async_unsetprom(state);
1967
1968 /*
1969 * Then, try to regain nonmembership to all mcg's.
1970 */
1971 ibd_async_setprom(state);
1972
1973 }
1974
1975 /*
1976 * Drop all sendonly membership (which also gets rid of the
1977 * AHs); try to reacquire all full membership.
1978 */
1979 mce = list_head(&state->id_mc_full);
1980 while ((pmce = mce) != NULL) {
1981 mce = list_next(&state->id_mc_full, mce);
1982 if (pmce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON)
1983 ibd_leave_group(state,
1984 pmce->mc_info.mc_adds_vect.av_dgid,
1985 IB_MC_JSTATE_SEND_ONLY_NON);
1986 else
1987 ibd_reacquire_group(state, pmce);
1988 }
1989
1990 /*
1991 * Recycle all active AHs to free list (and if there are
1992 * pending posts, make sure they will go into the free list
1993 * once the Tx's complete). Grab the lock to prevent
1994 * concurrent Tx's as well as Tx cleanups.
1995 */
1996 mutex_enter(&state->id_ac_mutex);
1997 ace = list_head(&state->id_ah_active);
1998 while ((pace = ace) != NULL) {
1999 boolean_t cycled;
2000
2001 ace = list_next(&state->id_ah_active, ace);
2002 mce = pace->ac_mce;
2003 if (pace->ac_chan != NULL) {
2004 ASSERT(mce == NULL);
2005 ASSERT(state->id_enable_rc == B_TRUE);
2006 if (pace->ac_chan->chan_state ==
2007 IBD_RC_STATE_ACT_ESTAB) {
2008 INC_REF(pace, 1);
2009 IBD_ACACHE_PULLOUT_ACTIVE(state, pace);
2010 pace->ac_chan->chan_state =
2011 IBD_RC_STATE_ACT_CLOSING;
2012 ibd_rc_signal_act_close(state, pace);
2013 } else {
2014 state->rc_act_close_simultaneous++;
2015 DPRINT(40, "ibd_async_link: other "
2016 "thread is closing it, ace=%p, "
2017 "ac_chan=%p, chan_state=%d",
2018 pace, pace->ac_chan,
2019 pace->ac_chan->chan_state);
2020 }
2021 } else {
2022 cycled = ibd_acache_recycle(state,
2023 &pace->ac_mac, B_TRUE);
2024 }
2025 /*
2026 * If this is for an mcg, it must be for a fullmember,
2027 * since we got rid of send-only members above when
2028 * processing the mce list.
2029 */
2030 ASSERT(cycled && ((mce == NULL) || (mce->mc_jstate ==
2031 IB_MC_JSTATE_FULL)));
2032
2033 /*
2034 * Check if the fullmember mce needs to be torn down,
2035 * ie whether the DLPI disable has already been done.
2036 * If so, do some of the work of tx_cleanup, namely
2037 * causing leave (which will fail), detach and
2038 * mce-freeing. tx_cleanup will put the AH into free
2039 * list. The reason to duplicate some of this
2040 * tx_cleanup work is because we want to delete the
2041 * AH right now instead of waiting for tx_cleanup, to
2042 * force subsequent Tx's to reacquire an AH.
2043 */
2044 if ((mce != NULL) && (mce->mc_fullreap))
2045 ibd_async_reap_group(state, mce,
2046 mce->mc_info.mc_adds_vect.av_dgid,
2047 mce->mc_jstate);
2048 }
2049 mutex_exit(&state->id_ac_mutex);
2050 }
2051
2052 /*
2053 * mac handle is guaranteed to exist since driver does ibt_close_hca()
2054 * (which stops further events from being delivered) before
2055 * mac_unregister(). At this point, it is guaranteed that mac_register
2056 * has already been done.
2057 */
2058 mutex_enter(&state->id_link_mutex);
2059 state->id_link_state = lstate;
2060 mac_link_update(state->id_mh, lstate);
2061 mutex_exit(&state->id_link_mutex);
2062
2063 ibd_async_done(state);
2064 }
2065
2066 /*
2067 * Check the pkey table to see if we can find the pkey we're looking for.
2068 * Set the pkey index in 'pkix' if found. Return 0 on success and -1 on
2069 * failure.
2070 */
2071 static int
2072 ibd_locate_pkey(ib_pkey_t *pkey_tbl, uint16_t pkey_tbl_sz, ib_pkey_t pkey,
2073 uint16_t *pkix)
2074 {
2075 uint16_t ndx;
2076
2077 ASSERT(pkix != NULL);
2078
2079 for (ndx = 0; ndx < pkey_tbl_sz; ndx++) {
2080 if (pkey_tbl[ndx] == pkey) {
2081 *pkix = ndx;
2082 return (0);
2083 }
2084 }
2085 return (-1);
2086 }
2087
2088 /*
2089 * Late HCA Initialization:
2090 * If plumb had succeeded without the availability of an active port or the
2091 * pkey, and either of their availability is now being indicated via PORT_UP
2092 * or PORT_CHANGE respectively, try a start of the interface.
2093 *
2094 * Normal Operation:
2095 * When the link is notified up, we need to do a few things, based
2096 * on the port's current p_init_type_reply claiming a reinit has been
2097 * done or not. The reinit steps are:
2098 * 1. If in InitTypeReply, NoLoadReply == PreserveContentReply == 0, verify
2099 * the old Pkey and GID0 are correct.
2100 * 2. Register for mcg traps (already done by ibmf).
2101 * 3. If PreservePresenceReply indicates the SM has restored port's presence
2102 * in subnet, nothing more to do. Else go to next steps (on async daemon).
2103 * 4. Give up all sendonly memberships.
2104 * 5. Acquire all full memberships.
2105 * 6. In promiscuous mode, acquire all non memberships.
2106 * 7. Recycle all AHs to free list.
2107 */
2108 static void
2109 ibd_link_mod(ibd_state_t *state, ibt_async_code_t code)
2110 {
2111 ibt_hca_portinfo_t *port_infop = NULL;
2112 ibt_status_t ibt_status;
2113 uint_t psize, port_infosz;
2114 ibd_link_op_t opcode;
2115 ibd_req_t *req;
2116 link_state_t new_link_state = LINK_STATE_UP;
2117 uint8_t itreply;
2118 uint16_t pkix;
2119 int ret;
2120
2121 /*
2122 * Let's not race with a plumb or an unplumb; if we detect a
2123 * pkey relocation event later on here, we may have to restart.
2124 */
2125 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
2126
2127 mutex_enter(&state->id_link_mutex);
2128
2129 /*
2130 * If the link state is unknown, a plumb has not yet been attempted
2131 * on the interface. Nothing to do.
2132 */
2133 if (state->id_link_state == LINK_STATE_UNKNOWN) {
2134 mutex_exit(&state->id_link_mutex);
2135 goto link_mod_return;
2136 }
2137
2138 /*
2139 * If link state is down because of plumb failure, and we are not in
2140 * late HCA init, and we were not successfully plumbed, nothing to do.
2141 */
2142 if ((state->id_link_state == LINK_STATE_DOWN) &&
2143 ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 0) &&
2144 ((state->id_mac_state & IBD_DRV_STARTED) == 0)) {
2145 mutex_exit(&state->id_link_mutex);
2146 goto link_mod_return;
2147 }
2148
2149 /*
2150 * If this routine was called in response to a port down event,
2151 * we just need to see if this should be informed.
2152 */
2153 if (code == IBT_ERROR_PORT_DOWN) {
2154 new_link_state = LINK_STATE_DOWN;
2155 goto update_link_state;
2156 }
2157
2158 /*
2159 * If it's not a port down event we've received, try to get the port
2160 * attributes first. If we fail here, the port is as good as down.
2161 * Otherwise, if the link went down by the time the handler gets
2162 * here, give up - we cannot even validate the pkey/gid since those
2163 * are not valid and this is as bad as a port down anyway.
2164 */
2165 ibt_status = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
2166 &port_infop, &psize, &port_infosz);
2167 if ((ibt_status != IBT_SUCCESS) || (psize != 1) ||
2168 (port_infop->p_linkstate != IBT_PORT_ACTIVE)) {
2169 new_link_state = LINK_STATE_DOWN;
2170 goto update_link_state;
2171 }
2172
2173 /*
2174 * If in the previous attempt, the pkey was not found either due to the
2175 * port state being down, or due to it's absence in the pkey table,
2176 * look for it now and try to start the interface.
2177 */
2178 if (state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) {
2179 mutex_exit(&state->id_link_mutex);
2180 if ((ret = ibd_start(state)) != 0) {
2181 DPRINT(10, "ibd_linkmod: cannot start from late HCA "
2182 "init, ret=%d", ret);
2183 }
2184 ibt_free_portinfo(port_infop, port_infosz);
2185 goto link_mod_return;
2186 }
2187
2188 /*
2189 * Check the SM InitTypeReply flags. If both NoLoadReply and
2190 * PreserveContentReply are 0, we don't know anything about the
2191 * data loaded into the port attributes, so we need to verify
2192 * if gid0 and pkey are still valid.
2193 */
2194 itreply = port_infop->p_init_type_reply;
2195 if (((itreply & SM_INIT_TYPE_REPLY_NO_LOAD_REPLY) == 0) &&
2196 ((itreply & SM_INIT_TYPE_PRESERVE_CONTENT_REPLY) == 0)) {
2197 /*
2198 * Check to see if the subnet part of GID0 has changed. If
2199 * not, check the simple case first to see if the pkey
2200 * index is the same as before; finally check to see if the
2201 * pkey has been relocated to a different index in the table.
2202 */
2203 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
2204 if (bcmp(port_infop->p_sgid_tbl,
2205 &state->id_sgid, sizeof (ib_gid_t)) != 0) {
2206
2207 new_link_state = LINK_STATE_DOWN;
2208
2209 } else if (port_infop->p_pkey_tbl[state->id_pkix] ==
2210 state->id_pkey) {
2211
2212 new_link_state = LINK_STATE_UP;
2213
2214 } else if (ibd_locate_pkey(port_infop->p_pkey_tbl,
2215 port_infop->p_pkey_tbl_sz, state->id_pkey, &pkix) == 0) {
2216
2217 ibt_free_portinfo(port_infop, port_infosz);
2218 mutex_exit(&state->id_link_mutex);
2219
2220 /*
2221 * Currently a restart is required if our pkey has moved
2222 * in the pkey table. If we get the ibt_recycle_ud() to
2223 * work as documented (expected), we may be able to
2224 * avoid a complete restart. Note that we've already
2225 * marked both the start and stop 'in-progress' flags,
2226 * so it is ok to go ahead and do this restart.
2227 */
2228 (void) ibd_undo_start(state, LINK_STATE_DOWN);
2229 if ((ret = ibd_start(state)) != 0) {
2230 DPRINT(10, "ibd_restart: cannot restart, "
2231 "ret=%d", ret);
2232 }
2233
2234 goto link_mod_return;
2235 } else {
2236 new_link_state = LINK_STATE_DOWN;
2237 }
2238 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
2239 }
2240
2241 update_link_state:
2242 if (port_infop) {
2243 ibt_free_portinfo(port_infop, port_infosz);
2244 }
2245
2246 /*
2247 * If we're reporting a link up, check InitTypeReply to see if
2248 * the SM has ensured that the port's presence in mcg, traps,
2249 * etc. is intact.
2250 */
2251 if (new_link_state == LINK_STATE_DOWN) {
2252 opcode = IBD_LINK_DOWN;
2253 } else {
2254 if ((itreply & SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) ==
2255 SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) {
2256 opcode = IBD_LINK_UP;
2257 } else {
2258 opcode = IBD_LINK_UP_ABSENT;
2259 }
2260 }
2261
2262 /*
2263 * If the old state is the same as the new state, and the SM indicated
2264 * no change in the port parameters, nothing to do.
2265 */
2266 if ((state->id_link_state == new_link_state) && (opcode !=
2267 IBD_LINK_UP_ABSENT)) {
2268 mutex_exit(&state->id_link_mutex);
2269 goto link_mod_return;
2270 }
2271
2272 /*
2273 * Ok, so there was a link state change; see if it's safe to ask
2274 * the async thread to do the work
2275 */
2276 if (!ibd_async_safe(state)) {
2277 state->id_link_state = new_link_state;
2278 mutex_exit(&state->id_link_mutex);
2279 goto link_mod_return;
2280 }
2281
2282 mutex_exit(&state->id_link_mutex);
2283
2284 /*
2285 * Queue up a request for ibd_async_link() to handle this link
2286 * state change event
2287 */
2288 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
2289 req->rq_ptr = (void *)opcode;
2290 ibd_queue_work_slot(state, req, IBD_ASYNC_LINK);
2291
2292 link_mod_return:
2293 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
2294 }
2295
2296 /*
2297 * For the port up/down events, IBTL guarantees there will not be concurrent
2298 * invocations of the handler. IBTL might coalesce link transition events,
2299 * and not invoke the handler for _each_ up/down transition, but it will
2300 * invoke the handler with last known state
2301 */
2302 static void
2303 ibd_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
2304 ibt_async_code_t code, ibt_async_event_t *event)
2305 {
2306 ibd_state_t *state = (ibd_state_t *)clnt_private;
2307
2308 switch (code) {
2309 case IBT_ERROR_CATASTROPHIC_CHAN:
2310 ibd_print_warn(state, "catastrophic channel error");
2311 break;
2312 case IBT_ERROR_CQ:
2313 ibd_print_warn(state, "completion queue error");
2314 break;
2315 case IBT_PORT_CHANGE_EVENT:
2316 /*
2317 * Events will be delivered to all instances that have
2318 * done ibt_open_hca() but not yet done ibt_close_hca().
2319 * Only need to do work for our port; IBTF will deliver
2320 * events for other ports on the hca we have ibt_open_hca'ed
2321 * too. Note that id_port is initialized in ibd_attach()
2322 * before we do an ibt_open_hca() in ibd_attach().
2323 */
2324 ASSERT(state->id_hca_hdl == hca_hdl);
2325 if (state->id_port != event->ev_port)
2326 break;
2327
2328 if ((event->ev_port_flags & IBT_PORT_CHANGE_PKEY) ==
2329 IBT_PORT_CHANGE_PKEY) {
2330 ibd_link_mod(state, code);
2331 }
2332 break;
2333 case IBT_ERROR_PORT_DOWN:
2334 case IBT_CLNT_REREG_EVENT:
2335 case IBT_EVENT_PORT_UP:
2336 /*
2337 * Events will be delivered to all instances that have
2338 * done ibt_open_hca() but not yet done ibt_close_hca().
2339 * Only need to do work for our port; IBTF will deliver
2340 * events for other ports on the hca we have ibt_open_hca'ed
2341 * too. Note that id_port is initialized in ibd_attach()
2342 * before we do an ibt_open_hca() in ibd_attach().
2343 */
2344 ASSERT(state->id_hca_hdl == hca_hdl);
2345 if (state->id_port != event->ev_port)
2346 break;
2347
2348 ibd_link_mod(state, code);
2349 break;
2350
2351 case IBT_HCA_ATTACH_EVENT:
2352 case IBT_HCA_DETACH_EVENT:
2353 /*
2354 * When a new card is plugged to the system, attach_event is
2355 * invoked. Additionally, a cfgadm needs to be run to make the
2356 * card known to the system, and an ifconfig needs to be run to
2357 * plumb up any ibd interfaces on the card. In the case of card
2358 * unplug, a cfgadm is run that will trigger any RCM scripts to
2359 * unplumb the ibd interfaces on the card; when the card is
2360 * actually unplugged, the detach_event is invoked;
2361 * additionally, if any ibd instances are still active on the
2362 * card (eg there were no associated RCM scripts), driver's
2363 * detach routine is invoked.
2364 */
2365 break;
2366 default:
2367 break;
2368 }
2369 }
2370
2371 static int
2372 ibd_register_mac(ibd_state_t *state, dev_info_t *dip)
2373 {
2374 mac_register_t *macp;
2375 int ret;
2376
2377 if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
2378 DPRINT(10, "ibd_register_mac: mac_alloc() failed");
2379 return (DDI_FAILURE);
2380 }
2381
2382 /*
2383 * Note that when we register with mac during attach, we don't
2384 * have the id_macaddr yet, so we'll simply be registering a
2385 * zero macaddr that we'll overwrite later during plumb (in
2386 * ibd_m_start()). Similar is the case with id_mtu - we'll
2387 * update the mac layer with the correct mtu during plumb.
2388 */
2389 macp->m_type_ident = MAC_PLUGIN_IDENT_IB;
2390 macp->m_driver = state;
2391 macp->m_dip = dip;
2392 macp->m_src_addr = (uint8_t *)&state->id_macaddr;
2393 macp->m_callbacks = &ibd_m_callbacks;
2394 macp->m_min_sdu = 0;
2395 macp->m_multicast_sdu = IBD_DEF_MAX_SDU;
2396 if (state->id_type == IBD_PORT_DRIVER) {
2397 macp->m_max_sdu = IBD_DEF_RC_MAX_SDU;
2398 } else if (state->id_enable_rc) {
2399 macp->m_max_sdu = state->rc_mtu - IPOIB_HDRSIZE;
2400 } else {
2401 macp->m_max_sdu = IBD_DEF_MAX_SDU;
2402 }
2403 macp->m_priv_props = ibd_priv_props;
2404
2405 /*
2406 * Register ourselves with the GLDv3 interface
2407 */
2408 if ((ret = mac_register(macp, &state->id_mh)) != 0) {
2409 mac_free(macp);
2410 DPRINT(10,
2411 "ibd_register_mac: mac_register() failed, ret=%d", ret);
2412 return (DDI_FAILURE);
2413 }
2414
2415 mac_free(macp);
2416 return (DDI_SUCCESS);
2417 }
2418
2419 static int
2420 ibd_record_capab(ibd_state_t *state)
2421 {
2422 ibt_hca_attr_t hca_attrs;
2423 ibt_status_t ibt_status;
2424
2425 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
2426
2427 /*
2428 * Query the HCA and fetch its attributes
2429 */
2430 ibt_status = ibt_query_hca(state->id_hca_hdl, &hca_attrs);
2431 ASSERT(ibt_status == IBT_SUCCESS);
2432
2433 /*
2434 * 1. Set the Hardware Checksum capability. Currently we only consider
2435 * full checksum offload.
2436 */
2437 if (state->id_enable_rc) {
2438 state->id_hwcksum_capab = 0;
2439 } else {
2440 if ((hca_attrs.hca_flags & IBT_HCA_CKSUM_FULL)
2441 == IBT_HCA_CKSUM_FULL) {
2442 state->id_hwcksum_capab = IBT_HCA_CKSUM_FULL;
2443 }
2444 }
2445
2446 /*
2447 * 2. Set LSO policy, capability and maximum length
2448 */
2449 if (state->id_enable_rc) {
2450 state->id_lso_capable = B_FALSE;
2451 state->id_lso_maxlen = 0;
2452 } else {
2453 if (hca_attrs.hca_max_lso_size > 0) {
2454 state->id_lso_capable = B_TRUE;
2455 if (hca_attrs.hca_max_lso_size > IBD_LSO_MAXLEN)
2456 state->id_lso_maxlen = IBD_LSO_MAXLEN;
2457 else
2458 state->id_lso_maxlen =
2459 hca_attrs.hca_max_lso_size;
2460 } else {
2461 state->id_lso_capable = B_FALSE;
2462 state->id_lso_maxlen = 0;
2463 }
2464 }
2465
2466 /*
2467 * 3. Set Reserved L_Key capability
2468 */
2469 if ((hca_attrs.hca_flags2 & IBT_HCA2_RES_LKEY) == IBT_HCA2_RES_LKEY) {
2470 state->id_hca_res_lkey_capab = 1;
2471 state->id_res_lkey = hca_attrs.hca_reserved_lkey;
2472 state->rc_enable_iov_map = B_TRUE;
2473 } else {
2474 /* If no reserved lkey, we will not use ibt_map_mem_iov */
2475 state->rc_enable_iov_map = B_FALSE;
2476 }
2477
2478 /*
2479 * 4. Set maximum sqseg value after checking to see if extended sgl
2480 * size information is provided by the hca
2481 */
2482 if (hca_attrs.hca_flags & IBT_HCA_WQE_SIZE_INFO) {
2483 state->id_max_sqseg = hca_attrs.hca_ud_send_sgl_sz;
2484 state->rc_tx_max_sqseg = hca_attrs.hca_conn_send_sgl_sz;
2485 } else {
2486 state->id_max_sqseg = hca_attrs.hca_max_sgl;
2487 state->rc_tx_max_sqseg = hca_attrs.hca_max_sgl;
2488 }
2489 if (state->id_max_sqseg > IBD_MAX_SQSEG) {
2490 state->id_max_sqseg = IBD_MAX_SQSEG;
2491 } else if (state->id_max_sqseg < IBD_MAX_SQSEG) {
2492 ibd_print_warn(state, "Set #sgl = %d instead of default %d",
2493 state->id_max_sqseg, IBD_MAX_SQSEG);
2494 }
2495 if (state->rc_tx_max_sqseg > IBD_MAX_SQSEG) {
2496 state->rc_tx_max_sqseg = IBD_MAX_SQSEG;
2497 } else if (state->rc_tx_max_sqseg < IBD_MAX_SQSEG) {
2498 ibd_print_warn(state, "RC mode: Set #sgl = %d instead of "
2499 "default %d", state->rc_tx_max_sqseg, IBD_MAX_SQSEG);
2500 }
2501
2502 /*
2503 * Translating the virtual address regions into physical regions
2504 * for using the Reserved LKey feature results in a wr sgl that
2505 * is a little longer. Since failing ibt_map_mem_iov() is costly,
2506 * we'll fix a high-water mark (65%) for when we should stop.
2507 */
2508 state->id_max_sqseg_hiwm = (state->id_max_sqseg * 65) / 100;
2509 state->rc_max_sqseg_hiwm = (state->rc_tx_max_sqseg * 65) / 100;
2510
2511 /*
2512 * 5. Set number of recv and send wqes after checking hca maximum
2513 * channel size. Store the max channel size in the state so that it
2514 * can be referred to when the swqe/rwqe change is requested via
2515 * dladm.
2516 */
2517
2518 state->id_hca_max_chan_sz = hca_attrs.hca_max_chan_sz;
2519
2520 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_rwqe)
2521 state->id_ud_num_rwqe = hca_attrs.hca_max_chan_sz;
2522
2523 state->id_rx_bufs_outstanding_limit = state->id_ud_num_rwqe -
2524 IBD_RWQE_MIN;
2525
2526 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_swqe)
2527 state->id_ud_num_swqe = hca_attrs.hca_max_chan_sz;
2528
2529 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))
2530
2531 return (DDI_SUCCESS);
2532 }
2533
2534 static int
2535 ibd_part_busy(ibd_state_t *state)
2536 {
2537 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) != 0) {
2538 DPRINT(10, "ibd_part_busy: failed: rx bufs outstanding\n");
2539 return (DDI_FAILURE);
2540 }
2541
2542 if (state->rc_srq_rwqe_list.dl_bufs_outstanding != 0) {
2543 DPRINT(10, "ibd_part_busy: failed: srq bufs outstanding\n");
2544 return (DDI_FAILURE);
2545 }
2546
2547 /*
2548 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB port is
2549 * connecting to a remote IPoIB port. We can't remove this port.
2550 */
2551 if (state->id_ah_op == IBD_OP_ONGOING) {
2552 DPRINT(10, "ibd_part_busy: failed: connecting\n");
2553 return (DDI_FAILURE);
2554 }
2555
2556 return (DDI_SUCCESS);
2557 }
2558
2559
2560 static void
2561 ibd_part_unattach(ibd_state_t *state)
2562 {
2563 uint32_t progress = state->id_mac_state;
2564 ibt_status_t ret;
2565
2566 /* make sure rx resources are freed */
2567 ibd_free_rx_rsrcs(state);
2568
2569 if (progress & IBD_DRV_RC_SRQ_ALLOCD) {
2570 ASSERT(state->id_enable_rc);
2571 ibd_rc_fini_srq_list(state);
2572 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD);
2573 }
2574
2575 if (progress & IBD_DRV_MAC_REGISTERED) {
2576 (void) mac_unregister(state->id_mh);
2577 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED);
2578 }
2579
2580 if (progress & IBD_DRV_ASYNC_THR_CREATED) {
2581 /*
2582 * No new async requests will be posted since the device
2583 * link state has been marked as unknown; completion handlers
2584 * have been turned off, so Tx handler will not cause any
2585 * more IBD_ASYNC_REAP requests.
2586 *
2587 * Queue a request for the async thread to exit, which will
2588 * be serviced after any pending ones. This can take a while,
2589 * specially if the SM is unreachable, since IBMF will slowly
2590 * timeout each SM request issued by the async thread. Reap
2591 * the thread before continuing on, we do not want it to be
2592 * lingering in modunloaded code.
2593 */
2594 ibd_queue_work_slot(state, &state->id_ah_req, IBD_ASYNC_EXIT);
2595 thread_join(state->id_async_thrid);
2596
2597 state->id_mac_state &= (~IBD_DRV_ASYNC_THR_CREATED);
2598 }
2599
2600 if (progress & IBD_DRV_REQ_LIST_INITED) {
2601 list_destroy(&state->id_req_list);
2602 mutex_destroy(&state->id_acache_req_lock);
2603 cv_destroy(&state->id_acache_req_cv);
2604 state->id_mac_state &= ~IBD_DRV_REQ_LIST_INITED;
2605 }
2606
2607 if (progress & IBD_DRV_PD_ALLOCD) {
2608 if ((ret = ibt_free_pd(state->id_hca_hdl,
2609 state->id_pd_hdl)) != IBT_SUCCESS) {
2610 ibd_print_warn(state, "failed to free "
2611 "protection domain, ret=%d", ret);
2612 }
2613 state->id_pd_hdl = NULL;
2614 state->id_mac_state &= (~IBD_DRV_PD_ALLOCD);
2615 }
2616
2617 if (progress & IBD_DRV_HCA_OPENED) {
2618 if ((ret = ibt_close_hca(state->id_hca_hdl)) !=
2619 IBT_SUCCESS) {
2620 ibd_print_warn(state, "failed to close "
2621 "HCA device, ret=%d", ret);
2622 }
2623 state->id_hca_hdl = NULL;
2624 state->id_mac_state &= (~IBD_DRV_HCA_OPENED);
2625 }
2626
2627 mutex_enter(&ibd_gstate.ig_mutex);
2628 if (progress & IBD_DRV_IBTL_ATTACH_DONE) {
2629 if ((ret = ibt_detach(state->id_ibt_hdl)) !=
2630 IBT_SUCCESS) {
2631 ibd_print_warn(state,
2632 "ibt_detach() failed, ret=%d", ret);
2633 }
2634 state->id_ibt_hdl = NULL;
2635 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE);
2636 ibd_gstate.ig_ibt_hdl_ref_cnt--;
2637 }
2638 if ((ibd_gstate.ig_ibt_hdl_ref_cnt == 0) &&
2639 (ibd_gstate.ig_ibt_hdl != NULL)) {
2640 if ((ret = ibt_detach(ibd_gstate.ig_ibt_hdl)) !=
2641 IBT_SUCCESS) {
2642 ibd_print_warn(state, "ibt_detach(): global "
2643 "failed, ret=%d", ret);
2644 }
2645 ibd_gstate.ig_ibt_hdl = NULL;
2646 }
2647 mutex_exit(&ibd_gstate.ig_mutex);
2648
2649 if (progress & IBD_DRV_TXINTR_ADDED) {
2650 ddi_remove_softintr(state->id_tx);
2651 state->id_tx = NULL;
2652 state->id_mac_state &= (~IBD_DRV_TXINTR_ADDED);
2653 }
2654
2655 if (progress & IBD_DRV_RXINTR_ADDED) {
2656 ddi_remove_softintr(state->id_rx);
2657 state->id_rx = NULL;
2658 state->id_mac_state &= (~IBD_DRV_RXINTR_ADDED);
2659 }
2660
2661 #ifdef DEBUG
2662 if (progress & IBD_DRV_RC_PRIVATE_STATE) {
2663 kstat_delete(state->rc_ksp);
2664 state->id_mac_state &= (~IBD_DRV_RC_PRIVATE_STATE);
2665 }
2666 #endif
2667
2668 if (progress & IBD_DRV_STATE_INITIALIZED) {
2669 ibd_state_fini(state);
2670 state->id_mac_state &= (~IBD_DRV_STATE_INITIALIZED);
2671 }
2672 }
2673
2674 int
2675 ibd_part_attach(ibd_state_t *state, dev_info_t *dip)
2676 {
2677 ibt_status_t ret;
2678 int rv;
2679 kthread_t *kht;
2680
2681 /*
2682 * Initialize mutexes and condition variables
2683 */
2684 if (ibd_state_init(state, dip) != DDI_SUCCESS) {
2685 DPRINT(10, "ibd_part_attach: failed in ibd_state_init()");
2686 return (DDI_FAILURE);
2687 }
2688 state->id_mac_state |= IBD_DRV_STATE_INITIALIZED;
2689
2690 /*
2691 * Allocate rx,tx softintr
2692 */
2693 if (ibd_rx_softintr == 1) {
2694 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_rx,
2695 NULL, NULL, ibd_intr, (caddr_t)state)) != DDI_SUCCESS) {
2696 DPRINT(10, "ibd_part_attach: failed in "
2697 "ddi_add_softintr(id_rx), ret=%d", rv);
2698 return (DDI_FAILURE);
2699 }
2700 state->id_mac_state |= IBD_DRV_RXINTR_ADDED;
2701 }
2702 if (ibd_tx_softintr == 1) {
2703 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_tx,
2704 NULL, NULL, ibd_tx_recycle,
2705 (caddr_t)state)) != DDI_SUCCESS) {
2706 DPRINT(10, "ibd_part_attach: failed in "
2707 "ddi_add_softintr(id_tx), ret=%d", rv);
2708 return (DDI_FAILURE);
2709 }
2710 state->id_mac_state |= IBD_DRV_TXINTR_ADDED;
2711 }
2712
2713 /*
2714 * Attach to IBTL
2715 */
2716 mutex_enter(&ibd_gstate.ig_mutex);
2717 if (ibd_gstate.ig_ibt_hdl == NULL) {
2718 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state,
2719 &ibd_gstate.ig_ibt_hdl)) != IBT_SUCCESS) {
2720 DPRINT(10, "ibd_part_attach: global: failed in "
2721 "ibt_attach(), ret=%d", ret);
2722 mutex_exit(&ibd_gstate.ig_mutex);
2723 return (DDI_FAILURE);
2724 }
2725 }
2726 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state,
2727 &state->id_ibt_hdl)) != IBT_SUCCESS) {
2728 DPRINT(10, "ibd_part_attach: failed in ibt_attach(), ret=%d",
2729 ret);
2730 mutex_exit(&ibd_gstate.ig_mutex);
2731 return (DDI_FAILURE);
2732 }
2733 ibd_gstate.ig_ibt_hdl_ref_cnt++;
2734 mutex_exit(&ibd_gstate.ig_mutex);
2735 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE;
2736
2737 /*
2738 * Open the HCA
2739 */
2740 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid,
2741 &state->id_hca_hdl)) != IBT_SUCCESS) {
2742 DPRINT(10, "ibd_part_attach: ibt_open_hca() failed, ret=%d",
2743 ret);
2744 return (DDI_FAILURE);
2745 }
2746 state->id_mac_state |= IBD_DRV_HCA_OPENED;
2747
2748 #ifdef DEBUG
2749 /* Initialize Driver Counters for Reliable Connected Mode */
2750 if (state->id_enable_rc) {
2751 if (ibd_rc_init_stats(state) != DDI_SUCCESS) {
2752 DPRINT(10, "ibd_part_attach: failed in "
2753 "ibd_rc_init_stats");
2754 return (DDI_FAILURE);
2755 }
2756 state->id_mac_state |= IBD_DRV_RC_PRIVATE_STATE;
2757 }
2758 #endif
2759
2760 /*
2761 * Record capabilities
2762 */
2763 (void) ibd_record_capab(state);
2764
2765 /*
2766 * Allocate a protection domain on the HCA
2767 */
2768 if ((ret = ibt_alloc_pd(state->id_hca_hdl, IBT_PD_NO_FLAGS,
2769 &state->id_pd_hdl)) != IBT_SUCCESS) {
2770 DPRINT(10, "ibd_part_attach: ibt_alloc_pd() failed, ret=%d",
2771 ret);
2772 return (DDI_FAILURE);
2773 }
2774 state->id_mac_state |= IBD_DRV_PD_ALLOCD;
2775
2776
2777 /*
2778 * We need to initialise the req_list that is required for the
2779 * operation of the async_thread.
2780 */
2781 mutex_init(&state->id_acache_req_lock, NULL, MUTEX_DRIVER, NULL);
2782 cv_init(&state->id_acache_req_cv, NULL, CV_DEFAULT, NULL);
2783 list_create(&state->id_req_list, sizeof (ibd_req_t),
2784 offsetof(ibd_req_t, rq_list));
2785 state->id_mac_state |= IBD_DRV_REQ_LIST_INITED;
2786
2787 /*
2788 * Create the async thread; thread_create never fails.
2789 */
2790 kht = thread_create(NULL, 0, ibd_async_work, state, 0, &p0,
2791 TS_RUN, minclsyspri);
2792 state->id_async_thrid = kht->t_did;
2793 state->id_mac_state |= IBD_DRV_ASYNC_THR_CREATED;
2794
2795 return (DDI_SUCCESS);
2796 }
2797
2798 /*
2799 * Attach device to the IO framework.
2800 */
2801 static int
2802 ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2803 {
2804 int ret;
2805
2806 switch (cmd) {
2807 case DDI_ATTACH:
2808 ret = ibd_port_attach(dip);
2809 break;
2810 default:
2811 ret = DDI_FAILURE;
2812 break;
2813 }
2814 return (ret);
2815 }
2816
2817 /*
2818 * Detach device from the IO framework.
2819 */
2820 static int
2821 ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2822 {
2823 ibd_state_t *state;
2824 int instance;
2825
2826 /*
2827 * IBD doesn't support suspend/resume
2828 */
2829 if (cmd != DDI_DETACH)
2830 return (DDI_FAILURE);
2831
2832 /*
2833 * Get the instance softstate
2834 */
2835 instance = ddi_get_instance(dip);
2836 state = ddi_get_soft_state(ibd_list, instance);
2837
2838 /*
2839 * Release all resources we're holding still. Note that if we'd
2840 * done ibd_attach(), ibd_m_start() and ibd_m_stop() correctly
2841 * so far, we should find all the flags we need in id_mac_state.
2842 */
2843 return (ibd_port_unattach(state, dip));
2844 }
2845
2846 /*
2847 * Pre ibt_attach() driver initialization
2848 */
2849 static int
2850 ibd_state_init(ibd_state_t *state, dev_info_t *dip)
2851 {
2852 char buf[64];
2853
2854 mutex_init(&state->id_link_mutex, NULL, MUTEX_DRIVER, NULL);
2855 state->id_link_state = LINK_STATE_UNKNOWN;
2856
2857 mutex_init(&state->id_trap_lock, NULL, MUTEX_DRIVER, NULL);
2858 cv_init(&state->id_trap_cv, NULL, CV_DEFAULT, NULL);
2859 state->id_trap_stop = B_TRUE;
2860 state->id_trap_inprog = 0;
2861
2862 mutex_init(&state->id_scq_poll_lock, NULL, MUTEX_DRIVER, NULL);
2863 mutex_init(&state->id_rcq_poll_lock, NULL, MUTEX_DRIVER, NULL);
2864 state->id_dip = dip;
2865
2866 mutex_init(&state->id_sched_lock, NULL, MUTEX_DRIVER, NULL);
2867
2868 mutex_init(&state->id_tx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2869 mutex_init(&state->id_tx_rel_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2870 mutex_init(&state->id_txpost_lock, NULL, MUTEX_DRIVER, NULL);
2871 state->id_tx_busy = 0;
2872 mutex_init(&state->id_lso_lock, NULL, MUTEX_DRIVER, NULL);
2873
2874 state->id_rx_list.dl_bufs_outstanding = 0;
2875 state->id_rx_list.dl_cnt = 0;
2876 mutex_init(&state->id_rx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2877 mutex_init(&state->id_rx_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2878 (void) sprintf(buf, "ibd_req%d_%x_%u", ddi_get_instance(dip),
2879 state->id_pkey, state->id_plinkid);
2880 state->id_req_kmc = kmem_cache_create(buf, sizeof (ibd_req_t),
2881 0, NULL, NULL, NULL, NULL, NULL, 0);
2882
2883 /* For Reliable Connected Mode */
2884 mutex_init(&state->rc_rx_lock, NULL, MUTEX_DRIVER, NULL);
2885 mutex_init(&state->rc_tx_large_bufs_lock, NULL, MUTEX_DRIVER, NULL);
2886 mutex_init(&state->rc_srq_rwqe_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2887 mutex_init(&state->rc_srq_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2888 mutex_init(&state->rc_pass_chan_list.chan_list_mutex, NULL,
2889 MUTEX_DRIVER, NULL);
2890 mutex_init(&state->rc_timeout_lock, NULL, MUTEX_DRIVER, NULL);
2891
2892 /*
2893 * Make the default link mode as RC. If this fails during connection
2894 * setup, the link mode is automatically transitioned to UD.
2895 * Also set the RC MTU.
2896 */
2897 state->id_enable_rc = IBD_DEF_LINK_MODE;
2898 state->rc_mtu = IBD_DEF_RC_MAX_MTU;
2899 state->id_mtu = IBD_DEF_MAX_MTU;
2900
2901 /* Iniatialize all tunables to default */
2902 state->id_lso_policy = IBD_DEF_LSO_POLICY;
2903 state->id_num_lso_bufs = IBD_DEF_NUM_LSO_BUFS;
2904 state->id_num_ah = IBD_DEF_NUM_AH;
2905 state->id_hash_size = IBD_DEF_HASH_SIZE;
2906 state->id_create_broadcast_group = IBD_DEF_CREATE_BCAST_GROUP;
2907 state->id_allow_coalesce_comp_tuning = IBD_DEF_COALESCE_COMPLETIONS;
2908 state->id_ud_rx_comp_count = IBD_DEF_UD_RX_COMP_COUNT;
2909 state->id_ud_rx_comp_usec = IBD_DEF_UD_RX_COMP_USEC;
2910 state->id_ud_tx_comp_count = IBD_DEF_UD_TX_COMP_COUNT;
2911 state->id_ud_tx_comp_usec = IBD_DEF_UD_TX_COMP_USEC;
2912 state->id_rc_rx_comp_count = IBD_DEF_RC_RX_COMP_COUNT;
2913 state->id_rc_rx_comp_usec = IBD_DEF_RC_RX_COMP_USEC;
2914 state->id_rc_tx_comp_count = IBD_DEF_RC_TX_COMP_COUNT;
2915 state->id_rc_tx_comp_usec = IBD_DEF_RC_TX_COMP_USEC;
2916 state->id_ud_tx_copy_thresh = IBD_DEF_UD_TX_COPY_THRESH;
2917 state->id_rc_rx_copy_thresh = IBD_DEF_RC_RX_COPY_THRESH;
2918 state->id_rc_tx_copy_thresh = IBD_DEF_RC_TX_COPY_THRESH;
2919 state->id_ud_num_rwqe = IBD_DEF_UD_NUM_RWQE;
2920 state->id_ud_num_swqe = IBD_DEF_UD_NUM_SWQE;
2921 state->id_rc_num_rwqe = IBD_DEF_RC_NUM_RWQE;
2922 state->id_rc_num_swqe = IBD_DEF_RC_NUM_SWQE;
2923 state->rc_enable_srq = IBD_DEF_RC_ENABLE_SRQ;
2924 state->id_rc_num_srq = IBD_DEF_RC_NUM_SRQ;
2925 state->id_rc_rx_rwqe_thresh = IBD_DEF_RC_RX_RWQE_THRESH;
2926
2927 return (DDI_SUCCESS);
2928 }
2929
2930 /*
2931 * Post ibt_detach() driver deconstruction
2932 */
2933 static void
2934 ibd_state_fini(ibd_state_t *state)
2935 {
2936 kmem_cache_destroy(state->id_req_kmc);
2937
2938 mutex_destroy(&state->id_rx_list.dl_mutex);
2939 mutex_destroy(&state->id_rx_free_list.dl_mutex);
2940
2941 mutex_destroy(&state->id_txpost_lock);
2942 mutex_destroy(&state->id_tx_list.dl_mutex);
2943 mutex_destroy(&state->id_tx_rel_list.dl_mutex);
2944 mutex_destroy(&state->id_lso_lock);
2945
2946 mutex_destroy(&state->id_sched_lock);
2947 mutex_destroy(&state->id_scq_poll_lock);
2948 mutex_destroy(&state->id_rcq_poll_lock);
2949
2950 cv_destroy(&state->id_trap_cv);
2951 mutex_destroy(&state->id_trap_lock);
2952 mutex_destroy(&state->id_link_mutex);
2953
2954 /* For Reliable Connected Mode */
2955 mutex_destroy(&state->rc_timeout_lock);
2956 mutex_destroy(&state->rc_srq_free_list.dl_mutex);
2957 mutex_destroy(&state->rc_srq_rwqe_list.dl_mutex);
2958 mutex_destroy(&state->rc_pass_chan_list.chan_list_mutex);
2959 mutex_destroy(&state->rc_tx_large_bufs_lock);
2960 mutex_destroy(&state->rc_rx_lock);
2961 }
2962
2963 /*
2964 * Fetch link speed from SA for snmp ifspeed reporting.
2965 */
2966 static uint64_t
2967 ibd_get_portspeed(ibd_state_t *state)
2968 {
2969 int ret;
2970 ibt_path_info_t path;
2971 ibt_path_attr_t path_attr;
2972 uint8_t num_paths;
2973 uint64_t ifspeed;
2974
2975 /*
2976 * Due to serdes 8b10b encoding on the wire, 2.5 Gbps on wire
2977 * translates to 2 Gbps data rate. Thus, 1X single data rate is
2978 * 2000000000. Start with that as default.
2979 */
2980 ifspeed = 2000000000;
2981
2982 bzero(&path_attr, sizeof (path_attr));
2983
2984 /*
2985 * Get the port speed from Loopback path information.
2986 */
2987 path_attr.pa_dgids = &state->id_sgid;
2988 path_attr.pa_num_dgids = 1;
2989 path_attr.pa_sgid = state->id_sgid;
2990
2991 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS,
2992 &path_attr, 1, &path, &num_paths) != IBT_SUCCESS)
2993 goto earlydone;
2994
2995 if (num_paths < 1)
2996 goto earlydone;
2997
2998 /*
2999 * In case SA does not return an expected value, report the default
3000 * speed as 1X.
3001 */
3002 ret = 1;
3003 switch (path.pi_prim_cep_path.cep_adds_vect.av_srate) {
3004 case IBT_SRATE_2: /* 1X SDR i.e 2.5 Gbps */
3005 ret = 1;
3006 break;
3007 case IBT_SRATE_10: /* 4X SDR or 1X QDR i.e 10 Gbps */
3008 ret = 4;
3009 break;
3010 case IBT_SRATE_30: /* 12X SDR i.e 30 Gbps */
3011 ret = 12;
3012 break;
3013 case IBT_SRATE_5: /* 1X DDR i.e 5 Gbps */
3014 ret = 2;
3015 break;
3016 case IBT_SRATE_20: /* 4X DDR or 8X SDR i.e 20 Gbps */
3017 ret = 8;
3018 break;
3019 case IBT_SRATE_40: /* 8X DDR or 4X QDR i.e 40 Gbps */
3020 ret = 16;
3021 break;
3022 case IBT_SRATE_60: /* 12X DDR i.e 60 Gbps */
3023 ret = 24;
3024 break;
3025 case IBT_SRATE_80: /* 8X QDR i.e 80 Gbps */
3026 ret = 32;
3027 break;
3028 case IBT_SRATE_120: /* 12X QDR i.e 120 Gbps */
3029 ret = 48;
3030 break;
3031 }
3032
3033 ifspeed *= ret;
3034
3035 earlydone:
3036 return (ifspeed);
3037 }
3038
3039 /*
3040 * Search input mcg list (id_mc_full or id_mc_non) for an entry
3041 * representing the input mcg mgid.
3042 */
3043 static ibd_mce_t *
3044 ibd_mcache_find(ib_gid_t mgid, struct list *mlist)
3045 {
3046 ibd_mce_t *ptr = list_head(mlist);
3047
3048 /*
3049 * Do plain linear search.
3050 */
3051 while (ptr != NULL) {
3052 if (bcmp(&mgid, &ptr->mc_info.mc_adds_vect.av_dgid,
3053 sizeof (ib_gid_t)) == 0)
3054 return (ptr);
3055 ptr = list_next(mlist, ptr);
3056 }
3057 return (NULL);
3058 }
3059
3060 /*
3061 * Execute IBA JOIN.
3062 */
3063 static ibt_status_t
3064 ibd_iba_join(ibd_state_t *state, ib_gid_t mgid, ibd_mce_t *mce)
3065 {
3066 ibt_mcg_attr_t mcg_attr;
3067
3068 bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
3069 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
3070 mcg_attr.mc_mgid = mgid;
3071 mcg_attr.mc_join_state = mce->mc_jstate;
3072 mcg_attr.mc_scope = state->id_scope;
3073 mcg_attr.mc_pkey = state->id_pkey;
3074 mcg_attr.mc_flow = state->id_mcinfo->mc_adds_vect.av_flow;
3075 mcg_attr.mc_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
3076 mcg_attr.mc_tclass = state->id_mcinfo->mc_adds_vect.av_tclass;
3077 return (ibt_join_mcg(state->id_sgid, &mcg_attr, &mce->mc_info,
3078 NULL, NULL));
3079 }
3080
3081 /*
3082 * This code JOINs the port in the proper way (depending on the join
3083 * state) so that IBA fabric will forward mcg packets to/from the port.
3084 * It also attaches the QPN to the mcg so it can receive those mcg
3085 * packets. This code makes sure not to attach the mcg to the QP if
3086 * that has been previously done due to the mcg being joined with a
3087 * different join state, even though this is not required by SWG_0216,
3088 * refid 3610.
3089 */
3090 static ibd_mce_t *
3091 ibd_join_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
3092 {
3093 ibt_status_t ibt_status;
3094 ibd_mce_t *mce, *tmce, *omce = NULL;
3095 boolean_t do_attach = B_TRUE;
3096
3097 DPRINT(2, "ibd_join_group : join_group state %d : %016llx:%016llx\n",
3098 jstate, mgid.gid_prefix, mgid.gid_guid);
3099
3100 /*
3101 * For enable_multicast Full member joins, we need to do some
3102 * extra work. If there is already an mce on the list that
3103 * indicates full membership, that means the membership has
3104 * not yet been dropped (since the disable_multicast was issued)
3105 * because there are pending Tx's to the mcg; in that case, just
3106 * mark the mce not to be reaped when the Tx completion queues
3107 * an async reap operation.
3108 *
3109 * If there is already an mce on the list indicating sendonly
3110 * membership, try to promote to full membership. Be careful
3111 * not to deallocate the old mce, since there might be an AH
3112 * pointing to it; instead, update the old mce with new data
3113 * that tracks the full membership.
3114 */
3115 if ((jstate == IB_MC_JSTATE_FULL) && ((omce =
3116 IBD_MCACHE_FIND_FULL(state, mgid)) != NULL)) {
3117 if (omce->mc_jstate == IB_MC_JSTATE_FULL) {
3118 ASSERT(omce->mc_fullreap);
3119 omce->mc_fullreap = B_FALSE;
3120 return (omce);
3121 } else {
3122 ASSERT(omce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON);
3123 }
3124 }
3125
3126 /*
3127 * Allocate the ibd_mce_t to track this JOIN.
3128 */
3129 mce = kmem_zalloc(sizeof (ibd_mce_t), KM_SLEEP);
3130 mce->mc_fullreap = B_FALSE;
3131 mce->mc_jstate = jstate;
3132
3133 if ((ibt_status = ibd_iba_join(state, mgid, mce)) != IBT_SUCCESS) {
3134 DPRINT(10, "ibd_join_group : failed ibt_join_mcg() %d",
3135 ibt_status);
3136 kmem_free(mce, sizeof (ibd_mce_t));
3137 return (NULL);
3138 }
3139
3140 /*
3141 * Is an IBA attach required? Not if the interface is already joined
3142 * to the mcg in a different appropriate join state.
3143 */
3144 if (jstate == IB_MC_JSTATE_NON) {
3145 tmce = IBD_MCACHE_FIND_FULL(state, mgid);
3146 if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL))
3147 do_attach = B_FALSE;
3148 } else if (jstate == IB_MC_JSTATE_FULL) {
3149 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
3150 do_attach = B_FALSE;
3151 } else { /* jstate == IB_MC_JSTATE_SEND_ONLY_NON */
3152 do_attach = B_FALSE;
3153 }
3154
3155 if (do_attach) {
3156 /*
3157 * Do the IBA attach.
3158 */
3159 DPRINT(10, "ibd_join_group: ibt_attach_mcg \n");
3160 if ((ibt_status = ibt_attach_mcg(state->id_chnl_hdl,
3161 &mce->mc_info)) != IBT_SUCCESS) {
3162 DPRINT(10, "ibd_join_group : failed qp attachment "
3163 "%d\n", ibt_status);
3164 /*
3165 * NOTE that we should probably preserve the join info
3166 * in the list and later try to leave again at detach
3167 * time.
3168 */
3169 (void) ibt_leave_mcg(state->id_sgid, mgid,
3170 state->id_sgid, jstate);
3171 kmem_free(mce, sizeof (ibd_mce_t));
3172 return (NULL);
3173 }
3174 }
3175
3176 /*
3177 * Insert the ibd_mce_t in the proper list.
3178 */
3179 if (jstate == IB_MC_JSTATE_NON) {
3180 IBD_MCACHE_INSERT_NON(state, mce);
3181 } else {
3182 /*
3183 * Set up the mc_req fields used for reaping the
3184 * mcg in case of delayed tx completion (see
3185 * ibd_tx_cleanup()). Also done for sendonly join in
3186 * case we are promoted to fullmembership later and
3187 * keep using the same mce.
3188 */
3189 mce->mc_req.rq_gid = mgid;
3190 mce->mc_req.rq_ptr = mce;
3191 /*
3192 * Check whether this is the case of trying to join
3193 * full member, and we were already joined send only.
3194 * We try to drop our SendOnly membership, but it is
3195 * possible that the mcg does not exist anymore (and
3196 * the subnet trap never reached us), so the leave
3197 * operation might fail.
3198 */
3199 if (omce != NULL) {
3200 (void) ibt_leave_mcg(state->id_sgid, mgid,
3201 state->id_sgid, IB_MC_JSTATE_SEND_ONLY_NON);
3202 omce->mc_jstate = IB_MC_JSTATE_FULL;
3203 bcopy(&mce->mc_info, &omce->mc_info,
3204 sizeof (ibt_mcg_info_t));
3205 kmem_free(mce, sizeof (ibd_mce_t));
3206 return (omce);
3207 }
3208 mutex_enter(&state->id_mc_mutex);
3209 IBD_MCACHE_INSERT_FULL(state, mce);
3210 mutex_exit(&state->id_mc_mutex);
3211 }
3212
3213 return (mce);
3214 }
3215
3216 /*
3217 * Called during port up event handling to attempt to reacquire full
3218 * membership to an mcg. Stripped down version of ibd_join_group().
3219 * Note that it is possible that the mcg might have gone away, and
3220 * gets recreated at this point.
3221 */
3222 static void
3223 ibd_reacquire_group(ibd_state_t *state, ibd_mce_t *mce)
3224 {
3225 ib_gid_t mgid;
3226
3227 /*
3228 * If the mc_fullreap flag is set, or this join fails, a subsequent
3229 * reap/leave is going to try to leave the group. We could prevent
3230 * that by adding a boolean flag into ibd_mce_t, if required.
3231 */
3232 if (mce->mc_fullreap)
3233 return;
3234
3235 mgid = mce->mc_info.mc_adds_vect.av_dgid;
3236
3237 DPRINT(2, "ibd_reacquire_group : %016llx:%016llx\n", mgid.gid_prefix,
3238 mgid.gid_guid);
3239
3240 /* While reacquiring, leave and then join the MCG */
3241 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid,
3242 mce->mc_jstate);
3243 if (ibd_iba_join(state, mgid, mce) != IBT_SUCCESS)
3244 ibd_print_warn(state, "Failure on port up to rejoin "
3245 "multicast gid %016llx:%016llx",
3246 (u_longlong_t)mgid.gid_prefix,
3247 (u_longlong_t)mgid.gid_guid);
3248 }
3249
3250 /*
3251 * This code handles delayed Tx completion cleanups for mcg's to which
3252 * disable_multicast has been issued, regular mcg related cleanups during
3253 * disable_multicast, disable_promiscuous and mcg traps, as well as
3254 * cleanups during driver detach time. Depending on the join state,
3255 * it deletes the mce from the appropriate list and issues the IBA
3256 * leave/detach; except in the disable_multicast case when the mce
3257 * is left on the active list for a subsequent Tx completion cleanup.
3258 */
3259 static void
3260 ibd_async_reap_group(ibd_state_t *state, ibd_mce_t *mce, ib_gid_t mgid,
3261 uint8_t jstate)
3262 {
3263 ibd_mce_t *tmce;
3264 boolean_t do_detach = B_TRUE;
3265
3266 /*
3267 * Before detaching, we must check whether the other list
3268 * contains the mcg; if we detach blindly, the consumer
3269 * who set up the other list will also stop receiving
3270 * traffic.
3271 */
3272 if (jstate == IB_MC_JSTATE_FULL) {
3273 /*
3274 * The following check is only relevant while coming
3275 * from the Tx completion path in the reap case.
3276 */
3277 if (!mce->mc_fullreap)
3278 return;
3279 mutex_enter(&state->id_mc_mutex);
3280 IBD_MCACHE_PULLOUT_FULL(state, mce);
3281 mutex_exit(&state->id_mc_mutex);
3282 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
3283 do_detach = B_FALSE;
3284 } else if (jstate == IB_MC_JSTATE_NON) {
3285 IBD_MCACHE_PULLOUT_NON(state, mce);
3286 tmce = IBD_MCACHE_FIND_FULL(state, mgid);
3287 if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL))
3288 do_detach = B_FALSE;
3289 } else { /* jstate == IB_MC_JSTATE_SEND_ONLY_NON */
3290 mutex_enter(&state->id_mc_mutex);
3291 IBD_MCACHE_PULLOUT_FULL(state, mce);
3292 mutex_exit(&state->id_mc_mutex);
3293 do_detach = B_FALSE;
3294 }
3295
3296 /*
3297 * If we are reacting to a mcg trap and leaving our sendonly or
3298 * non membership, the mcg is possibly already gone, so attempting
3299 * to leave might fail. On the other hand, we must try to leave
3300 * anyway, since this might be a trap from long ago, and we could
3301 * have potentially sendonly joined to a recent incarnation of
3302 * the mcg and are about to loose track of this information.
3303 */
3304 if (do_detach) {
3305 DPRINT(2, "ibd_async_reap_group : ibt_detach_mcg : "
3306 "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid);
3307 (void) ibt_detach_mcg(state->id_chnl_hdl, &mce->mc_info);
3308 }
3309
3310 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, jstate);
3311 kmem_free(mce, sizeof (ibd_mce_t));
3312 }
3313
3314 /*
3315 * Async code executed due to multicast and promiscuous disable requests
3316 * and mcg trap handling; also executed during driver detach. Mostly, a
3317 * leave and detach is done; except for the fullmember case when Tx
3318 * requests are pending, whence arrangements are made for subsequent
3319 * cleanup on Tx completion.
3320 */
3321 static void
3322 ibd_leave_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
3323 {
3324 ipoib_mac_t mcmac;
3325 boolean_t recycled;
3326 ibd_mce_t *mce;
3327
3328 DPRINT(2, "ibd_leave_group : leave_group state %d : %016llx:%016llx\n",
3329 jstate, mgid.gid_prefix, mgid.gid_guid);
3330
3331 if (jstate == IB_MC_JSTATE_NON) {
3332 recycled = B_TRUE;
3333 mce = IBD_MCACHE_FIND_NON(state, mgid);
3334 /*
3335 * In case we are handling a mcg trap, we might not find
3336 * the mcg in the non list.
3337 */
3338 if (mce == NULL) {
3339 return;
3340 }
3341 } else {
3342 mce = IBD_MCACHE_FIND_FULL(state, mgid);
3343
3344 /*
3345 * In case we are handling a mcg trap, make sure the trap
3346 * is not arriving late; if we have an mce that indicates
3347 * that we are already a fullmember, that would be a clear
3348 * indication that the trap arrived late (ie, is for a
3349 * previous incarnation of the mcg).
3350 */
3351 if (jstate == IB_MC_JSTATE_SEND_ONLY_NON) {
3352 if ((mce == NULL) || (mce->mc_jstate ==
3353 IB_MC_JSTATE_FULL)) {
3354 return;
3355 }
3356 } else {
3357 ASSERT(jstate == IB_MC_JSTATE_FULL);
3358
3359 /*
3360 * If join group failed, mce will be NULL here.
3361 * This is because in GLDv3 driver, set multicast
3362 * will always return success.
3363 */
3364 if (mce == NULL) {
3365 return;
3366 }
3367
3368 mce->mc_fullreap = B_TRUE;
3369 }
3370
3371 /*
3372 * If no pending Tx's remain that reference the AH
3373 * for the mcg, recycle it from active to free list.
3374 * Else in the IB_MC_JSTATE_FULL case, just mark the AH,
3375 * so the last completing Tx will cause an async reap
3376 * operation to be invoked, at which time we will drop our
3377 * membership to the mcg so that the pending Tx's complete
3378 * successfully. Refer to comments on "AH and MCE active
3379 * list manipulation" at top of this file. The lock protects
3380 * against Tx fast path and Tx cleanup code.
3381 */
3382 mutex_enter(&state->id_ac_mutex);
3383 ibd_h2n_mac(&mcmac, IB_MC_QPN, mgid.gid_prefix, mgid.gid_guid);
3384 recycled = ibd_acache_recycle(state, &mcmac, (jstate ==
3385 IB_MC_JSTATE_SEND_ONLY_NON));
3386 mutex_exit(&state->id_ac_mutex);
3387 }
3388
3389 if (recycled) {
3390 DPRINT(2, "ibd_leave_group : leave_group reaping : "
3391 "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid);
3392 ibd_async_reap_group(state, mce, mgid, jstate);
3393 }
3394 }
3395
3396 /*
3397 * Find the broadcast address as defined by IPoIB; implicitly
3398 * determines the IBA scope, mtu, tclass etc of the link the
3399 * interface is going to be a member of.
3400 */
3401 static ibt_status_t
3402 ibd_find_bgroup(ibd_state_t *state)
3403 {
3404 ibt_mcg_attr_t mcg_attr;
3405 uint_t numg;
3406 uchar_t scopes[] = { IB_MC_SCOPE_SUBNET_LOCAL,
3407 IB_MC_SCOPE_SITE_LOCAL, IB_MC_SCOPE_ORG_LOCAL,
3408 IB_MC_SCOPE_GLOBAL };
3409 int i, mcgmtu;
3410 boolean_t found = B_FALSE;
3411 int ret;
3412 ibt_mcg_info_t mcg_info;
3413
3414 state->id_bgroup_created = B_FALSE;
3415 state->id_bgroup_present = B_FALSE;
3416
3417 query_bcast_grp:
3418 bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
3419 mcg_attr.mc_pkey = state->id_pkey;
3420 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
3421 state->id_mgid.gid_guid = IB_MGID_IPV4_LOWGRP_MASK;
3422 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
3423
3424 for (i = 0; i < sizeof (scopes)/sizeof (scopes[0]); i++) {
3425 state->id_scope = mcg_attr.mc_scope = scopes[i];
3426
3427 /*
3428 * Look for the IPoIB broadcast group.
3429 */
3430 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
3431 state->id_mgid.gid_prefix =
3432 (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) |
3433 ((uint64_t)state->id_scope << 48) |
3434 ((uint32_t)(state->id_pkey << 16)));
3435 mcg_attr.mc_mgid = state->id_mgid;
3436 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
3437 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 1,
3438 &state->id_mcinfo, &numg) == IBT_SUCCESS) {
3439 found = B_TRUE;
3440 break;
3441 }
3442 }
3443
3444 if (!found) {
3445 if (state->id_create_broadcast_group) {
3446 /*
3447 * If we created the broadcast group, but failed to
3448 * find it, we can't do anything except leave the
3449 * one we created and return failure.
3450 */
3451 if (state->id_bgroup_created) {
3452 ibd_print_warn(state, "IPoIB broadcast group "
3453 "absent. Unable to query after create.");
3454 goto find_bgroup_fail;
3455 }
3456
3457 /*
3458 * Create the ipoib broadcast group if it didn't exist
3459 */
3460 bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
3461 mcg_attr.mc_qkey = IBD_DEFAULT_QKEY;
3462 mcg_attr.mc_join_state = IB_MC_JSTATE_FULL;
3463 mcg_attr.mc_scope = IB_MC_SCOPE_SUBNET_LOCAL;
3464 mcg_attr.mc_pkey = state->id_pkey;
3465 mcg_attr.mc_flow = 0;
3466 mcg_attr.mc_sl = 0;
3467 mcg_attr.mc_tclass = 0;
3468 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
3469 state->id_mgid.gid_prefix =
3470 (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) |
3471 ((uint64_t)IB_MC_SCOPE_SUBNET_LOCAL << 48) |
3472 ((uint32_t)(state->id_pkey << 16)));
3473 mcg_attr.mc_mgid = state->id_mgid;
3474 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
3475
3476 if ((ret = ibt_join_mcg(state->id_sgid, &mcg_attr,
3477 &mcg_info, NULL, NULL)) != IBT_SUCCESS) {
3478 ibd_print_warn(state, "IPoIB broadcast group "
3479 "absent, create failed: ret = %d\n", ret);
3480 state->id_bgroup_created = B_FALSE;
3481 return (IBT_FAILURE);
3482 }
3483 state->id_bgroup_created = B_TRUE;
3484 goto query_bcast_grp;
3485 } else {
3486 ibd_print_warn(state, "IPoIB broadcast group absent");
3487 return (IBT_FAILURE);
3488 }
3489 }
3490
3491 /*
3492 * Assert that the mcg mtu <= id_mtu. Fill in updated id_mtu.
3493 */
3494 mcgmtu = (128 << state->id_mcinfo->mc_mtu);
3495 if (state->id_mtu < mcgmtu) {
3496 ibd_print_warn(state, "IPoIB broadcast group MTU %d "
3497 "greater than port's maximum MTU %d", mcgmtu,
3498 state->id_mtu);
3499 ibt_free_mcg_info(state->id_mcinfo, 1);
3500 goto find_bgroup_fail;
3501 }
3502 state->id_mtu = mcgmtu;
3503 state->id_bgroup_present = B_TRUE;
3504
3505 return (IBT_SUCCESS);
3506
3507 find_bgroup_fail:
3508 if (state->id_bgroup_created) {
3509 (void) ibt_leave_mcg(state->id_sgid,
3510 mcg_info.mc_adds_vect.av_dgid, state->id_sgid,
3511 IB_MC_JSTATE_FULL);
3512 }
3513
3514 return (IBT_FAILURE);
3515 }
3516
3517 static int
3518 ibd_alloc_tx_copybufs(ibd_state_t *state)
3519 {
3520 ibt_mr_attr_t mem_attr;
3521
3522 /*
3523 * Allocate one big chunk for all regular tx copy bufs
3524 */
3525 state->id_tx_buf_sz = state->id_mtu;
3526 if (state->id_lso_policy && state->id_lso_capable &&
3527 (state->id_ud_tx_copy_thresh > state->id_mtu)) {
3528 state->id_tx_buf_sz = state->id_ud_tx_copy_thresh;
3529 }
3530
3531 state->id_tx_bufs = kmem_zalloc(state->id_ud_num_swqe *
3532 state->id_tx_buf_sz, KM_SLEEP);
3533
3534 state->id_tx_wqes = kmem_zalloc(state->id_ud_num_swqe *
3535 sizeof (ibd_swqe_t), KM_SLEEP);
3536
3537 /*
3538 * Do one memory registration on the entire txbuf area
3539 */
3540 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_tx_bufs;
3541 mem_attr.mr_len = state->id_ud_num_swqe * state->id_tx_buf_sz;
3542 mem_attr.mr_as = NULL;
3543 mem_attr.mr_flags = IBT_MR_SLEEP;
3544 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr,
3545 &state->id_tx_mr_hdl, &state->id_tx_mr_desc) != IBT_SUCCESS) {
3546 DPRINT(10, "ibd_alloc_tx_copybufs: ibt_register_mr failed");
3547 kmem_free(state->id_tx_wqes,
3548 state->id_ud_num_swqe * sizeof (ibd_swqe_t));
3549 kmem_free(state->id_tx_bufs,
3550 state->id_ud_num_swqe * state->id_tx_buf_sz);
3551 state->id_tx_bufs = NULL;
3552 return (DDI_FAILURE);
3553 }
3554
3555 return (DDI_SUCCESS);
3556 }
3557
3558 static int
3559 ibd_alloc_tx_lsobufs(ibd_state_t *state)
3560 {
3561 ibt_mr_attr_t mem_attr;
3562 ibd_lsobuf_t *buflist;
3563 ibd_lsobuf_t *lbufp;
3564 ibd_lsobuf_t *tail;
3565 ibd_lsobkt_t *bktp;
3566 uint8_t *membase;
3567 uint8_t *memp;
3568 uint_t memsz;
3569 int i;
3570
3571 /*
3572 * Allocate the lso bucket
3573 */
3574 bktp = kmem_zalloc(sizeof (ibd_lsobkt_t), KM_SLEEP);
3575
3576 /*
3577 * Allocate the entire lso memory and register it
3578 */
3579 memsz = state->id_num_lso_bufs * IBD_LSO_BUFSZ;
3580 membase = kmem_zalloc(memsz, KM_SLEEP);
3581
3582 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)membase;
3583 mem_attr.mr_len = memsz;
3584 mem_attr.mr_as = NULL;
3585 mem_attr.mr_flags = IBT_MR_SLEEP;
3586 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl,
3587 &mem_attr, &bktp->bkt_mr_hdl, &bktp->bkt_mr_desc) != IBT_SUCCESS) {
3588 DPRINT(10, "ibd_alloc_tx_lsobufs: ibt_register_mr failed");
3589 kmem_free(membase, memsz);
3590 kmem_free(bktp, sizeof (ibd_lsobkt_t));
3591 return (DDI_FAILURE);
3592 }
3593
3594 mutex_enter(&state->id_lso_lock);
3595
3596 /*
3597 * Now allocate the buflist. Note that the elements in the buflist and
3598 * the buffers in the lso memory have a permanent 1-1 relation, so we
3599 * can always derive the address of a buflist entry from the address of
3600 * an lso buffer.
3601 */
3602 buflist = kmem_zalloc(state->id_num_lso_bufs * sizeof (ibd_lsobuf_t),
3603 KM_SLEEP);
3604
3605 /*
3606 * Set up the lso buf chain
3607 */
3608 memp = membase;
3609 lbufp = buflist;
3610 for (i = 0; i < state->id_num_lso_bufs; i++) {
3611 lbufp->lb_isfree = 1;
3612 lbufp->lb_buf = memp;
3613 lbufp->lb_next = lbufp + 1;
3614
3615 tail = lbufp;
3616
3617 memp += IBD_LSO_BUFSZ;
3618 lbufp++;
3619 }
3620 tail->lb_next = NULL;
3621
3622 /*
3623 * Set up the LSO buffer information in ibd state
3624 */
3625 bktp->bkt_bufl = buflist;
3626 bktp->bkt_free_head = buflist;
3627 bktp->bkt_mem = membase;
3628 bktp->bkt_nelem = state->id_num_lso_bufs;
3629 bktp->bkt_nfree = bktp->bkt_nelem;
3630
3631 state->id_lso = bktp;
3632 mutex_exit(&state->id_lso_lock);
3633
3634 return (DDI_SUCCESS);
3635 }
3636
3637 /*
3638 * Statically allocate Tx buffer list(s).
3639 */
3640 static int
3641 ibd_init_txlist(ibd_state_t *state)
3642 {
3643 ibd_swqe_t *swqe;
3644 ibt_lkey_t lkey;
3645 int i;
3646 uint_t len;
3647 uint8_t *bufaddr;
3648
3649 if (ibd_alloc_tx_copybufs(state) != DDI_SUCCESS)
3650 return (DDI_FAILURE);
3651
3652 if (state->id_lso_policy && state->id_lso_capable) {
3653 if (ibd_alloc_tx_lsobufs(state) != DDI_SUCCESS)
3654 state->id_lso_capable = B_FALSE;
3655 }
3656
3657 mutex_enter(&state->id_tx_list.dl_mutex);
3658 state->id_tx_list.dl_head = NULL;
3659 state->id_tx_list.dl_pending_sends = B_FALSE;
3660 state->id_tx_list.dl_cnt = 0;
3661 mutex_exit(&state->id_tx_list.dl_mutex);
3662 mutex_enter(&state->id_tx_rel_list.dl_mutex);
3663 state->id_tx_rel_list.dl_head = NULL;
3664 state->id_tx_rel_list.dl_pending_sends = B_FALSE;
3665 state->id_tx_rel_list.dl_cnt = 0;
3666 mutex_exit(&state->id_tx_rel_list.dl_mutex);
3667
3668 /*
3669 * Allocate and setup the swqe list
3670 */
3671 lkey = state->id_tx_mr_desc.md_lkey;
3672 bufaddr = state->id_tx_bufs;
3673 len = state->id_tx_buf_sz;
3674 swqe = state->id_tx_wqes;
3675 mutex_enter(&state->id_tx_list.dl_mutex);
3676 for (i = 0; i < state->id_ud_num_swqe; i++, swqe++, bufaddr += len) {
3677 swqe->swqe_next = NULL;
3678 swqe->swqe_im_mblk = NULL;
3679
3680 swqe->swqe_copybuf.ic_sgl.ds_va = (ib_vaddr_t)(uintptr_t)
3681 bufaddr;
3682 swqe->swqe_copybuf.ic_sgl.ds_key = lkey;
3683 swqe->swqe_copybuf.ic_sgl.ds_len = 0; /* set in send */
3684
3685 swqe->w_swr.wr_id = (ibt_wrid_t)(uintptr_t)swqe;
3686 swqe->w_swr.wr_flags = IBT_WR_NO_FLAGS;
3687 swqe->w_swr.wr_trans = IBT_UD_SRV;
3688
3689 /* These are set in send */
3690 swqe->w_swr.wr_nds = 0;
3691 swqe->w_swr.wr_sgl = NULL;
3692 swqe->w_swr.wr_opcode = IBT_WRC_SEND;
3693
3694 /* add to list */
3695 state->id_tx_list.dl_cnt++;
3696 swqe->swqe_next = state->id_tx_list.dl_head;
3697 state->id_tx_list.dl_head = SWQE_TO_WQE(swqe);
3698 }
3699 mutex_exit(&state->id_tx_list.dl_mutex);
3700
3701 return (DDI_SUCCESS);
3702 }
3703
3704 static int
3705 ibd_acquire_lsobufs(ibd_state_t *state, uint_t req_sz, ibt_wr_ds_t *sgl_p,
3706 uint32_t *nds_p)
3707 {
3708 ibd_lsobkt_t *bktp;
3709 ibd_lsobuf_t *lbufp;
3710 ibd_lsobuf_t *nextp;
3711 ibt_lkey_t lso_lkey;
3712 uint_t frag_sz;
3713 uint_t num_needed;
3714 int i;
3715
3716 ASSERT(sgl_p != NULL);
3717 ASSERT(nds_p != NULL);
3718 ASSERT(req_sz != 0);
3719
3720 /*
3721 * Determine how many bufs we'd need for the size requested
3722 */
3723 num_needed = req_sz / IBD_LSO_BUFSZ;
3724 if ((frag_sz = req_sz % IBD_LSO_BUFSZ) != 0)
3725 num_needed++;
3726
3727 mutex_enter(&state->id_lso_lock);
3728
3729 /*
3730 * If we don't have enough lso bufs, return failure
3731 */
3732 ASSERT(state->id_lso != NULL);
3733 bktp = state->id_lso;
3734 if (bktp->bkt_nfree < num_needed) {
3735 mutex_exit(&state->id_lso_lock);
3736 return (-1);
3737 }
3738
3739 /*
3740 * Pick the first 'num_needed' bufs from the free list
3741 */
3742 lso_lkey = bktp->bkt_mr_desc.md_lkey;
3743 lbufp = bktp->bkt_free_head;
3744 for (i = 0; i < num_needed; i++) {
3745 ASSERT(lbufp->lb_isfree != 0);
3746 ASSERT(lbufp->lb_buf != NULL);
3747
3748 nextp = lbufp->lb_next;
3749
3750 sgl_p[i].ds_va = (ib_vaddr_t)(uintptr_t)lbufp->lb_buf;
3751 sgl_p[i].ds_key = lso_lkey;
3752 sgl_p[i].ds_len = IBD_LSO_BUFSZ;
3753
3754 lbufp->lb_isfree = 0;
3755 lbufp->lb_next = NULL;
3756
3757 lbufp = nextp;
3758 }
3759 bktp->bkt_free_head = lbufp;
3760
3761 /*
3762 * If the requested size is not a multiple of IBD_LSO_BUFSZ, we need
3763 * to adjust the last sgl entry's length. Since we know we need atleast
3764 * one, the i-1 use below is ok.
3765 */
3766 if (frag_sz) {
3767 sgl_p[i-1].ds_len = frag_sz;
3768 }
3769
3770 /*
3771 * Update nfree count and return
3772 */
3773 bktp->bkt_nfree -= num_needed;
3774
3775 mutex_exit(&state->id_lso_lock);
3776
3777 *nds_p = num_needed;
3778
3779 return (0);
3780 }
3781
3782 static void
3783 ibd_release_lsobufs(ibd_state_t *state, ibt_wr_ds_t *sgl_p, uint32_t nds)
3784 {
3785 ibd_lsobkt_t *bktp;
3786 ibd_lsobuf_t *lbufp;
3787 uint8_t *lso_mem_end;
3788 uint_t ndx;
3789 int i;
3790
3791 mutex_enter(&state->id_lso_lock);
3792
3793 bktp = state->id_lso;
3794 ASSERT(bktp != NULL);
3795
3796 lso_mem_end = bktp->bkt_mem + bktp->bkt_nelem * IBD_LSO_BUFSZ;
3797 for (i = 0; i < nds; i++) {
3798 uint8_t *va;
3799
3800 va = (uint8_t *)(uintptr_t)sgl_p[i].ds_va;
3801 ASSERT(va >= bktp->bkt_mem && va < lso_mem_end);
3802
3803 /*
3804 * Figure out the buflist element this sgl buffer corresponds
3805 * to and put it back at the head
3806 */
3807 ndx = (va - bktp->bkt_mem) / IBD_LSO_BUFSZ;
3808 lbufp = bktp->bkt_bufl + ndx;
3809
3810 ASSERT(lbufp->lb_isfree == 0);
3811 ASSERT(lbufp->lb_buf == va);
3812
3813 lbufp->lb_isfree = 1;
3814 lbufp->lb_next = bktp->bkt_free_head;
3815 bktp->bkt_free_head = lbufp;
3816 }
3817 bktp->bkt_nfree += nds;
3818
3819 mutex_exit(&state->id_lso_lock);
3820 }
3821
3822 static void
3823 ibd_free_tx_copybufs(ibd_state_t *state)
3824 {
3825 /*
3826 * Unregister txbuf mr
3827 */
3828 if (ibt_deregister_mr(state->id_hca_hdl,
3829 state->id_tx_mr_hdl) != IBT_SUCCESS) {
3830 DPRINT(10, "ibd_free_tx_copybufs: ibt_deregister_mr failed");
3831 }
3832 state->id_tx_mr_hdl = NULL;
3833
3834 /*
3835 * Free txbuf memory
3836 */
3837 kmem_free(state->id_tx_wqes, state->id_ud_num_swqe *
3838 sizeof (ibd_swqe_t));
3839 kmem_free(state->id_tx_bufs, state->id_ud_num_swqe *
3840 state->id_tx_buf_sz);
3841 state->id_tx_wqes = NULL;
3842 state->id_tx_bufs = NULL;
3843 }
3844
3845 static void
3846 ibd_free_tx_lsobufs(ibd_state_t *state)
3847 {
3848 ibd_lsobkt_t *bktp;
3849
3850 mutex_enter(&state->id_lso_lock);
3851
3852 if ((bktp = state->id_lso) == NULL) {
3853 mutex_exit(&state->id_lso_lock);
3854 return;
3855 }
3856
3857 /*
3858 * First, free the buflist
3859 */
3860 ASSERT(bktp->bkt_bufl != NULL);
3861 kmem_free(bktp->bkt_bufl, bktp->bkt_nelem * sizeof (ibd_lsobuf_t));
3862
3863 /*
3864 * Unregister the LSO memory and free it
3865 */
3866 ASSERT(bktp->bkt_mr_hdl != NULL);
3867 if (ibt_deregister_mr(state->id_hca_hdl,
3868 bktp->bkt_mr_hdl) != IBT_SUCCESS) {
3869 DPRINT(10,
3870 "ibd_free_lsobufs: ibt_deregister_mr failed");
3871 }
3872 ASSERT(bktp->bkt_mem);
3873 kmem_free(bktp->bkt_mem, bktp->bkt_nelem * IBD_LSO_BUFSZ);
3874
3875 /*
3876 * Finally free the bucket
3877 */
3878 kmem_free(bktp, sizeof (ibd_lsobkt_t));
3879 state->id_lso = NULL;
3880
3881 mutex_exit(&state->id_lso_lock);
3882 }
3883
3884 /*
3885 * Free the statically allocated Tx buffer list.
3886 */
3887 static void
3888 ibd_fini_txlist(ibd_state_t *state)
3889 {
3890 /*
3891 * Free the allocated swqes
3892 */
3893 mutex_enter(&state->id_tx_list.dl_mutex);
3894 mutex_enter(&state->id_tx_rel_list.dl_mutex);
3895 state->id_tx_list.dl_head = NULL;
3896 state->id_tx_list.dl_pending_sends = B_FALSE;
3897 state->id_tx_list.dl_cnt = 0;
3898 state->id_tx_rel_list.dl_head = NULL;
3899 state->id_tx_rel_list.dl_pending_sends = B_FALSE;
3900 state->id_tx_rel_list.dl_cnt = 0;
3901 mutex_exit(&state->id_tx_rel_list.dl_mutex);
3902 mutex_exit(&state->id_tx_list.dl_mutex);
3903
3904 ibd_free_tx_lsobufs(state);
3905 ibd_free_tx_copybufs(state);
3906 }
3907
3908 /*
3909 * post a list of rwqes, NULL terminated.
3910 */
3911 static void
3912 ibd_post_recv_list(ibd_state_t *state, ibd_rwqe_t *rwqe)
3913 {
3914 uint_t i;
3915 uint_t num_posted;
3916 ibt_status_t ibt_status;
3917 ibt_recv_wr_t wrs[IBD_RX_POST_CNT];
3918
3919 while (rwqe) {
3920 /* Post up to IBD_RX_POST_CNT receive work requests */
3921 for (i = 0; i < IBD_RX_POST_CNT; i++) {
3922 wrs[i] = rwqe->w_rwr;
3923 rwqe = WQE_TO_RWQE(rwqe->rwqe_next);
3924 if (rwqe == NULL) {
3925 i++;
3926 break;
3927 }
3928 }
3929
3930 /*
3931 * If posting fails for some reason, we'll never receive
3932 * completion intimation, so we'll need to cleanup. But
3933 * we need to make sure we don't clean up nodes whose
3934 * wrs have been successfully posted. We assume that the
3935 * hca driver returns on the first failure to post and
3936 * therefore the first 'num_posted' entries don't need
3937 * cleanup here.
3938 */
3939 atomic_add_32(&state->id_rx_list.dl_cnt, i);
3940
3941 num_posted = 0;
3942 ibt_status = ibt_post_recv(state->id_chnl_hdl, wrs, i,
3943 &num_posted);
3944 if (ibt_status != IBT_SUCCESS) {
3945 /* This cannot happen unless the device has an error. */
3946 ibd_print_warn(state, "ibd_post_recv: FATAL: "
3947 "posting multiple wrs failed: "
3948 "requested=%d, done=%d, ret=%d",
3949 IBD_RX_POST_CNT, num_posted, ibt_status);
3950 atomic_add_32(&state->id_rx_list.dl_cnt,
3951 num_posted - i);
3952 }
3953 }
3954 }
3955
3956 /*
3957 * Grab a list of rwqes from the array of lists, and post the list.
3958 */
3959 static void
3960 ibd_post_recv_intr(ibd_state_t *state)
3961 {
3962 ibd_rx_queue_t *rxp;
3963 ibd_rwqe_t *list;
3964
3965 /* rotate through the rx_queue array, expecting an adequate number */
3966 state->id_rx_post_queue_index =
3967 (state->id_rx_post_queue_index + 1) &
3968 (state->id_rx_nqueues - 1);
3969
3970 rxp = state->id_rx_queues + state->id_rx_post_queue_index;
3971 mutex_enter(&rxp->rx_post_lock);
3972 list = WQE_TO_RWQE(rxp->rx_head);
3973 rxp->rx_head = NULL;
3974 rxp->rx_cnt = 0;
3975 mutex_exit(&rxp->rx_post_lock);
3976 ibd_post_recv_list(state, list);
3977 }
3978
3979 /* macro explained below */
3980 #define RX_QUEUE_HASH(rwqe) \
3981 (((uintptr_t)(rwqe) >> 8) & (state->id_rx_nqueues - 1))
3982
3983 /*
3984 * Add a rwqe to one of the the Rx lists. If the list is large enough
3985 * (exactly IBD_RX_POST_CNT), post the list to the hardware.
3986 *
3987 * Note: one of 2^N lists is chosen via a hash. This is done
3988 * because using one list is contentious. If the first list is busy
3989 * (mutex_tryenter fails), use a second list (just call mutex_enter).
3990 *
3991 * The number 8 in RX_QUEUE_HASH is a random choice that provides
3992 * even distribution of mapping rwqes to the 2^N queues.
3993 */
3994 static void
3995 ibd_post_recv(ibd_state_t *state, ibd_rwqe_t *rwqe)
3996 {
3997 ibd_rx_queue_t *rxp;
3998
3999 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe);
4000
4001 if (!mutex_tryenter(&rxp->rx_post_lock)) {
4002 /* Failed. Try a different queue ("ptr + 16" ensures that). */
4003 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe + 16);
4004 mutex_enter(&rxp->rx_post_lock);
4005 }
4006 rwqe->rwqe_next = rxp->rx_head;
4007 if (++rxp->rx_cnt >= IBD_RX_POST_CNT - 2) {
4008 uint_t active = atomic_inc_32_nv(&state->id_rx_post_active);
4009
4010 /* only call ibt_post_recv() every Nth time through here */
4011 if ((active & (state->id_rx_nqueues - 1)) == 0) {
4012 rxp->rx_head = NULL;
4013 rxp->rx_cnt = 0;
4014 mutex_exit(&rxp->rx_post_lock);
4015 ibd_post_recv_list(state, rwqe);
4016 return;
4017 }
4018 }
4019 rxp->rx_head = RWQE_TO_WQE(rwqe);
4020 mutex_exit(&rxp->rx_post_lock);
4021 }
4022
4023 static int
4024 ibd_alloc_rx_copybufs(ibd_state_t *state)
4025 {
4026 ibt_mr_attr_t mem_attr;
4027 int i;
4028
4029 /*
4030 * Allocate one big chunk for all regular rx copy bufs
4031 */
4032 state->id_rx_buf_sz = state->id_mtu + IPOIB_GRH_SIZE;
4033
4034 state->id_rx_bufs = kmem_zalloc(state->id_ud_num_rwqe *
4035 state->id_rx_buf_sz, KM_SLEEP);
4036
4037 state->id_rx_wqes = kmem_zalloc(state->id_ud_num_rwqe *
4038 sizeof (ibd_rwqe_t), KM_SLEEP);
4039
4040 state->id_rx_nqueues = 1 << IBD_LOG_RX_POST;
4041 state->id_rx_queues = kmem_zalloc(state->id_rx_nqueues *
4042 sizeof (ibd_rx_queue_t), KM_SLEEP);
4043 for (i = 0; i < state->id_rx_nqueues; i++) {
4044 ibd_rx_queue_t *rxp = state->id_rx_queues + i;
4045 mutex_init(&rxp->rx_post_lock, NULL, MUTEX_DRIVER, NULL);
4046 }
4047
4048 /*
4049 * Do one memory registration on the entire rxbuf area
4050 */
4051 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_rx_bufs;
4052 mem_attr.mr_len = state->id_ud_num_rwqe * state->id_rx_buf_sz;
4053 mem_attr.mr_as = NULL;
4054 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
4055 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr,
4056 &state->id_rx_mr_hdl, &state->id_rx_mr_desc) != IBT_SUCCESS) {
4057 DPRINT(10, "ibd_alloc_rx_copybufs: ibt_register_mr failed");
4058 kmem_free(state->id_rx_wqes,
4059 state->id_ud_num_rwqe * sizeof (ibd_rwqe_t));
4060 kmem_free(state->id_rx_bufs,
4061 state->id_ud_num_rwqe * state->id_rx_buf_sz);
4062 state->id_rx_bufs = NULL;
4063 state->id_rx_wqes = NULL;
4064 return (DDI_FAILURE);
4065 }
4066
4067 return (DDI_SUCCESS);
4068 }
4069
4070 /*
4071 * Allocate the statically allocated Rx buffer list.
4072 */
4073 static int
4074 ibd_init_rxlist(ibd_state_t *state)
4075 {
4076 ibd_rwqe_t *rwqe, *next;
4077 ibd_wqe_t *list;
4078 ibt_lkey_t lkey;
4079 int i;
4080 uint_t len;
4081 uint8_t *bufaddr;
4082
4083 mutex_enter(&state->id_rx_free_list.dl_mutex);
4084 if (state->id_rx_free_list.dl_head != NULL) {
4085 /* rx rsrcs were never freed. Just repost them */
4086 len = state->id_rx_buf_sz;
4087 list = state->id_rx_free_list.dl_head;
4088 state->id_rx_free_list.dl_head = NULL;
4089 state->id_rx_free_list.dl_cnt = 0;
4090 mutex_exit(&state->id_rx_free_list.dl_mutex);
4091 for (rwqe = WQE_TO_RWQE(list); rwqe != NULL;
4092 rwqe = WQE_TO_RWQE(rwqe->rwqe_next)) {
4093 if ((rwqe->rwqe_im_mblk = desballoc(
4094 rwqe->rwqe_copybuf.ic_bufaddr, len, 0,
4095 &rwqe->w_freemsg_cb)) == NULL) {
4096 /* allow freemsg_cb to free the rwqes */
4097 if (atomic_dec_32_nv(&state->id_running) != 0) {
4098 cmn_err(CE_WARN, "ibd_init_rxlist: "
4099 "id_running was not 1\n");
4100 }
4101 DPRINT(10, "ibd_init_rxlist : "
4102 "failed in desballoc()");
4103 for (rwqe = WQE_TO_RWQE(list); rwqe != NULL;
4104 rwqe = next) {
4105 next = WQE_TO_RWQE(rwqe->rwqe_next);
4106 if (rwqe->rwqe_im_mblk) {
4107 atomic_inc_32(&state->
4108 id_rx_list.
4109 dl_bufs_outstanding);
4110 freemsg(rwqe->rwqe_im_mblk);
4111 } else
4112 ibd_free_rwqe(state, rwqe);
4113 }
4114 atomic_inc_32(&state->id_running);
4115 return (DDI_FAILURE);
4116 }
4117 }
4118 ibd_post_recv_list(state, WQE_TO_RWQE(list));
4119 return (DDI_SUCCESS);
4120 }
4121 mutex_exit(&state->id_rx_free_list.dl_mutex);
4122
4123 if (ibd_alloc_rx_copybufs(state) != DDI_SUCCESS)
4124 return (DDI_FAILURE);
4125
4126 /*
4127 * Allocate and setup the rwqe list
4128 */
4129 len = state->id_rx_buf_sz;
4130 lkey = state->id_rx_mr_desc.md_lkey;
4131 rwqe = state->id_rx_wqes;
4132 bufaddr = state->id_rx_bufs;
4133 list = NULL;
4134 for (i = 0; i < state->id_ud_num_rwqe; i++, rwqe++, bufaddr += len) {
4135 rwqe->w_state = state;
4136 rwqe->w_freemsg_cb.free_func = ibd_freemsg_cb;
4137 rwqe->w_freemsg_cb.free_arg = (char *)rwqe;
4138
4139 rwqe->rwqe_copybuf.ic_bufaddr = bufaddr;
4140
4141 if ((rwqe->rwqe_im_mblk = desballoc(bufaddr, len, 0,
4142 &rwqe->w_freemsg_cb)) == NULL) {
4143 DPRINT(10, "ibd_init_rxlist : failed in desballoc()");
4144 /* allow freemsg_cb to free the rwqes */
4145 if (atomic_dec_32_nv(&state->id_running) != 0) {
4146 cmn_err(CE_WARN, "ibd_init_rxlist: "
4147 "id_running was not 1\n");
4148 }
4149 DPRINT(10, "ibd_init_rxlist : "
4150 "failed in desballoc()");
4151 for (rwqe = WQE_TO_RWQE(list); rwqe != NULL;
4152 rwqe = next) {
4153 next = WQE_TO_RWQE(rwqe->rwqe_next);
4154 freemsg(rwqe->rwqe_im_mblk);
4155 }
4156 atomic_inc_32(&state->id_running);
4157
4158 /* remove reference to free'd rwqes */
4159 mutex_enter(&state->id_rx_free_list.dl_mutex);
4160 state->id_rx_free_list.dl_head = NULL;
4161 state->id_rx_free_list.dl_cnt = 0;
4162 mutex_exit(&state->id_rx_free_list.dl_mutex);
4163
4164 ibd_fini_rxlist(state);
4165 return (DDI_FAILURE);
4166 }
4167
4168 rwqe->rwqe_copybuf.ic_sgl.ds_key = lkey;
4169 rwqe->rwqe_copybuf.ic_sgl.ds_va =
4170 (ib_vaddr_t)(uintptr_t)bufaddr;
4171 rwqe->rwqe_copybuf.ic_sgl.ds_len = len;
4172 rwqe->w_rwr.wr_id = (ibt_wrid_t)(uintptr_t)rwqe;
4173 rwqe->w_rwr.wr_nds = 1;
4174 rwqe->w_rwr.wr_sgl = &rwqe->rwqe_copybuf.ic_sgl;
4175
4176 rwqe->rwqe_next = list;
4177 list = RWQE_TO_WQE(rwqe);
4178 }
4179 ibd_post_recv_list(state, WQE_TO_RWQE(list));
4180
4181 return (DDI_SUCCESS);
4182 }
4183
4184 static void
4185 ibd_free_rx_copybufs(ibd_state_t *state)
4186 {
4187 int i;
4188
4189 /*
4190 * Unregister rxbuf mr
4191 */
4192 if (ibt_deregister_mr(state->id_hca_hdl,
4193 state->id_rx_mr_hdl) != IBT_SUCCESS) {
4194 DPRINT(10, "ibd_free_rx_copybufs: ibt_deregister_mr failed");
4195 }
4196 state->id_rx_mr_hdl = NULL;
4197
4198 /*
4199 * Free rxbuf memory
4200 */
4201 for (i = 0; i < state->id_rx_nqueues; i++) {
4202 ibd_rx_queue_t *rxp = state->id_rx_queues + i;
4203 mutex_destroy(&rxp->rx_post_lock);
4204 }
4205 kmem_free(state->id_rx_queues, state->id_rx_nqueues *
4206 sizeof (ibd_rx_queue_t));
4207 kmem_free(state->id_rx_wqes, state->id_ud_num_rwqe *
4208 sizeof (ibd_rwqe_t));
4209 kmem_free(state->id_rx_bufs, state->id_ud_num_rwqe *
4210 state->id_rx_buf_sz);
4211 state->id_rx_queues = NULL;
4212 state->id_rx_wqes = NULL;
4213 state->id_rx_bufs = NULL;
4214 }
4215
4216 static void
4217 ibd_free_rx_rsrcs(ibd_state_t *state)
4218 {
4219 mutex_enter(&state->id_rx_free_list.dl_mutex);
4220 if (state->id_rx_free_list.dl_head == NULL) {
4221 /* already freed */
4222 mutex_exit(&state->id_rx_free_list.dl_mutex);
4223 return;
4224 }
4225 ASSERT(state->id_rx_free_list.dl_cnt == state->id_ud_num_rwqe);
4226 ibd_free_rx_copybufs(state);
4227 state->id_rx_free_list.dl_cnt = 0;
4228 state->id_rx_free_list.dl_head = NULL;
4229 mutex_exit(&state->id_rx_free_list.dl_mutex);
4230 }
4231
4232 /*
4233 * Free the statically allocated Rx buffer list.
4234 */
4235 static void
4236 ibd_fini_rxlist(ibd_state_t *state)
4237 {
4238 ibd_rwqe_t *rwqe;
4239 int i;
4240
4241 /* run through the rx_queue's, calling freemsg() */
4242 for (i = 0; i < state->id_rx_nqueues; i++) {
4243 ibd_rx_queue_t *rxp = state->id_rx_queues + i;
4244 mutex_enter(&rxp->rx_post_lock);
4245 for (rwqe = WQE_TO_RWQE(rxp->rx_head); rwqe;
4246 rwqe = WQE_TO_RWQE(rwqe->rwqe_next)) {
4247 freemsg(rwqe->rwqe_im_mblk);
4248 rxp->rx_cnt--;
4249 }
4250 rxp->rx_head = NULL;
4251 mutex_exit(&rxp->rx_post_lock);
4252 }
4253
4254 /* cannot free rx resources unless gld returned everything */
4255 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) == 0)
4256 ibd_free_rx_rsrcs(state);
4257 }
4258
4259 /*
4260 * Free an allocated recv wqe.
4261 */
4262 /* ARGSUSED */
4263 static void
4264 ibd_free_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe)
4265 {
4266 /*
4267 * desballoc() failed (no memory).
4268 *
4269 * This rwqe is placed on a free list so that it
4270 * can be reinstated when memory is available.
4271 *
4272 * NOTE: no code currently exists to reinstate
4273 * these "lost" rwqes.
4274 */
4275 mutex_enter(&state->id_rx_free_list.dl_mutex);
4276 state->id_rx_free_list.dl_cnt++;
4277 rwqe->rwqe_next = state->id_rx_free_list.dl_head;
4278 state->id_rx_free_list.dl_head = RWQE_TO_WQE(rwqe);
4279 mutex_exit(&state->id_rx_free_list.dl_mutex);
4280 }
4281
4282 /*
4283 * IBA Rx completion queue handler. Guaranteed to be single
4284 * threaded and nonreentrant for this CQ.
4285 */
4286 /* ARGSUSED */
4287 static void
4288 ibd_rcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
4289 {
4290 ibd_state_t *state = (ibd_state_t *)arg;
4291
4292 atomic_inc_64(&state->id_num_intrs);
4293
4294 if (ibd_rx_softintr == 1) {
4295 mutex_enter(&state->id_rcq_poll_lock);
4296 if (state->id_rcq_poll_busy & IBD_CQ_POLLING) {
4297 state->id_rcq_poll_busy |= IBD_REDO_CQ_POLLING;
4298 mutex_exit(&state->id_rcq_poll_lock);
4299 return;
4300 } else {
4301 mutex_exit(&state->id_rcq_poll_lock);
4302 ddi_trigger_softintr(state->id_rx);
4303 }
4304 } else
4305 (void) ibd_intr((caddr_t)state);
4306 }
4307
4308 /*
4309 * CQ handler for Tx completions, when the Tx CQ is in
4310 * interrupt driven mode.
4311 */
4312 /* ARGSUSED */
4313 static void
4314 ibd_scq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
4315 {
4316 ibd_state_t *state = (ibd_state_t *)arg;
4317
4318 atomic_inc_64(&state->id_num_intrs);
4319
4320 if (ibd_tx_softintr == 1) {
4321 mutex_enter(&state->id_scq_poll_lock);
4322 if (state->id_scq_poll_busy & IBD_CQ_POLLING) {
4323 state->id_scq_poll_busy |= IBD_REDO_CQ_POLLING;
4324 mutex_exit(&state->id_scq_poll_lock);
4325 return;
4326 } else {
4327 mutex_exit(&state->id_scq_poll_lock);
4328 ddi_trigger_softintr(state->id_tx);
4329 }
4330 } else
4331 (void) ibd_tx_recycle((caddr_t)state);
4332 }
4333
4334 /*
4335 * Multicast group create/delete trap handler. These will be delivered
4336 * on a kernel thread (handling can thus block) and can be invoked
4337 * concurrently. The handler can be invoked anytime after it is
4338 * registered and before ibt_detach().
4339 */
4340 /* ARGSUSED */
4341 static void
4342 ibd_snet_notices_handler(void *arg, ib_gid_t gid, ibt_subnet_event_code_t code,
4343 ibt_subnet_event_t *event)
4344 {
4345 ibd_state_t *state = (ibd_state_t *)arg;
4346 ibd_req_t *req;
4347
4348 /*
4349 * The trap handler will get invoked once for every event for
4350 * every port. The input "gid" is the GID0 of the port the
4351 * trap came in on; we just need to act on traps that came
4352 * to our port, meaning the port on which the ipoib interface
4353 * resides. Since ipoib uses GID0 of the port, we just match
4354 * the gids to check whether we need to handle the trap.
4355 */
4356 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
4357 if (bcmp(&gid, &state->id_sgid, sizeof (ib_gid_t)) != 0)
4358 return;
4359 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
4360
4361 DPRINT(10, "ibd_notices_handler : %d\n", code);
4362
4363 switch (code) {
4364 case IBT_SM_EVENT_UNAVAILABLE:
4365 /*
4366 * If we are in promiscuous mode or have
4367 * sendnonmembers, we need to print a warning
4368 * message right now. Else, just store the
4369 * information, print when we enter promiscuous
4370 * mode or attempt nonmember send. We might
4371 * also want to stop caching sendnonmember.
4372 */
4373 ibd_print_warn(state, "IBA multicast support "
4374 "degraded due to unavailability of multicast "
4375 "traps");
4376 break;
4377 case IBT_SM_EVENT_AVAILABLE:
4378 /*
4379 * If we printed a warning message above or
4380 * while trying to nonmember send or get into
4381 * promiscuous mode, print an okay message.
4382 */
4383 ibd_print_warn(state, "IBA multicast support "
4384 "restored due to availability of multicast "
4385 "traps");
4386 break;
4387 case IBT_SM_EVENT_MCG_CREATED:
4388 case IBT_SM_EVENT_MCG_DELETED:
4389 /*
4390 * If it is a "deleted" event and we are in late hca
4391 * init, nothing to do.
4392 */
4393 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ==
4394 IBD_DRV_IN_LATE_HCA_INIT) && (code ==
4395 IBT_SM_EVENT_MCG_DELETED)) {
4396 break;
4397 }
4398 /*
4399 * Common processing of creation/deletion traps.
4400 * First check if the instance is being
4401 * [de]initialized; back off then, without doing
4402 * anything more, since we are not sure if the
4403 * async thread is around, or whether we might
4404 * be racing with the detach code in ibd_m_stop()
4405 * that scans the mcg list.
4406 */
4407 if (!ibd_async_safe(state))
4408 return;
4409
4410 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
4411 req->rq_gid = event->sm_notice_gid;
4412 req->rq_ptr = (void *)code;
4413 ibd_queue_work_slot(state, req, IBD_ASYNC_TRAP);
4414 break;
4415 }
4416 }
4417
4418 static void
4419 ibd_async_trap(ibd_state_t *state, ibd_req_t *req)
4420 {
4421 ib_gid_t mgid = req->rq_gid;
4422 ibt_subnet_event_code_t code = (ibt_subnet_event_code_t)req->rq_ptr;
4423 int ret;
4424 ib_pkey_t pkey = (mgid.gid_prefix >> 16) & 0xffff;
4425
4426 DPRINT(10, "ibd_async_trap : %d\n", code);
4427
4428 /*
4429 * Check if we have already joined the IPoIB broadcast group for our
4430 * PKEY. If joined, perform the rest of the operation.
4431 * Else, the interface is not initialised. Do the initialisation here
4432 * by calling ibd_start() and return.
4433 */
4434
4435 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ==
4436 IBD_DRV_IN_LATE_HCA_INIT) && (state->id_bgroup_present == 0) &&
4437 (code == IBT_SM_EVENT_MCG_CREATED)) {
4438 /*
4439 * If we are in late HCA init and a notification for the
4440 * creation of a MCG came in, check if it is the IPoIB MCG for
4441 * this pkey. If not, return.
4442 */
4443 if ((mgid.gid_guid != IB_MGID_IPV4_LOWGRP_MASK) || (pkey !=
4444 state->id_pkey)) {
4445 ibd_async_done(state);
4446 return;
4447 }
4448 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
4449 /*
4450 * Check if there is still a necessity to start the interface.
4451 * It is possible that the user attempted unplumb at just about
4452 * the same time, and if unplumb succeeded, we have nothing to
4453 * do.
4454 */
4455 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ==
4456 IBD_DRV_IN_LATE_HCA_INIT) &&
4457 ((ret = ibd_start(state)) != 0)) {
4458 DPRINT(10, "ibd_async_trap: cannot start from late HCA "
4459 "init, ret=%d", ret);
4460 }
4461 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
4462 ibd_async_done(state);
4463 return;
4464 }
4465
4466 /*
4467 * Atomically search the nonmember and sendonlymember lists and
4468 * delete.
4469 */
4470 ibd_leave_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON);
4471
4472 if (state->id_prom_op == IBD_OP_COMPLETED) {
4473 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON);
4474
4475 /*
4476 * If in promiscuous mode, try to join/attach to the new
4477 * mcg. Given the unreliable out-of-order mode of trap
4478 * delivery, we can never be sure whether it is a problem
4479 * if the join fails. Thus, we warn the admin of a failure
4480 * if this was a creation trap. Note that the trap might
4481 * actually be reporting a long past event, and the mcg
4482 * might already have been deleted, thus we might be warning
4483 * in vain.
4484 */
4485 if ((ibd_join_group(state, mgid, IB_MC_JSTATE_NON) ==
4486 NULL) && (code == IBT_SM_EVENT_MCG_CREATED))
4487 ibd_print_warn(state, "IBA promiscuous mode missed "
4488 "new multicast gid %016llx:%016llx",
4489 (u_longlong_t)mgid.gid_prefix,
4490 (u_longlong_t)mgid.gid_guid);
4491 }
4492
4493 /*
4494 * Free the request slot allocated by the subnet event thread.
4495 */
4496 ibd_async_done(state);
4497 }
4498
4499 /*
4500 * GLDv3 entry point to get capabilities.
4501 */
4502 static boolean_t
4503 ibd_m_getcapab(void *arg, mac_capab_t cap, void *cap_data)
4504 {
4505 ibd_state_t *state = arg;
4506
4507 if (state->id_type == IBD_PORT_DRIVER)
4508 return (B_FALSE);
4509
4510 switch (cap) {
4511 case MAC_CAPAB_HCKSUM: {
4512 uint32_t *txflags = cap_data;
4513
4514 /*
4515 * We either do full checksum or not do it at all
4516 */
4517 if (state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL)
4518 *txflags = HCK_FULLCKSUM | HCKSUM_INET_FULL_V4;
4519 else
4520 return (B_FALSE);
4521 break;
4522 }
4523
4524 case MAC_CAPAB_LSO: {
4525 mac_capab_lso_t *cap_lso = cap_data;
4526
4527 /*
4528 * In addition to the capability and policy, since LSO
4529 * relies on hw checksum, we'll not enable LSO if we
4530 * don't have hw checksum. Of course, if the HCA doesn't
4531 * provide the reserved lkey capability, enabling LSO will
4532 * actually affect performance adversely, so we'll disable
4533 * LSO even for that case.
4534 */
4535 if (!state->id_lso_policy || !state->id_lso_capable)
4536 return (B_FALSE);
4537
4538 if ((state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) == 0)
4539 return (B_FALSE);
4540
4541 if (state->id_hca_res_lkey_capab == 0) {
4542 ibd_print_warn(state, "no reserved-lkey capability, "
4543 "disabling LSO");
4544 return (B_FALSE);
4545 }
4546
4547 cap_lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
4548 cap_lso->lso_basic_tcp_ipv4.lso_max = state->id_lso_maxlen - 1;
4549 break;
4550 }
4551
4552 default:
4553 return (B_FALSE);
4554 }
4555
4556 return (B_TRUE);
4557 }
4558
4559 /*
4560 * callback function for set/get of properties
4561 */
4562 static int
4563 ibd_m_setprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
4564 uint_t pr_valsize, const void *pr_val)
4565 {
4566 ibd_state_t *state = arg;
4567 int err = 0;
4568 uint32_t link_mode;
4569
4570 /* Cannot set properties on a port driver */
4571 if (state->id_type == IBD_PORT_DRIVER) {
4572 return (ENOTSUP);
4573 }
4574
4575 switch (pr_num) {
4576 case MAC_PROP_IB_LINKMODE:
4577 if (state->id_mac_state & IBD_DRV_STARTED) {
4578 err = EBUSY;
4579 break;
4580 }
4581 if (pr_val == NULL) {
4582 err = EINVAL;
4583 break;
4584 }
4585 bcopy(pr_val, &link_mode, sizeof (link_mode));
4586 if (link_mode != IBD_LINK_MODE_UD &&
4587 link_mode != IBD_LINK_MODE_RC) {
4588 err = EINVAL;
4589 } else {
4590 if (link_mode == IBD_LINK_MODE_RC) {
4591 if (state->id_enable_rc) {
4592 return (0);
4593 }
4594 state->id_enable_rc = 1;
4595 /* inform MAC framework of new MTU */
4596 err = mac_maxsdu_update2(state->id_mh,
4597 state->rc_mtu - IPOIB_HDRSIZE,
4598 state->id_mtu - IPOIB_HDRSIZE);
4599 } else {
4600 if (!state->id_enable_rc) {
4601 return (0);
4602 }
4603 state->id_enable_rc = 0;
4604 err = mac_maxsdu_update2(state->id_mh,
4605 state->id_mtu - IPOIB_HDRSIZE,
4606 state->id_mtu - IPOIB_HDRSIZE);
4607 }
4608 (void) ibd_record_capab(state);
4609 mac_capab_update(state->id_mh);
4610 }
4611 break;
4612 case MAC_PROP_PRIVATE:
4613 err = ibd_set_priv_prop(state, pr_name,
4614 pr_valsize, pr_val);
4615 break;
4616 default:
4617 err = ENOTSUP;
4618 break;
4619 }
4620 return (err);
4621 }
4622
4623 static int
4624 ibd_m_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
4625 uint_t pr_valsize, void *pr_val)
4626 {
4627 ibd_state_t *state = arg;
4628 int err = 0;
4629
4630 switch (pr_num) {
4631 case MAC_PROP_MTU:
4632 break;
4633 default:
4634 if (state->id_type == IBD_PORT_DRIVER) {
4635 return (ENOTSUP);
4636 }
4637 break;
4638 }
4639
4640 switch (pr_num) {
4641 case MAC_PROP_IB_LINKMODE:
4642 *(uint_t *)pr_val = state->id_enable_rc;
4643 break;
4644 case MAC_PROP_PRIVATE:
4645 err = ibd_get_priv_prop(state, pr_name, pr_valsize,
4646 pr_val);
4647 break;
4648 default:
4649 err = ENOTSUP;
4650 break;
4651 }
4652 return (err);
4653 }
4654
4655 static void
4656 ibd_m_propinfo(void *arg, const char *pr_name, mac_prop_id_t pr_num,
4657 mac_prop_info_handle_t prh)
4658 {
4659 ibd_state_t *state = arg;
4660
4661 switch (pr_num) {
4662 case MAC_PROP_IB_LINKMODE: {
4663 mac_prop_info_set_default_uint32(prh, IBD_DEF_LINK_MODE);
4664 break;
4665 }
4666 case MAC_PROP_MTU: {
4667 uint32_t min, max;
4668 if (state->id_type == IBD_PORT_DRIVER) {
4669 min = 1500;
4670 max = IBD_DEF_RC_MAX_SDU;
4671 } else if (state->id_enable_rc) {
4672 min = max = IBD_DEF_RC_MAX_SDU;
4673 } else {
4674 min = max = state->id_mtu - IPOIB_HDRSIZE;
4675 }
4676 mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ);
4677 mac_prop_info_set_range_uint32(prh, min, max);
4678 break;
4679 }
4680 case MAC_PROP_PRIVATE: {
4681 char valstr[64];
4682 int value;
4683
4684 if (strcmp(pr_name, "_ibd_broadcast_group") == 0) {
4685 mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ);
4686 return;
4687 } else if (strcmp(pr_name, "_ibd_coalesce_completions") == 0) {
4688 value = IBD_DEF_COALESCE_COMPLETIONS;
4689 } else if (strcmp(pr_name,
4690 "_ibd_create_broadcast_group") == 0) {
4691 value = IBD_DEF_CREATE_BCAST_GROUP;
4692 } else if (strcmp(pr_name, "_ibd_hash_size") == 0) {
4693 value = IBD_DEF_HASH_SIZE;
4694 } else if (strcmp(pr_name, "_ibd_lso_enable") == 0) {
4695 value = IBD_DEF_LSO_POLICY;
4696 } else if (strcmp(pr_name, "_ibd_num_ah") == 0) {
4697 value = IBD_DEF_NUM_AH;
4698 } else if (strcmp(pr_name, "_ibd_num_lso_bufs") == 0) {
4699 value = IBD_DEF_NUM_LSO_BUFS;
4700 } else if (strcmp(pr_name, "_ibd_rc_enable_srq") == 0) {
4701 value = IBD_DEF_RC_ENABLE_SRQ;
4702 } else if (strcmp(pr_name, "_ibd_rc_num_rwqe") == 0) {
4703 value = IBD_DEF_RC_NUM_RWQE;
4704 } else if (strcmp(pr_name, "_ibd_rc_num_srq") == 0) {
4705 value = IBD_DEF_RC_NUM_SRQ;
4706 } else if (strcmp(pr_name, "_ibd_rc_num_swqe") == 0) {
4707 value = IBD_DEF_RC_NUM_SWQE;
4708 } else if (strcmp(pr_name, "_ibd_rc_rx_comp_count") == 0) {
4709 value = IBD_DEF_RC_RX_COMP_COUNT;
4710 } else if (strcmp(pr_name, "_ibd_rc_rx_comp_usec") == 0) {
4711 value = IBD_DEF_RC_RX_COMP_USEC;
4712 } else if (strcmp(pr_name, "_ibd_rc_rx_copy_thresh") == 0) {
4713 value = IBD_DEF_RC_RX_COPY_THRESH;
4714 } else if (strcmp(pr_name, "_ibd_rc_rx_rwqe_thresh") == 0) {
4715 value = IBD_DEF_RC_RX_RWQE_THRESH;
4716 } else if (strcmp(pr_name, "_ibd_rc_tx_comp_count") == 0) {
4717 value = IBD_DEF_RC_TX_COMP_COUNT;
4718 } else if (strcmp(pr_name, "_ibd_rc_tx_comp_usec") == 0) {
4719 value = IBD_DEF_RC_TX_COMP_USEC;
4720 } else if (strcmp(pr_name, "_ibd_rc_tx_copy_thresh") == 0) {
4721 value = IBD_DEF_RC_TX_COPY_THRESH;
4722 } else if (strcmp(pr_name, "_ibd_ud_num_rwqe") == 0) {
4723 value = IBD_DEF_UD_NUM_RWQE;
4724 } else if (strcmp(pr_name, "_ibd_ud_num_swqe") == 0) {
4725 value = IBD_DEF_UD_NUM_SWQE;
4726 } else if (strcmp(pr_name, "_ibd_ud_rx_comp_count") == 0) {
4727 value = IBD_DEF_UD_RX_COMP_COUNT;
4728 } else if (strcmp(pr_name, "_ibd_ud_rx_comp_usec") == 0) {
4729 value = IBD_DEF_UD_RX_COMP_USEC;
4730 } else if (strcmp(pr_name, "_ibd_ud_tx_comp_count") == 0) {
4731 value = IBD_DEF_UD_TX_COMP_COUNT;
4732 } else if (strcmp(pr_name, "_ibd_ud_tx_comp_usec") == 0) {
4733 value = IBD_DEF_UD_TX_COMP_USEC;
4734 } else if (strcmp(pr_name, "_ibd_ud_tx_copy_thresh") == 0) {
4735 value = IBD_DEF_UD_TX_COPY_THRESH;
4736 } else {
4737 return;
4738 }
4739
4740 (void) snprintf(valstr, sizeof (valstr), "%d", value);
4741 mac_prop_info_set_default_str(prh, valstr);
4742 break;
4743 }
4744 } /* switch (pr_num) */
4745 }
4746
4747 /* ARGSUSED2 */
4748 static int
4749 ibd_set_priv_prop(ibd_state_t *state, const char *pr_name,
4750 uint_t pr_valsize, const void *pr_val)
4751 {
4752 int err = 0;
4753 long result;
4754
4755 if (strcmp(pr_name, "_ibd_coalesce_completions") == 0) {
4756 if (pr_val == NULL) {
4757 return (EINVAL);
4758 }
4759 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4760 if (result < 0 || result > 1) {
4761 err = EINVAL;
4762 } else {
4763 state->id_allow_coalesce_comp_tuning = (result == 1) ?
4764 B_TRUE: B_FALSE;
4765 }
4766 return (err);
4767 }
4768 if (strcmp(pr_name, "_ibd_create_broadcast_group") == 0) {
4769 if (state->id_mac_state & IBD_DRV_STARTED) {
4770 return (EBUSY);
4771 }
4772 if (pr_val == NULL) {
4773 return (EINVAL);
4774 }
4775 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4776 if (result < 0 || result > 1) {
4777 err = EINVAL;
4778 } else {
4779 state->id_create_broadcast_group = (result == 1) ?
4780 B_TRUE: B_FALSE;
4781 }
4782 return (err);
4783 }
4784 if (strcmp(pr_name, "_ibd_hash_size") == 0) {
4785 if (state->id_mac_state & IBD_DRV_STARTED) {
4786 return (EBUSY);
4787 }
4788 if (pr_val == NULL) {
4789 return (EINVAL);
4790 }
4791 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4792 if (result < IBD_MIN_HASH_SIZE || result > IBD_MAX_HASH_SIZE) {
4793 err = EINVAL;
4794 } else {
4795 state->id_hash_size = (uint32_t)result;
4796 }
4797 return (err);
4798 }
4799 if (strcmp(pr_name, "_ibd_lso_enable") == 0) {
4800 if (state->id_mac_state & IBD_DRV_STARTED) {
4801 return (EBUSY);
4802 }
4803 if (pr_val == NULL) {
4804 return (EINVAL);
4805 }
4806 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4807 if (result < 0 || result > 1) {
4808 err = EINVAL;
4809 } else {
4810 state->id_lso_policy = (result == 1) ?
4811 B_TRUE: B_FALSE;
4812 }
4813 mac_capab_update(state->id_mh);
4814 return (err);
4815 }
4816 if (strcmp(pr_name, "_ibd_num_ah") == 0) {
4817 if (state->id_mac_state & IBD_DRV_STARTED) {
4818 return (EBUSY);
4819 }
4820 if (pr_val == NULL) {
4821 return (EINVAL);
4822 }
4823 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4824 if (result < IBD_MIN_NUM_AH || result > IBD_MAX_NUM_AH) {
4825 err = EINVAL;
4826 } else {
4827 state->id_num_ah = (uint32_t)result;
4828 }
4829 return (err);
4830 }
4831 if (strcmp(pr_name, "_ibd_num_lso_bufs") == 0) {
4832 if (state->id_mac_state & IBD_DRV_STARTED) {
4833 return (EBUSY);
4834 }
4835 if (!state->id_lso_policy || !state->id_lso_capable) {
4836 return (EINVAL);
4837 }
4838 if (pr_val == NULL) {
4839 return (EINVAL);
4840 }
4841 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4842 if (result < IBD_MIN_NUM_LSO_BUFS ||
4843 result > IBD_MAX_NUM_LSO_BUFS) {
4844 err = EINVAL;
4845 } else {
4846 state->id_num_lso_bufs = (uint32_t)result;
4847 }
4848 return (err);
4849 }
4850 if (strcmp(pr_name, "_ibd_rc_enable_srq") == 0) {
4851 if (state->id_mac_state & IBD_DRV_STARTED) {
4852 return (EBUSY);
4853 }
4854 if (pr_val == NULL) {
4855 return (EINVAL);
4856 }
4857 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4858 if (result < 0 || result > 1) {
4859 err = EINVAL;
4860 } else {
4861 state->rc_enable_srq = (result == 1) ?
4862 B_TRUE: B_FALSE;
4863 }
4864 if (!state->rc_enable_srq) {
4865 state->id_rc_num_srq = 0;
4866 }
4867 return (err);
4868 }
4869 if (strcmp(pr_name, "_ibd_rc_num_rwqe") == 0) {
4870 if (state->id_mac_state & IBD_DRV_STARTED) {
4871 return (EBUSY);
4872 }
4873 if (pr_val == NULL) {
4874 return (EINVAL);
4875 }
4876 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4877 if (result < IBD_MIN_RC_NUM_RWQE ||
4878 result > IBD_MAX_RC_NUM_RWQE) {
4879 err = EINVAL;
4880 } else {
4881 state->id_rc_num_rwqe = (uint32_t)result;
4882 if (state->id_allow_coalesce_comp_tuning &&
4883 state->id_rc_rx_comp_count > state->id_rc_num_rwqe)
4884 state->id_rc_rx_comp_count =
4885 state->id_rc_num_rwqe;
4886 if (state->id_rc_num_srq > state->id_rc_num_rwqe)
4887 state->id_rc_num_srq =
4888 state->id_rc_num_rwqe - 1;
4889 /*
4890 * If rx_rwqe_threshold is greater than the number of
4891 * rwqes, pull it back to 25% of number of rwqes.
4892 */
4893 if (state->id_rc_rx_rwqe_thresh > state->id_rc_num_rwqe)
4894 state->id_rc_rx_rwqe_thresh =
4895 (state->id_rc_num_rwqe >> 2);
4896
4897 }
4898 return (err);
4899 }
4900 if (strcmp(pr_name, "_ibd_rc_num_srq") == 0) {
4901 if (state->id_mac_state & IBD_DRV_STARTED) {
4902 return (EBUSY);
4903 }
4904 if (pr_val == NULL) {
4905 return (EINVAL);
4906 }
4907 if (!state->rc_enable_srq)
4908 return (EINVAL);
4909
4910 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4911 if (result < IBD_MIN_RC_NUM_SRQ ||
4912 result >= state->id_rc_num_rwqe) {
4913 err = EINVAL;
4914 } else
4915 state->id_rc_num_srq = (uint32_t)result;
4916 return (err);
4917 }
4918 if (strcmp(pr_name, "_ibd_rc_num_swqe") == 0) {
4919 if (state->id_mac_state & IBD_DRV_STARTED) {
4920 return (EBUSY);
4921 }
4922 if (pr_val == NULL) {
4923 return (EINVAL);
4924 }
4925 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4926 if (result < IBD_MIN_RC_NUM_SWQE ||
4927 result > IBD_MAX_RC_NUM_SWQE) {
4928 err = EINVAL;
4929 } else {
4930 state->id_rc_num_swqe = (uint32_t)result;
4931 if (state->id_allow_coalesce_comp_tuning &&
4932 state->id_rc_tx_comp_count > state->id_rc_num_swqe)
4933 state->id_rc_tx_comp_count =
4934 state->id_rc_num_swqe;
4935 }
4936 return (err);
4937 }
4938 if (strcmp(pr_name, "_ibd_rc_rx_comp_count") == 0) {
4939 if (!state->id_allow_coalesce_comp_tuning) {
4940 return (ENOTSUP);
4941 }
4942 if (pr_val == NULL) {
4943 return (EINVAL);
4944 }
4945 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4946 if (result < 1 || result > state->id_rc_num_rwqe) {
4947 err = EINVAL;
4948 } else {
4949 state->id_rc_rx_comp_count = (uint32_t)result;
4950 }
4951 return (err);
4952 }
4953 if (strcmp(pr_name, "_ibd_rc_rx_comp_usec") == 0) {
4954 if (!state->id_allow_coalesce_comp_tuning) {
4955 return (ENOTSUP);
4956 }
4957 if (pr_val == NULL) {
4958 return (EINVAL);
4959 }
4960 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4961 if (result < 1) {
4962 err = EINVAL;
4963 } else {
4964 state->id_rc_rx_comp_usec = (uint32_t)result;
4965 }
4966 return (err);
4967 }
4968 if (strcmp(pr_name, "_ibd_rc_rx_copy_thresh") == 0) {
4969 if (state->id_mac_state & IBD_DRV_STARTED) {
4970 return (EBUSY);
4971 }
4972 if (pr_val == NULL) {
4973 return (EINVAL);
4974 }
4975 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4976 if (result < IBD_MIN_RC_RX_COPY_THRESH ||
4977 result > state->rc_mtu) {
4978 err = EINVAL;
4979 } else {
4980 state->id_rc_rx_copy_thresh = (uint32_t)result;
4981 }
4982 return (err);
4983 }
4984 if (strcmp(pr_name, "_ibd_rc_rx_rwqe_thresh") == 0) {
4985 if (state->id_mac_state & IBD_DRV_STARTED) {
4986 return (EBUSY);
4987 }
4988 if (pr_val == NULL) {
4989 return (EINVAL);
4990 }
4991 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
4992 if (result < IBD_MIN_RC_RX_RWQE_THRESH ||
4993 result >= state->id_rc_num_rwqe) {
4994 err = EINVAL;
4995 } else {
4996 state->id_rc_rx_rwqe_thresh = (uint32_t)result;
4997 }
4998 return (err);
4999 }
5000 if (strcmp(pr_name, "_ibd_rc_tx_comp_count") == 0) {
5001 if (!state->id_allow_coalesce_comp_tuning) {
5002 return (ENOTSUP);
5003 }
5004 if (pr_val == NULL) {
5005 return (EINVAL);
5006 }
5007 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5008 if (result < 1 || result > state->id_rc_num_swqe) {
5009 err = EINVAL;
5010 } else {
5011 state->id_rc_tx_comp_count = (uint32_t)result;
5012 }
5013 return (err);
5014 }
5015 if (strcmp(pr_name, "_ibd_rc_tx_comp_usec") == 0) {
5016 if (!state->id_allow_coalesce_comp_tuning) {
5017 return (ENOTSUP);
5018 }
5019 if (pr_val == NULL) {
5020 return (EINVAL);
5021 }
5022 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5023 if (result < 1)
5024 err = EINVAL;
5025 else {
5026 state->id_rc_tx_comp_usec = (uint32_t)result;
5027 }
5028 return (err);
5029 }
5030 if (strcmp(pr_name, "_ibd_rc_tx_copy_thresh") == 0) {
5031 if (state->id_mac_state & IBD_DRV_STARTED) {
5032 return (EBUSY);
5033 }
5034 if (pr_val == NULL) {
5035 return (EINVAL);
5036 }
5037 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5038 if (result < IBD_MIN_RC_TX_COPY_THRESH ||
5039 result > state->rc_mtu) {
5040 err = EINVAL;
5041 } else {
5042 state->id_rc_tx_copy_thresh = (uint32_t)result;
5043 }
5044 return (err);
5045 }
5046 if (strcmp(pr_name, "_ibd_ud_num_rwqe") == 0) {
5047 if (state->id_mac_state & IBD_DRV_STARTED) {
5048 return (EBUSY);
5049 }
5050 if (pr_val == NULL) {
5051 return (EINVAL);
5052 }
5053 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5054 if (result < IBD_MIN_UD_NUM_RWQE ||
5055 result > IBD_MAX_UD_NUM_RWQE) {
5056 err = EINVAL;
5057 } else {
5058 if (result > state->id_hca_max_chan_sz) {
5059 state->id_ud_num_rwqe =
5060 state->id_hca_max_chan_sz;
5061 } else {
5062 state->id_ud_num_rwqe = (uint32_t)result;
5063 }
5064 if (state->id_allow_coalesce_comp_tuning &&
5065 state->id_ud_rx_comp_count > state->id_ud_num_rwqe)
5066 state->id_ud_rx_comp_count =
5067 state->id_ud_num_rwqe;
5068 }
5069 return (err);
5070 }
5071 if (strcmp(pr_name, "_ibd_ud_num_swqe") == 0) {
5072 if (state->id_mac_state & IBD_DRV_STARTED) {
5073 return (EBUSY);
5074 }
5075 if (pr_val == NULL) {
5076 return (EINVAL);
5077 }
5078 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5079 if (result < IBD_MIN_UD_NUM_SWQE ||
5080 result > IBD_MAX_UD_NUM_SWQE) {
5081 err = EINVAL;
5082 } else {
5083 if (result > state->id_hca_max_chan_sz) {
5084 state->id_ud_num_swqe =
5085 state->id_hca_max_chan_sz;
5086 } else {
5087 state->id_ud_num_swqe = (uint32_t)result;
5088 }
5089 if (state->id_allow_coalesce_comp_tuning &&
5090 state->id_ud_tx_comp_count > state->id_ud_num_swqe)
5091 state->id_ud_tx_comp_count =
5092 state->id_ud_num_swqe;
5093 }
5094 return (err);
5095 }
5096 if (strcmp(pr_name, "_ibd_ud_rx_comp_count") == 0) {
5097 if (!state->id_allow_coalesce_comp_tuning) {
5098 return (ENOTSUP);
5099 }
5100 if (pr_val == NULL) {
5101 return (EINVAL);
5102 }
5103 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5104 if (result < 1 || result > state->id_ud_num_rwqe) {
5105 err = EINVAL;
5106 } else {
5107 state->id_ud_rx_comp_count = (uint32_t)result;
5108 }
5109 return (err);
5110 }
5111 if (strcmp(pr_name, "_ibd_ud_rx_comp_usec") == 0) {
5112 if (!state->id_allow_coalesce_comp_tuning) {
5113 return (ENOTSUP);
5114 }
5115 if (pr_val == NULL) {
5116 return (EINVAL);
5117 }
5118 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5119 if (result < 1) {
5120 err = EINVAL;
5121 } else {
5122 state->id_ud_rx_comp_usec = (uint32_t)result;
5123 }
5124 return (err);
5125 }
5126 if (strcmp(pr_name, "_ibd_ud_tx_comp_count") == 0) {
5127 if (!state->id_allow_coalesce_comp_tuning) {
5128 return (ENOTSUP);
5129 }
5130 if (pr_val == NULL) {
5131 return (EINVAL);
5132 }
5133 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5134 if (result < 1 || result > state->id_ud_num_swqe) {
5135 err = EINVAL;
5136 } else {
5137 state->id_ud_tx_comp_count = (uint32_t)result;
5138 }
5139 return (err);
5140 }
5141 if (strcmp(pr_name, "_ibd_ud_tx_comp_usec") == 0) {
5142 if (!state->id_allow_coalesce_comp_tuning) {
5143 return (ENOTSUP);
5144 }
5145 if (pr_val == NULL) {
5146 return (EINVAL);
5147 }
5148 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5149 if (result < 1) {
5150 err = EINVAL;
5151 } else {
5152 state->id_ud_tx_comp_usec = (uint32_t)result;
5153 }
5154 return (err);
5155 }
5156 if (strcmp(pr_name, "_ibd_ud_tx_copy_thresh") == 0) {
5157 if (state->id_mac_state & IBD_DRV_STARTED) {
5158 return (EBUSY);
5159 }
5160 if (pr_val == NULL) {
5161 return (EINVAL);
5162 }
5163 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result);
5164 if (result < IBD_MIN_UD_TX_COPY_THRESH ||
5165 result > IBD_MAX_UD_TX_COPY_THRESH) {
5166 err = EINVAL;
5167 } else {
5168 state->id_ud_tx_copy_thresh = (uint32_t)result;
5169 }
5170 return (err);
5171 }
5172 return (ENOTSUP);
5173 }
5174
5175 static int
5176 ibd_get_priv_prop(ibd_state_t *state, const char *pr_name, uint_t pr_valsize,
5177 void *pr_val)
5178 {
5179 int err = ENOTSUP;
5180 int value;
5181
5182 if (strcmp(pr_name, "_ibd_broadcast_group") == 0) {
5183 value = state->id_bgroup_present;
5184 err = 0;
5185 goto done;
5186 }
5187 if (strcmp(pr_name, "_ibd_coalesce_completions") == 0) {
5188 value = state->id_allow_coalesce_comp_tuning;
5189 err = 0;
5190 goto done;
5191 }
5192 if (strcmp(pr_name, "_ibd_create_broadcast_group") == 0) {
5193 value = state->id_create_broadcast_group;
5194 err = 0;
5195 goto done;
5196 }
5197 if (strcmp(pr_name, "_ibd_hash_size") == 0) {
5198 value = state->id_hash_size;
5199 err = 0;
5200 goto done;
5201 }
5202 if (strcmp(pr_name, "_ibd_lso_enable") == 0) {
5203 value = state->id_lso_policy;
5204 err = 0;
5205 goto done;
5206 }
5207 if (strcmp(pr_name, "_ibd_num_ah") == 0) {
5208 value = state->id_num_ah;
5209 err = 0;
5210 goto done;
5211 }
5212 if (strcmp(pr_name, "_ibd_num_lso_bufs") == 0) {
5213 value = state->id_num_lso_bufs;
5214 err = 0;
5215 goto done;
5216 }
5217 if (strcmp(pr_name, "_ibd_rc_enable_srq") == 0) {
5218 value = state->rc_enable_srq;
5219 err = 0;
5220 goto done;
5221 }
5222 if (strcmp(pr_name, "_ibd_rc_num_rwqe") == 0) {
5223 value = state->id_rc_num_rwqe;
5224 err = 0;
5225 goto done;
5226 }
5227 if (strcmp(pr_name, "_ibd_rc_num_srq") == 0) {
5228 value = state->id_rc_num_srq;
5229 err = 0;
5230 goto done;
5231 }
5232 if (strcmp(pr_name, "_ibd_rc_num_swqe") == 0) {
5233 value = state->id_rc_num_swqe;
5234 err = 0;
5235 goto done;
5236 }
5237 if (strcmp(pr_name, "_ibd_rc_rx_comp_count") == 0) {
5238 value = state->id_rc_rx_comp_count;
5239 err = 0;
5240 goto done;
5241 }
5242 if (strcmp(pr_name, "_ibd_rc_rx_comp_usec") == 0) {
5243 value = state->id_rc_rx_comp_usec;
5244 err = 0;
5245 goto done;
5246 }
5247 if (strcmp(pr_name, "_ibd_rc_rx_copy_thresh") == 0) {
5248 value = state->id_rc_rx_copy_thresh;
5249 err = 0;
5250 goto done;
5251 }
5252 if (strcmp(pr_name, "_ibd_rc_rx_rwqe_thresh") == 0) {
5253 value = state->id_rc_rx_rwqe_thresh;
5254 err = 0;
5255 goto done;
5256 }
5257 if (strcmp(pr_name, "_ibd_rc_tx_comp_count") == 0) {
5258 value = state->id_rc_tx_comp_count;
5259 err = 0;
5260 goto done;
5261 }
5262 if (strcmp(pr_name, "_ibd_rc_tx_comp_usec") == 0) {
5263 value = state->id_rc_tx_comp_usec;
5264 err = 0;
5265 goto done;
5266 }
5267 if (strcmp(pr_name, "_ibd_rc_tx_copy_thresh") == 0) {
5268 value = state->id_rc_tx_copy_thresh;
5269 err = 0;
5270 goto done;
5271 }
5272 if (strcmp(pr_name, "_ibd_ud_num_rwqe") == 0) {
5273 value = state->id_ud_num_rwqe;
5274 err = 0;
5275 goto done;
5276 }
5277 if (strcmp(pr_name, "_ibd_ud_num_swqe") == 0) {
5278 value = state->id_ud_num_swqe;
5279 err = 0;
5280 goto done;
5281 }
5282 if (strcmp(pr_name, "_ibd_ud_rx_comp_count") == 0) {
5283 value = state->id_ud_rx_comp_count;
5284 err = 0;
5285 goto done;
5286 }
5287 if (strcmp(pr_name, "_ibd_ud_rx_comp_usec") == 0) {
5288 value = state->id_ud_rx_comp_usec;
5289 err = 0;
5290 goto done;
5291 }
5292 if (strcmp(pr_name, "_ibd_ud_tx_comp_count") == 0) {
5293 value = state->id_ud_tx_comp_count;
5294 err = 0;
5295 goto done;
5296 }
5297 if (strcmp(pr_name, "_ibd_ud_tx_comp_usec") == 0) {
5298 value = state->id_ud_tx_comp_usec;
5299 err = 0;
5300 goto done;
5301 }
5302 if (strcmp(pr_name, "_ibd_ud_tx_copy_thresh") == 0) {
5303 value = state->id_ud_tx_copy_thresh;
5304 err = 0;
5305 goto done;
5306 }
5307 done:
5308 if (err == 0) {
5309 (void) snprintf(pr_val, pr_valsize, "%d", value);
5310 }
5311 return (err);
5312 }
5313
5314 static int
5315 ibd_get_port_details(ibd_state_t *state)
5316 {
5317 ibt_hca_portinfo_t *port_infop;
5318 ibt_status_t ret;
5319 uint_t psize, port_infosz;
5320
5321 mutex_enter(&state->id_link_mutex);
5322
5323 /*
5324 * Query for port information
5325 */
5326 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
5327 &port_infop, &psize, &port_infosz);
5328 if ((ret != IBT_SUCCESS) || (psize != 1)) {
5329 mutex_exit(&state->id_link_mutex);
5330 DPRINT(10, "ibd_get_port_details: ibt_query_hca_ports() "
5331 "failed, ret=%d", ret);
5332 return (ENETDOWN);
5333 }
5334
5335 /*
5336 * If the link is active, verify the pkey
5337 */
5338 if (port_infop->p_linkstate == IBT_PORT_ACTIVE) {
5339 if ((ret = ibt_pkey2index(state->id_hca_hdl, state->id_port,
5340 state->id_pkey, &state->id_pkix)) != IBT_SUCCESS) {
5341 state->id_link_state = LINK_STATE_DOWN;
5342 } else {
5343 state->id_link_state = LINK_STATE_UP;
5344 }
5345 state->id_mtu = (128 << port_infop->p_mtu);
5346 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
5347 state->id_sgid = *port_infop->p_sgid_tbl;
5348 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
5349 /*
5350 * Now that the port is active, record the port speed
5351 */
5352 state->id_link_speed = ibd_get_portspeed(state);
5353 } else {
5354 /* Make sure that these are handled in PORT_UP/CHANGE */
5355 state->id_mtu = 0;
5356 state->id_link_state = LINK_STATE_DOWN;
5357 state->id_link_speed = 0;
5358 }
5359 mutex_exit(&state->id_link_mutex);
5360 ibt_free_portinfo(port_infop, port_infosz);
5361
5362 return (0);
5363 }
5364
5365 static int
5366 ibd_alloc_cqs(ibd_state_t *state)
5367 {
5368 ibt_hca_attr_t hca_attrs;
5369 ibt_cq_attr_t cq_attr;
5370 ibt_status_t ret;
5371 uint32_t real_size;
5372 uint_t num_rwqe_change = 0;
5373 uint_t num_swqe_change = 0;
5374
5375 ret = ibt_query_hca(state->id_hca_hdl, &hca_attrs);
5376 ASSERT(ret == IBT_SUCCESS);
5377
5378 /*
5379 * Allocate Rx/combined CQ:
5380 * Theoretically, there is no point in having more than #rwqe
5381 * plus #swqe cqe's, except that the CQ will be signaled for
5382 * overflow when the last wqe completes, if none of the previous
5383 * cqe's have been polled. Thus, we allocate just a few less wqe's
5384 * to make sure such overflow does not occur.
5385 */
5386 cq_attr.cq_sched = NULL;
5387 cq_attr.cq_flags = IBT_CQ_NO_FLAGS;
5388
5389 /*
5390 * Allocate Receive CQ.
5391 */
5392 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_rwqe + 1)) {
5393 cq_attr.cq_size = state->id_ud_num_rwqe + 1;
5394 } else {
5395 cq_attr.cq_size = hca_attrs.hca_max_cq_sz;
5396 num_rwqe_change = state->id_ud_num_rwqe;
5397 state->id_ud_num_rwqe = cq_attr.cq_size - 1;
5398 }
5399
5400 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
5401 &state->id_rcq_hdl, &real_size)) != IBT_SUCCESS) {
5402 DPRINT(10, "ibd_alloc_cqs: ibt_alloc_cq(rcq) "
5403 "failed, ret=%d\n", ret);
5404 return (DDI_FAILURE);
5405 }
5406
5407 if ((ret = ibt_modify_cq(state->id_rcq_hdl, state->id_ud_rx_comp_count,
5408 state->id_ud_rx_comp_usec, 0)) != IBT_SUCCESS) {
5409 DPRINT(10, "ibd_alloc_cqs: Receive CQ interrupt "
5410 "moderation failed, ret=%d\n", ret);
5411 }
5412
5413 /* make the #rx wc's the same as max rx chain size */
5414 state->id_rxwcs_size = IBD_MAX_RX_MP_LEN;
5415 state->id_rxwcs = kmem_alloc(sizeof (ibt_wc_t) *
5416 state->id_rxwcs_size, KM_SLEEP);
5417
5418 /*
5419 * Allocate Send CQ.
5420 */
5421 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_swqe + 1)) {
5422 cq_attr.cq_size = state->id_ud_num_swqe + 1;
5423 } else {
5424 cq_attr.cq_size = hca_attrs.hca_max_cq_sz;
5425 num_swqe_change = state->id_ud_num_swqe;
5426 state->id_ud_num_swqe = cq_attr.cq_size - 1;
5427 }
5428
5429 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
5430 &state->id_scq_hdl, &real_size)) != IBT_SUCCESS) {
5431 DPRINT(10, "ibd_alloc_cqs: ibt_alloc_cq(scq) "
5432 "failed, ret=%d\n", ret);
5433 kmem_free(state->id_rxwcs, sizeof (ibt_wc_t) *
5434 state->id_rxwcs_size);
5435 (void) ibt_free_cq(state->id_rcq_hdl);
5436 return (DDI_FAILURE);
5437 }
5438 if ((ret = ibt_modify_cq(state->id_scq_hdl, state->id_ud_tx_comp_count,
5439 state->id_ud_tx_comp_usec, 0)) != IBT_SUCCESS) {
5440 DPRINT(10, "ibd_alloc_cqs: Send CQ interrupt "
5441 "moderation failed, ret=%d\n", ret);
5442 }
5443
5444 state->id_txwcs_size = IBD_TX_POLL_THRESH;
5445 state->id_txwcs = kmem_alloc(sizeof (ibt_wc_t) *
5446 state->id_txwcs_size, KM_SLEEP);
5447
5448 /*
5449 * Print message in case we could not allocate as many wqe's
5450 * as was requested.
5451 */
5452 if (num_rwqe_change) {
5453 ibd_print_warn(state, "Setting #rwqe = %d instead of default "
5454 "%d", state->id_ud_num_rwqe, num_rwqe_change);
5455 }
5456 if (num_swqe_change) {
5457 ibd_print_warn(state, "Setting #swqe = %d instead of default "
5458 "%d", state->id_ud_num_swqe, num_swqe_change);
5459 }
5460
5461 return (DDI_SUCCESS);
5462 }
5463
5464 static int
5465 ibd_setup_ud_channel(ibd_state_t *state)
5466 {
5467 ibt_ud_chan_alloc_args_t ud_alloc_attr;
5468 ibt_ud_chan_query_attr_t ud_chan_attr;
5469 ibt_status_t ret;
5470
5471 ud_alloc_attr.ud_flags = IBT_ALL_SIGNALED;
5472 if (state->id_hca_res_lkey_capab)
5473 ud_alloc_attr.ud_flags |= IBT_FAST_REG_RES_LKEY;
5474 if (state->id_lso_policy && state->id_lso_capable)
5475 ud_alloc_attr.ud_flags |= IBT_USES_LSO;
5476
5477 ud_alloc_attr.ud_hca_port_num = state->id_port;
5478 ud_alloc_attr.ud_sizes.cs_sq_sgl = state->id_max_sqseg;
5479 ud_alloc_attr.ud_sizes.cs_rq_sgl = IBD_MAX_RQSEG;
5480 ud_alloc_attr.ud_sizes.cs_sq = state->id_ud_num_swqe;
5481 ud_alloc_attr.ud_sizes.cs_rq = state->id_ud_num_rwqe;
5482 ud_alloc_attr.ud_qkey = state->id_mcinfo->mc_qkey;
5483 ud_alloc_attr.ud_scq = state->id_scq_hdl;
5484 ud_alloc_attr.ud_rcq = state->id_rcq_hdl;
5485 ud_alloc_attr.ud_pd = state->id_pd_hdl;
5486 ud_alloc_attr.ud_pkey_ix = state->id_pkix;
5487 ud_alloc_attr.ud_clone_chan = NULL;
5488
5489 if ((ret = ibt_alloc_ud_channel(state->id_hca_hdl, IBT_ACHAN_NO_FLAGS,
5490 &ud_alloc_attr, &state->id_chnl_hdl, NULL)) != IBT_SUCCESS) {
5491 DPRINT(10, "ibd_setup_ud_channel: ibt_alloc_ud_channel() "
5492 "failed, ret=%d\n", ret);
5493 return (DDI_FAILURE);
5494 }
5495
5496 if ((ret = ibt_query_ud_channel(state->id_chnl_hdl,
5497 &ud_chan_attr)) != IBT_SUCCESS) {
5498 DPRINT(10, "ibd_setup_ud_channel: ibt_query_ud_channel() "
5499 "failed, ret=%d\n", ret);
5500 (void) ibt_free_channel(state->id_chnl_hdl);
5501 return (DDI_FAILURE);
5502 }
5503
5504 state->id_qpnum = ud_chan_attr.ud_qpn;
5505
5506 return (DDI_SUCCESS);
5507 }
5508
5509 static int
5510 ibd_undo_start(ibd_state_t *state, link_state_t cur_link_state)
5511 {
5512 uint32_t progress = state->id_mac_state;
5513 uint_t attempts;
5514 ibt_status_t ret;
5515 ib_gid_t mgid;
5516 ibd_mce_t *mce;
5517 uint8_t jstate;
5518 timeout_id_t tid;
5519
5520 if (atomic_dec_32_nv(&state->id_running) != 0)
5521 cmn_err(CE_WARN, "ibd_undo_start: id_running was not 1\n");
5522
5523 /*
5524 * Before we try to stop/undo whatever we did in ibd_start(),
5525 * we need to mark the link state appropriately to prevent the
5526 * ip layer from using this instance for any new transfers. Note
5527 * that if the original state of the link was "up" when we're
5528 * here, we'll set the final link state to "unknown", to behave
5529 * in the same fashion as other ethernet drivers.
5530 */
5531 mutex_enter(&state->id_link_mutex);
5532 if (cur_link_state == LINK_STATE_DOWN) {
5533 state->id_link_state = cur_link_state;
5534 } else {
5535 state->id_link_state = LINK_STATE_UNKNOWN;
5536 }
5537 mutex_exit(&state->id_link_mutex);
5538 bzero(&state->id_macaddr, sizeof (ipoib_mac_t));
5539 mac_link_update(state->id_mh, state->id_link_state);
5540
5541 state->id_mac_state &= (~IBD_DRV_PORT_DETAILS_OBTAINED);
5542 if (progress & IBD_DRV_STARTED) {
5543 state->id_mac_state &= (~IBD_DRV_STARTED);
5544 }
5545
5546 if (progress & IBD_DRV_IN_LATE_HCA_INIT) {
5547 state->id_mac_state &= (~IBD_DRV_IN_LATE_HCA_INIT);
5548 }
5549
5550 /* Stop listen under Reliable Connected Mode */
5551 if (progress & IBD_DRV_RC_LISTEN) {
5552 ASSERT(state->id_enable_rc);
5553 if (state->rc_listen_hdl != NULL) {
5554 ibd_rc_stop_listen(state);
5555 }
5556 state->id_mac_state &= (~IBD_DRV_RC_LISTEN);
5557 }
5558
5559 /* Stop timeout routine */
5560 if (progress & IBD_DRV_RC_TIMEOUT) {
5561 ASSERT(state->id_enable_rc);
5562 mutex_enter(&state->rc_timeout_lock);
5563 state->rc_timeout_start = B_FALSE;
5564 tid = state->rc_timeout;
5565 state->rc_timeout = 0;
5566 mutex_exit(&state->rc_timeout_lock);
5567 if (tid != 0)
5568 (void) untimeout(tid);
5569 state->id_mac_state &= (~IBD_DRV_RC_TIMEOUT);
5570 }
5571
5572 if ((state->id_enable_rc) && (progress & IBD_DRV_ACACHE_INITIALIZED)) {
5573 attempts = 100;
5574 while (state->id_ah_op == IBD_OP_ONGOING) {
5575 /*
5576 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB
5577 * port is connecting to a remote IPoIB port. Wait for
5578 * the end of this connecting operation.
5579 */
5580 delay(drv_usectohz(100000));
5581 if (--attempts == 0) {
5582 state->rc_stop_connect++;
5583 DPRINT(40, "ibd_undo_start: connecting");
5584 break;
5585 }
5586 }
5587 mutex_enter(&state->id_sched_lock);
5588 state->id_sched_needed = 0;
5589 mutex_exit(&state->id_sched_lock);
5590 (void) ibd_rc_close_all_chan(state);
5591 }
5592
5593 /*
5594 * First, stop receive interrupts; this stops the driver from
5595 * handing up buffers to higher layers. Wait for receive buffers
5596 * to be returned and give up after 1 second.
5597 */
5598 if (progress & IBD_DRV_RCQ_NOTIFY_ENABLED) {
5599 attempts = 10;
5600 while (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding,
5601 0) > 0) {
5602 delay(drv_usectohz(100000));
5603 if (--attempts == 0) {
5604 /*
5605 * There are pending bufs with the network
5606 * layer and we have no choice but to wait
5607 * for them to be done with. Reap all the
5608 * Tx/Rx completions that were posted since
5609 * we turned off the notification and
5610 * return failure.
5611 */
5612 cmn_err(CE_CONT, "!ibd: bufs outstanding\n");
5613 DPRINT(2, "ibd_undo_start: "
5614 "reclaiming failed");
5615 break;
5616 }
5617 }
5618 state->id_mac_state &= (~IBD_DRV_RCQ_NOTIFY_ENABLED);
5619 }
5620
5621 if (progress & IBD_DRV_RC_LARGEBUF_ALLOCD) {
5622 ibd_rc_fini_tx_largebuf_list(state);
5623 state->id_mac_state &= (~IBD_DRV_RC_LARGEBUF_ALLOCD);
5624 }
5625
5626 if (progress & IBD_DRV_RC_SRQ_ALLOCD) {
5627 ASSERT(state->id_enable_rc);
5628 if (state->rc_srq_rwqe_list.dl_bufs_outstanding == 0) {
5629 if (state->id_ah_op == IBD_OP_ONGOING) {
5630 delay(drv_usectohz(10000));
5631 if (state->id_ah_op == IBD_OP_ONGOING) {
5632 /*
5633 * "state->id_ah_op == IBD_OP_ONGOING"
5634 * means this IPoIB port is connecting
5635 * to a remote IPoIB port. We can't
5636 * delete SRQ here.
5637 */
5638 state->rc_stop_connect++;
5639 DPRINT(40, "ibd_undo_start: "
5640 "connecting");
5641 } else {
5642 ibd_rc_fini_srq_list(state);
5643 state->id_mac_state &=
5644 (~IBD_DRV_RC_SRQ_ALLOCD);
5645 }
5646 } else {
5647 ibd_rc_fini_srq_list(state);
5648 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD);
5649 }
5650 } else {
5651 DPRINT(40, "ibd_undo_start: srq bufs outstanding\n");
5652 }
5653 }
5654
5655 if (progress & IBD_DRV_SM_NOTICES_REGISTERED) {
5656 ibt_register_subnet_notices(state->id_ibt_hdl, NULL, NULL);
5657
5658 mutex_enter(&state->id_trap_lock);
5659 state->id_trap_stop = B_TRUE;
5660 while (state->id_trap_inprog > 0)
5661 cv_wait(&state->id_trap_cv, &state->id_trap_lock);
5662 mutex_exit(&state->id_trap_lock);
5663
5664 state->id_mac_state &= (~IBD_DRV_SM_NOTICES_REGISTERED);
5665 }
5666
5667 if (progress & IBD_DRV_SCQ_NOTIFY_ENABLED) {
5668 /*
5669 * Flushing the channel ensures that all pending WQE's
5670 * are marked with flush_error and handed to the CQ. It
5671 * does not guarantee the invocation of the CQ handler.
5672 * This call is guaranteed to return successfully for
5673 * UD QPNs.
5674 */
5675 if ((ret = ibt_flush_channel(state->id_chnl_hdl)) !=
5676 IBT_SUCCESS) {
5677 DPRINT(10, "ibd_undo_start: flush_channel "
5678 "failed, ret=%d", ret);
5679 }
5680
5681 /*
5682 * Give some time for the TX CQ handler to process the
5683 * completions.
5684 */
5685 attempts = 10;
5686 mutex_enter(&state->id_tx_list.dl_mutex);
5687 mutex_enter(&state->id_tx_rel_list.dl_mutex);
5688 while (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt
5689 != state->id_ud_num_swqe) {
5690 if (--attempts == 0)
5691 break;
5692 mutex_exit(&state->id_tx_rel_list.dl_mutex);
5693 mutex_exit(&state->id_tx_list.dl_mutex);
5694 delay(drv_usectohz(100000));
5695 mutex_enter(&state->id_tx_list.dl_mutex);
5696 mutex_enter(&state->id_tx_rel_list.dl_mutex);
5697 }
5698 ibt_set_cq_handler(state->id_scq_hdl, 0, 0);
5699 if (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt !=
5700 state->id_ud_num_swqe) {
5701 cmn_err(CE_WARN, "tx resources not freed\n");
5702 }
5703 mutex_exit(&state->id_tx_rel_list.dl_mutex);
5704 mutex_exit(&state->id_tx_list.dl_mutex);
5705
5706 attempts = 10;
5707 while (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) {
5708 if (--attempts == 0)
5709 break;
5710 delay(drv_usectohz(100000));
5711 }
5712 ibt_set_cq_handler(state->id_rcq_hdl, 0, 0);
5713 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) {
5714 cmn_err(CE_WARN, "rx resources not freed\n");
5715 }
5716
5717 state->id_mac_state &= (~IBD_DRV_SCQ_NOTIFY_ENABLED);
5718 }
5719
5720 if (progress & IBD_DRV_BCAST_GROUP_JOINED) {
5721 /*
5722 * Drop all residual full/non membership. This includes full
5723 * membership to the broadcast group, and any nonmembership
5724 * acquired during transmits. We do this after the Tx completion
5725 * handlers are done, since those might result in some late
5726 * leaves; this also eliminates a potential race with that
5727 * path wrt the mc full list insert/delete. Trap handling
5728 * has also been suppressed at this point. Thus, no locks
5729 * are required while traversing the mc full list.
5730 */
5731 DPRINT(2, "ibd_undo_start: clear full cache entries");
5732 mce = list_head(&state->id_mc_full);
5733 while (mce != NULL) {
5734 mgid = mce->mc_info.mc_adds_vect.av_dgid;
5735 jstate = mce->mc_jstate;
5736 mce = list_next(&state->id_mc_full, mce);
5737 ibd_leave_group(state, mgid, jstate);
5738 }
5739 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_JOINED);
5740 }
5741
5742 if (progress & IBD_DRV_RXLIST_ALLOCD) {
5743 ibd_fini_rxlist(state);
5744 state->id_mac_state &= (~IBD_DRV_RXLIST_ALLOCD);
5745 }
5746
5747 if (progress & IBD_DRV_TXLIST_ALLOCD) {
5748 ibd_fini_txlist(state);
5749 state->id_mac_state &= (~IBD_DRV_TXLIST_ALLOCD);
5750 }
5751
5752 if (progress & IBD_DRV_UD_CHANNEL_SETUP) {
5753 if ((ret = ibt_free_channel(state->id_chnl_hdl)) !=
5754 IBT_SUCCESS) {
5755 DPRINT(10, "ibd_undo_start: free_channel "
5756 "failed, ret=%d", ret);
5757 }
5758
5759 state->id_mac_state &= (~IBD_DRV_UD_CHANNEL_SETUP);
5760 }
5761
5762 if (progress & IBD_DRV_CQS_ALLOCD) {
5763 kmem_free(state->id_txwcs,
5764 sizeof (ibt_wc_t) * state->id_txwcs_size);
5765 if ((ret = ibt_free_cq(state->id_scq_hdl)) !=
5766 IBT_SUCCESS) {
5767 DPRINT(10, "ibd_undo_start: free_cq(scq) "
5768 "failed, ret=%d", ret);
5769 }
5770
5771 kmem_free(state->id_rxwcs,
5772 sizeof (ibt_wc_t) * state->id_rxwcs_size);
5773 if ((ret = ibt_free_cq(state->id_rcq_hdl)) != IBT_SUCCESS) {
5774 DPRINT(10, "ibd_undo_start: free_cq(rcq) failed, "
5775 "ret=%d", ret);
5776 }
5777
5778 state->id_txwcs = NULL;
5779 state->id_rxwcs = NULL;
5780 state->id_scq_hdl = NULL;
5781 state->id_rcq_hdl = NULL;
5782
5783 state->id_mac_state &= (~IBD_DRV_CQS_ALLOCD);
5784 }
5785
5786 if (progress & IBD_DRV_ACACHE_INITIALIZED) {
5787 mutex_enter(&state->id_ac_mutex);
5788 mod_hash_destroy_hash(state->id_ah_active_hash);
5789 mutex_exit(&state->id_ac_mutex);
5790 ibd_acache_fini(state);
5791
5792 state->id_mac_state &= (~IBD_DRV_ACACHE_INITIALIZED);
5793 }
5794
5795 if (progress & IBD_DRV_BCAST_GROUP_FOUND) {
5796 /*
5797 * If we'd created the ipoib broadcast group and had
5798 * successfully joined it, leave it now
5799 */
5800 if (state->id_bgroup_created) {
5801 mgid = state->id_mcinfo->mc_adds_vect.av_dgid;
5802 jstate = IB_MC_JSTATE_FULL;
5803 (void) ibt_leave_mcg(state->id_sgid, mgid,
5804 state->id_sgid, jstate);
5805 }
5806 ibt_free_mcg_info(state->id_mcinfo, 1);
5807
5808 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_FOUND);
5809 }
5810
5811 return (DDI_SUCCESS);
5812 }
5813
5814 /*
5815 * These pair of routines are used to set/clear the condition that
5816 * the caller is likely to do something to change the id_mac_state.
5817 * If there's already someone doing either a start or a stop (possibly
5818 * due to the async handler detecting a pkey relocation event, a plumb
5819 * or dlpi_open, or an unplumb or dlpi_close coming in), we wait until
5820 * that's done.
5821 */
5822 static void
5823 ibd_set_mac_progress(ibd_state_t *state, uint_t flag)
5824 {
5825 mutex_enter(&state->id_macst_lock);
5826 while (state->id_mac_state & IBD_DRV_RESTART_IN_PROGRESS)
5827 cv_wait(&state->id_macst_cv, &state->id_macst_lock);
5828
5829 state->id_mac_state |= flag;
5830 mutex_exit(&state->id_macst_lock);
5831 }
5832
5833 static void
5834 ibd_clr_mac_progress(ibd_state_t *state, uint_t flag)
5835 {
5836 mutex_enter(&state->id_macst_lock);
5837 state->id_mac_state &= (~flag);
5838 cv_signal(&state->id_macst_cv);
5839 mutex_exit(&state->id_macst_lock);
5840 }
5841
5842 /*
5843 * GLDv3 entry point to start hardware.
5844 */
5845 /*ARGSUSED*/
5846 static int
5847 ibd_m_start(void *arg)
5848 {
5849 ibd_state_t *state = arg;
5850 int ret;
5851
5852 if (state->id_type == IBD_PORT_DRIVER)
5853 return (EINVAL);
5854
5855 ibd_set_mac_progress(state, IBD_DRV_START_IN_PROGRESS);
5856 if (state->id_mac_state & IBD_DRV_IN_DELETION) {
5857 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS);
5858 return (EIO);
5859 }
5860
5861 ret = ibd_start(state);
5862 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS);
5863 return (ret);
5864 }
5865
5866 static int
5867 ibd_start(ibd_state_t *state)
5868 {
5869 int err;
5870 ibt_status_t ret;
5871 int late_hca_init = 0;
5872
5873 if (state->id_mac_state & IBD_DRV_STARTED)
5874 return (DDI_SUCCESS);
5875
5876 /*
5877 * We do not increment the running flag when calling ibd_start() as
5878 * a result of some event which moves the state away from late HCA
5879 * initialization viz. MCG_CREATED, PORT_CHANGE or link availability.
5880 */
5881 if (!(state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) &&
5882 (atomic_inc_32_nv(&state->id_running) != 1)) {
5883 DPRINT(10, "ibd_start: id_running is non-zero");
5884 cmn_err(CE_WARN, "ibd_start: id_running was not 0\n");
5885 atomic_dec_32(&state->id_running);
5886 return (EINVAL);
5887 }
5888
5889 /*
5890 * Get port details; if we fail here, something bad happened.
5891 * Fail plumb.
5892 */
5893 if ((err = ibd_get_port_details(state)) != 0) {
5894 DPRINT(10, "ibd_start: ibd_get_port_details() failed");
5895 goto start_fail;
5896 }
5897 /*
5898 * If state->id_link_state is DOWN, it indicates that either the port
5899 * is down, or the pkey is not available. In both cases, resort to late
5900 * initialization. Register for subnet notices, and return success.
5901 */
5902 state->id_mac_state |= IBD_DRV_PORT_DETAILS_OBTAINED;
5903 if (state->id_link_state == LINK_STATE_DOWN) {
5904 late_hca_init = 1;
5905 goto late_hca_init_return;
5906 }
5907
5908 /*
5909 * Find the IPoIB broadcast group
5910 */
5911 if (ibd_find_bgroup(state) != IBT_SUCCESS) {
5912 /* Resort to late initialization */
5913 late_hca_init = 1;
5914 goto reg_snet_notices;
5915 }
5916 state->id_mac_state |= IBD_DRV_BCAST_GROUP_FOUND;
5917
5918 /*
5919 * Initialize per-interface caches and lists; if we fail here,
5920 * it is most likely due to a lack of resources
5921 */
5922 if (ibd_acache_init(state) != DDI_SUCCESS) {
5923 DPRINT(10, "ibd_start: ibd_acache_init() failed");
5924 err = ENOMEM;
5925 goto start_fail;
5926 }
5927 state->id_mac_state |= IBD_DRV_ACACHE_INITIALIZED;
5928
5929 /*
5930 * Allocate send and receive completion queues
5931 */
5932 if (ibd_alloc_cqs(state) != DDI_SUCCESS) {
5933 DPRINT(10, "ibd_start: ibd_alloc_cqs() failed");
5934 err = ENOMEM;
5935 goto start_fail;
5936 }
5937 state->id_mac_state |= IBD_DRV_CQS_ALLOCD;
5938
5939 /*
5940 * Setup a UD channel
5941 */
5942 if (ibd_setup_ud_channel(state) != DDI_SUCCESS) {
5943 err = ENOMEM;
5944 DPRINT(10, "ibd_start: ibd_setup_ud_channel() failed");
5945 goto start_fail;
5946 }
5947 state->id_mac_state |= IBD_DRV_UD_CHANNEL_SETUP;
5948
5949 /*
5950 * Allocate and initialize the tx buffer list
5951 */
5952 if (ibd_init_txlist(state) != DDI_SUCCESS) {
5953 DPRINT(10, "ibd_start: ibd_init_txlist() failed");
5954 err = ENOMEM;
5955 goto start_fail;
5956 }
5957 state->id_mac_state |= IBD_DRV_TXLIST_ALLOCD;
5958
5959 /*
5960 * Create the send cq handler here
5961 */
5962 ibt_set_cq_handler(state->id_scq_hdl, ibd_scq_handler, state);
5963 if ((ret = ibt_enable_cq_notify(state->id_scq_hdl,
5964 IBT_NEXT_COMPLETION)) != IBT_SUCCESS) {
5965 DPRINT(10, "ibd_start: ibt_enable_cq_notify(scq) "
5966 "failed, ret=%d", ret);
5967 err = EINVAL;
5968 goto start_fail;
5969 }
5970 state->id_mac_state |= IBD_DRV_SCQ_NOTIFY_ENABLED;
5971
5972 /*
5973 * Allocate and initialize the rx buffer list
5974 */
5975 if (ibd_init_rxlist(state) != DDI_SUCCESS) {
5976 DPRINT(10, "ibd_start: ibd_init_rxlist() failed");
5977 err = ENOMEM;
5978 goto start_fail;
5979 }
5980 state->id_mac_state |= IBD_DRV_RXLIST_ALLOCD;
5981
5982 /*
5983 * Join IPoIB broadcast group
5984 */
5985 if (ibd_join_group(state, state->id_mgid, IB_MC_JSTATE_FULL) == NULL) {
5986 DPRINT(10, "ibd_start: ibd_join_group() failed");
5987 err = ENOTACTIVE;
5988 goto start_fail;
5989 }
5990 state->id_mac_state |= IBD_DRV_BCAST_GROUP_JOINED;
5991
5992 /*
5993 * When we did mac_register() in ibd_attach(), we didn't register
5994 * the real macaddr and we didn't have the true port mtu. Now that
5995 * we're almost ready, set the local mac address and broadcast
5996 * addresses and update gldv3 about the real values of these
5997 * parameters.
5998 */
5999 if (state->id_enable_rc) {
6000 ibd_h2n_mac(&state->id_macaddr,
6001 IBD_MAC_ADDR_RC + state->id_qpnum,
6002 state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
6003 ibd_h2n_mac(&state->rc_macaddr_loopback, state->id_qpnum,
6004 state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
6005 } else {
6006 ibd_h2n_mac(&state->id_macaddr, state->id_qpnum,
6007 state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
6008 }
6009 ibd_h2n_mac(&state->id_bcaddr, IB_QPN_MASK,
6010 state->id_mgid.gid_prefix, state->id_mgid.gid_guid);
6011
6012 if (!state->id_enable_rc) {
6013 (void) mac_maxsdu_update2(state->id_mh,
6014 state->id_mtu - IPOIB_HDRSIZE,
6015 state->id_mtu - IPOIB_HDRSIZE);
6016 }
6017 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr);
6018
6019 /*
6020 * Setup the receive cq handler
6021 */
6022 ibt_set_cq_handler(state->id_rcq_hdl, ibd_rcq_handler, state);
6023 if ((ret = ibt_enable_cq_notify(state->id_rcq_hdl,
6024 IBT_NEXT_COMPLETION)) != IBT_SUCCESS) {
6025 DPRINT(10, "ibd_start: ibt_enable_cq_notify(rcq) "
6026 "failed, ret=%d", ret);
6027 err = EINVAL;
6028 goto start_fail;
6029 }
6030 state->id_mac_state |= IBD_DRV_RCQ_NOTIFY_ENABLED;
6031
6032 reg_snet_notices:
6033 /*
6034 * In case of normal initialization sequence,
6035 * Setup the subnet notices handler after we've initialized the acache/
6036 * mcache and started the async thread, both of which are required for
6037 * the trap handler to function properly.
6038 *
6039 * Now that the async thread has been started (and we've already done
6040 * a mac_register() during attach so mac_tx_update() can be called
6041 * if necessary without any problem), we can enable the trap handler
6042 * to queue requests to the async thread.
6043 *
6044 * In case of late hca initialization, the subnet notices handler will
6045 * only handle MCG created/deleted event. The action performed as part
6046 * of handling these events is to start the interface. So, the
6047 * acache/mcache initialization is not a necessity in such cases for
6048 * registering the subnet notices handler. Also, if we are in
6049 * ibd_start() as a result of, say, some event handling after entering
6050 * late hca initialization phase no need to register again.
6051 */
6052 if ((state->id_mac_state & IBD_DRV_SM_NOTICES_REGISTERED) == 0) {
6053 ibt_register_subnet_notices(state->id_ibt_hdl,
6054 ibd_snet_notices_handler, state);
6055 mutex_enter(&state->id_trap_lock);
6056 state->id_trap_stop = B_FALSE;
6057 mutex_exit(&state->id_trap_lock);
6058 state->id_mac_state |= IBD_DRV_SM_NOTICES_REGISTERED;
6059 }
6060
6061 late_hca_init_return:
6062 if (late_hca_init == 1) {
6063 state->id_mac_state |= IBD_DRV_IN_LATE_HCA_INIT;
6064 /*
6065 * In case of late initialization, mark the link state as down,
6066 * immaterial of the actual link state as reported in the
6067 * port_info.
6068 */
6069 state->id_link_state = LINK_STATE_DOWN;
6070 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr);
6071 mac_link_update(state->id_mh, state->id_link_state);
6072 return (DDI_SUCCESS);
6073 }
6074
6075 if (state->id_enable_rc) {
6076 if (state->rc_enable_srq) {
6077 if (state->id_mac_state & IBD_DRV_RC_SRQ_ALLOCD) {
6078 if (ibd_rc_repost_srq_free_list(state) !=
6079 IBT_SUCCESS) {
6080 err = ENOMEM;
6081 goto start_fail;
6082 }
6083 } else {
6084 /* Allocate SRQ resource */
6085 if (ibd_rc_init_srq_list(state) !=
6086 IBT_SUCCESS) {
6087 err = ENOMEM;
6088 goto start_fail;
6089 }
6090 state->id_mac_state |= IBD_DRV_RC_SRQ_ALLOCD;
6091 }
6092 }
6093
6094 if (ibd_rc_init_tx_largebuf_list(state) != IBT_SUCCESS) {
6095 DPRINT(10, "ibd_start: ibd_rc_init_tx_largebuf_list() "
6096 "failed");
6097 err = ENOMEM;
6098 goto start_fail;
6099 }
6100 state->id_mac_state |= IBD_DRV_RC_LARGEBUF_ALLOCD;
6101
6102 /* RC: begin to listen only after everything is available */
6103 if (ibd_rc_listen(state) != IBT_SUCCESS) {
6104 DPRINT(10, "ibd_start: ibd_rc_listen() failed");
6105 err = EINVAL;
6106 goto start_fail;
6107 }
6108 state->id_mac_state |= IBD_DRV_RC_LISTEN;
6109 }
6110
6111 /*
6112 * Indicate link status to GLDv3 and higher layers. By default,
6113 * we assume we are in up state (which must have been true at
6114 * least at the time the broadcast mcg's were probed); if there
6115 * were any up/down transitions till the time we come here, the
6116 * async handler will have updated last known state, which we
6117 * use to tell GLDv3. The async handler will not send any
6118 * notifications to GLDv3 till we reach here in the initialization
6119 * sequence.
6120 */
6121 mac_link_update(state->id_mh, state->id_link_state);
6122 state->id_mac_state &= ~IBD_DRV_IN_LATE_HCA_INIT;
6123 state->id_mac_state |= IBD_DRV_STARTED;
6124
6125 /* Start timer after everything is ready */
6126 if (state->id_enable_rc) {
6127 mutex_enter(&state->rc_timeout_lock);
6128 state->rc_timeout_start = B_TRUE;
6129 state->rc_timeout = timeout(ibd_rc_conn_timeout_call, state,
6130 SEC_TO_TICK(ibd_rc_conn_timeout));
6131 mutex_exit(&state->rc_timeout_lock);
6132 state->id_mac_state |= IBD_DRV_RC_TIMEOUT;
6133 }
6134
6135 return (DDI_SUCCESS);
6136
6137 start_fail:
6138 /*
6139 * If we ran into a problem during ibd_start() and ran into
6140 * some other problem during undoing our partial work, we can't
6141 * do anything about it. Ignore any errors we might get from
6142 * ibd_undo_start() and just return the original error we got.
6143 */
6144 (void) ibd_undo_start(state, LINK_STATE_DOWN);
6145 return (err);
6146 }
6147
6148 /*
6149 * GLDv3 entry point to stop hardware from receiving packets.
6150 */
6151 /*ARGSUSED*/
6152 static void
6153 ibd_m_stop(void *arg)
6154 {
6155 ibd_state_t *state = (ibd_state_t *)arg;
6156
6157 if (state->id_type == IBD_PORT_DRIVER)
6158 return;
6159
6160 ibd_set_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS);
6161
6162 (void) ibd_undo_start(state, state->id_link_state);
6163
6164 ibd_clr_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS);
6165 }
6166
6167 /*
6168 * GLDv3 entry point to modify device's mac address. We do not
6169 * allow address modifications.
6170 */
6171 static int
6172 ibd_m_unicst(void *arg, const uint8_t *macaddr)
6173 {
6174 ibd_state_t *state = arg;
6175
6176 if (state->id_type == IBD_PORT_DRIVER)
6177 return (EINVAL);
6178
6179 /*
6180 * Don't bother even comparing the macaddr if we haven't
6181 * completed ibd_m_start().
6182 */
6183 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6184 return (0);
6185
6186 if (bcmp(macaddr, &state->id_macaddr, IPOIB_ADDRL) == 0)
6187 return (0);
6188 else
6189 return (EINVAL);
6190 }
6191
6192 /*
6193 * The blocking part of the IBA join/leave operations are done out
6194 * of here on the async thread.
6195 */
6196 static void
6197 ibd_async_multicast(ibd_state_t *state, ib_gid_t mgid, int op)
6198 {
6199 DPRINT(3, "ibd_async_multicast : async_setmc op %d :"
6200 "%016llx:%016llx\n", op, mgid.gid_prefix, mgid.gid_guid);
6201
6202 if (op == IBD_ASYNC_JOIN) {
6203 if (ibd_join_group(state, mgid, IB_MC_JSTATE_FULL) == NULL) {
6204 ibd_print_warn(state, "Join multicast group failed :"
6205 "%016llx:%016llx", mgid.gid_prefix, mgid.gid_guid);
6206 }
6207 } else {
6208 /*
6209 * Here, we must search for the proper mcg_info and
6210 * use that to leave the group.
6211 */
6212 ibd_leave_group(state, mgid, IB_MC_JSTATE_FULL);
6213 }
6214 }
6215
6216 /*
6217 * GLDv3 entry point for multicast enable/disable requests.
6218 * This function queues the operation to the async thread and
6219 * return success for a valid multicast address.
6220 */
6221 static int
6222 ibd_m_multicst(void *arg, boolean_t add, const uint8_t *mcmac)
6223 {
6224 ibd_state_t *state = (ibd_state_t *)arg;
6225 ipoib_mac_t maddr, *mcast;
6226 ib_gid_t mgid;
6227 ibd_req_t *req;
6228
6229 if (state->id_type == IBD_PORT_DRIVER)
6230 return (EINVAL);
6231
6232 /*
6233 * If we haven't completed ibd_m_start(), async thread wouldn't
6234 * have been started and id_bcaddr wouldn't be set, so there's
6235 * no point in continuing.
6236 */
6237 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6238 return (0);
6239
6240 /*
6241 * The incoming multicast address might not be aligned properly
6242 * on a 4 byte boundary to be considered an ipoib_mac_t. We force
6243 * it to look like one though, to get the offsets of the mc gid,
6244 * since we know we are not going to dereference any values with
6245 * the ipoib_mac_t pointer.
6246 */
6247 bcopy(mcmac, &maddr, sizeof (ipoib_mac_t));
6248 mcast = &maddr;
6249
6250 /*
6251 * Check validity of MCG address. We could additionally check
6252 * that a enable/disable is not being issued on the "broadcast"
6253 * mcg, but since this operation is only invokable by privileged
6254 * programs anyway, we allow the flexibility to those dlpi apps.
6255 * Note that we do not validate the "scope" of the IBA mcg.
6256 */
6257 if ((ntohl(mcast->ipoib_qpn) & IB_QPN_MASK) != IB_MC_QPN)
6258 return (EINVAL);
6259
6260 /*
6261 * fill in multicast pkey and scope
6262 */
6263 IBD_FILL_SCOPE_PKEY(mcast, state->id_scope, state->id_pkey);
6264
6265 /*
6266 * If someone is trying to JOIN/LEAVE the broadcast group, we do
6267 * nothing (i.e. we stay JOINed to the broadcast group done in
6268 * ibd_m_start()), to mimic ethernet behavior. IPv4 specifically
6269 * requires to be joined to broadcast groups at all times.
6270 * ibd_join_group() has an ASSERT(omce->mc_fullreap) that also
6271 * depends on this.
6272 */
6273 if (bcmp(mcast, &state->id_bcaddr, IPOIB_ADDRL) == 0)
6274 return (0);
6275
6276 ibd_n2h_gid(mcast, &mgid);
6277 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
6278 if (req == NULL)
6279 return (ENOMEM);
6280
6281 req->rq_gid = mgid;
6282
6283 if (add) {
6284 DPRINT(1, "ibd_m_multicst : %016llx:%016llx\n",
6285 mgid.gid_prefix, mgid.gid_guid);
6286 ibd_queue_work_slot(state, req, IBD_ASYNC_JOIN);
6287 } else {
6288 DPRINT(1, "ibd_m_multicst : unset_multicast : "
6289 "%016llx:%016llx", mgid.gid_prefix, mgid.gid_guid);
6290 ibd_queue_work_slot(state, req, IBD_ASYNC_LEAVE);
6291 }
6292 return (0);
6293 }
6294
6295 /*
6296 * The blocking part of the IBA promiscuous operations are done
6297 * out of here on the async thread. The dlpireq parameter indicates
6298 * whether this invocation is due to a dlpi request or due to
6299 * a port up/down event.
6300 */
6301 static void
6302 ibd_async_unsetprom(ibd_state_t *state)
6303 {
6304 ibd_mce_t *mce = list_head(&state->id_mc_non);
6305 ib_gid_t mgid;
6306
6307 DPRINT(2, "ibd_async_unsetprom : async_unset_promisc");
6308
6309 while (mce != NULL) {
6310 mgid = mce->mc_info.mc_adds_vect.av_dgid;
6311 mce = list_next(&state->id_mc_non, mce);
6312 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON);
6313 }
6314 state->id_prom_op = IBD_OP_NOTSTARTED;
6315 }
6316
6317 /*
6318 * The blocking part of the IBA promiscuous operations are done
6319 * out of here on the async thread. The dlpireq parameter indicates
6320 * whether this invocation is due to a dlpi request or due to
6321 * a port up/down event.
6322 */
6323 static void
6324 ibd_async_setprom(ibd_state_t *state)
6325 {
6326 ibt_mcg_attr_t mcg_attr;
6327 ibt_mcg_info_t *mcg_info;
6328 ib_gid_t mgid;
6329 uint_t numg;
6330 int i;
6331 char ret = IBD_OP_COMPLETED;
6332
6333 DPRINT(2, "ibd_async_setprom : async_set_promisc");
6334
6335 /*
6336 * Obtain all active MC groups on the IB fabric with
6337 * specified criteria (scope + Pkey + Qkey + mtu).
6338 */
6339 bzero(&mcg_attr, sizeof (mcg_attr));
6340 mcg_attr.mc_pkey = state->id_pkey;
6341 mcg_attr.mc_scope = state->id_scope;
6342 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
6343 mcg_attr.mc_mtu_req.r_mtu = state->id_mcinfo->mc_mtu;
6344 mcg_attr.mc_mtu_req.r_selector = IBT_EQU;
6345 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 0, &mcg_info, &numg) !=
6346 IBT_SUCCESS) {
6347 ibd_print_warn(state, "Could not get list of IBA multicast "
6348 "groups");
6349 ret = IBD_OP_ERRORED;
6350 goto done;
6351 }
6352
6353 /*
6354 * Iterate over the returned mcg's and join as NonMember
6355 * to the IP mcg's.
6356 */
6357 for (i = 0; i < numg; i++) {
6358 /*
6359 * Do a NonMember JOIN on the MC group.
6360 */
6361 mgid = mcg_info[i].mc_adds_vect.av_dgid;
6362 if (ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == NULL)
6363 ibd_print_warn(state, "IBA promiscuous mode missed "
6364 "multicast gid %016llx:%016llx",
6365 (u_longlong_t)mgid.gid_prefix,
6366 (u_longlong_t)mgid.gid_guid);
6367 }
6368
6369 ibt_free_mcg_info(mcg_info, numg);
6370 DPRINT(4, "ibd_async_setprom : async_set_promisc completes");
6371 done:
6372 state->id_prom_op = ret;
6373 }
6374
6375 /*
6376 * GLDv3 entry point for multicast promiscuous enable/disable requests.
6377 * GLDv3 assumes phys state receives more packets than multi state,
6378 * which is not true for IPoIB. Thus, treat the multi and phys
6379 * promiscuous states the same way to work with GLDv3's assumption.
6380 */
6381 static int
6382 ibd_m_promisc(void *arg, boolean_t on)
6383 {
6384 ibd_state_t *state = (ibd_state_t *)arg;
6385 ibd_req_t *req;
6386
6387 if (state->id_type == IBD_PORT_DRIVER)
6388 return (EINVAL);
6389
6390 /*
6391 * Async thread wouldn't have been started if we haven't
6392 * passed ibd_m_start()
6393 */
6394 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6395 return (0);
6396
6397 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
6398 if (req == NULL)
6399 return (ENOMEM);
6400 if (on) {
6401 DPRINT(1, "ibd_m_promisc : set_promisc : %d", on);
6402 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMON);
6403 } else {
6404 DPRINT(1, "ibd_m_promisc : unset_promisc");
6405 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMOFF);
6406 }
6407
6408 return (0);
6409 }
6410
6411 /*
6412 * GLDv3 entry point for gathering statistics.
6413 */
6414 static int
6415 ibd_m_stat(void *arg, uint_t stat, uint64_t *val)
6416 {
6417 ibd_state_t *state = (ibd_state_t *)arg;
6418
6419 switch (stat) {
6420 case MAC_STAT_IFSPEED:
6421 *val = state->id_link_speed;
6422 break;
6423 case MAC_STAT_MULTIRCV:
6424 *val = state->id_multi_rcv;
6425 break;
6426 case MAC_STAT_BRDCSTRCV:
6427 *val = state->id_brd_rcv;
6428 break;
6429 case MAC_STAT_MULTIXMT:
6430 *val = state->id_multi_xmt;
6431 break;
6432 case MAC_STAT_BRDCSTXMT:
6433 *val = state->id_brd_xmt;
6434 break;
6435 case MAC_STAT_RBYTES:
6436 *val = state->id_rcv_bytes + state->rc_rcv_trans_byte
6437 + state->rc_rcv_copy_byte;
6438 break;
6439 case MAC_STAT_IPACKETS:
6440 *val = state->id_rcv_pkt + state->rc_rcv_trans_pkt
6441 + state->rc_rcv_copy_pkt;
6442 break;
6443 case MAC_STAT_OBYTES:
6444 *val = state->id_xmt_bytes + state->rc_xmt_bytes;
6445 break;
6446 case MAC_STAT_OPACKETS:
6447 *val = state->id_xmt_pkt + state->rc_xmt_small_pkt +
6448 state->rc_xmt_fragmented_pkt +
6449 state->rc_xmt_map_fail_pkt + state->rc_xmt_map_succ_pkt;
6450 break;
6451 case MAC_STAT_OERRORS:
6452 *val = state->id_ah_error; /* failed AH translation */
6453 break;
6454 case MAC_STAT_IERRORS:
6455 *val = 0;
6456 break;
6457 case MAC_STAT_NOXMTBUF:
6458 *val = state->id_tx_short + state->rc_swqe_short +
6459 state->rc_xmt_buf_short;
6460 break;
6461 case MAC_STAT_NORCVBUF:
6462 default:
6463 return (ENOTSUP);
6464 }
6465
6466 return (0);
6467 }
6468
6469 static void
6470 ibd_async_txsched(ibd_state_t *state)
6471 {
6472 ibd_resume_transmission(state);
6473 }
6474
6475 static void
6476 ibd_resume_transmission(ibd_state_t *state)
6477 {
6478 int flag;
6479 int met_thresh = 0;
6480 int thresh = 0;
6481 int ret = -1;
6482
6483 mutex_enter(&state->id_sched_lock);
6484 if (state->id_sched_needed & IBD_RSRC_SWQE) {
6485 mutex_enter(&state->id_tx_list.dl_mutex);
6486 mutex_enter(&state->id_tx_rel_list.dl_mutex);
6487 met_thresh = state->id_tx_list.dl_cnt +
6488 state->id_tx_rel_list.dl_cnt;
6489 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6490 mutex_exit(&state->id_tx_list.dl_mutex);
6491 thresh = IBD_FREE_SWQES_THRESH;
6492 flag = IBD_RSRC_SWQE;
6493 } else if (state->id_sched_needed & IBD_RSRC_LSOBUF) {
6494 ASSERT(state->id_lso != NULL);
6495 mutex_enter(&state->id_lso_lock);
6496 met_thresh = state->id_lso->bkt_nfree;
6497 thresh = IBD_FREE_LSOS_THRESH;
6498 mutex_exit(&state->id_lso_lock);
6499 flag = IBD_RSRC_LSOBUF;
6500 if (met_thresh > thresh)
6501 state->id_sched_lso_cnt++;
6502 }
6503 if (met_thresh > thresh) {
6504 state->id_sched_needed &= ~flag;
6505 state->id_sched_cnt++;
6506 ret = 0;
6507 }
6508 mutex_exit(&state->id_sched_lock);
6509
6510 if (ret == 0)
6511 mac_tx_update(state->id_mh);
6512 }
6513
6514 /*
6515 * Release the send wqe back into free list.
6516 */
6517 static void
6518 ibd_release_swqe(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail, int n)
6519 {
6520 /*
6521 * Add back on Tx list for reuse.
6522 */
6523 ASSERT(tail->swqe_next == NULL);
6524 mutex_enter(&state->id_tx_rel_list.dl_mutex);
6525 state->id_tx_rel_list.dl_pending_sends = B_FALSE;
6526 tail->swqe_next = state->id_tx_rel_list.dl_head;
6527 state->id_tx_rel_list.dl_head = SWQE_TO_WQE(head);
6528 state->id_tx_rel_list.dl_cnt += n;
6529 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6530 }
6531
6532 /*
6533 * Acquire a send wqe from free list.
6534 * Returns error number and send wqe pointer.
6535 */
6536 static ibd_swqe_t *
6537 ibd_acquire_swqe(ibd_state_t *state)
6538 {
6539 ibd_swqe_t *wqe;
6540
6541 mutex_enter(&state->id_tx_rel_list.dl_mutex);
6542 if (state->id_tx_rel_list.dl_head != NULL) {
6543 /* transfer id_tx_rel_list to id_tx_list */
6544 state->id_tx_list.dl_head =
6545 state->id_tx_rel_list.dl_head;
6546 state->id_tx_list.dl_cnt =
6547 state->id_tx_rel_list.dl_cnt;
6548 state->id_tx_list.dl_pending_sends = B_FALSE;
6549
6550 /* clear id_tx_rel_list */
6551 state->id_tx_rel_list.dl_head = NULL;
6552 state->id_tx_rel_list.dl_cnt = 0;
6553 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6554
6555 wqe = WQE_TO_SWQE(state->id_tx_list.dl_head);
6556 state->id_tx_list.dl_cnt -= 1;
6557 state->id_tx_list.dl_head = wqe->swqe_next;
6558 } else { /* no free swqe */
6559 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6560 state->id_tx_list.dl_pending_sends = B_TRUE;
6561 DPRINT(5, "ibd_acquire_swqe: out of Tx wqe");
6562 state->id_tx_short++;
6563 wqe = NULL;
6564 }
6565 return (wqe);
6566 }
6567
6568 static int
6569 ibd_setup_lso(ibd_swqe_t *node, mblk_t *mp, uint32_t mss,
6570 ibt_ud_dest_hdl_t ud_dest)
6571 {
6572 mblk_t *nmp;
6573 int iph_len, tcph_len;
6574 ibt_wr_lso_t *lso;
6575 uintptr_t ip_start, tcp_start;
6576 uint8_t *dst;
6577 uint_t pending, mblen;
6578
6579 /*
6580 * The code in ibd_send would've set 'wr.ud.udwr_dest' by default;
6581 * we need to adjust it here for lso.
6582 */
6583 lso = &(node->w_swr.wr.ud_lso);
6584 lso->lso_ud_dest = ud_dest;
6585 lso->lso_mss = mss;
6586
6587 /*
6588 * Calculate the LSO header size and set it in the UD LSO structure.
6589 * Note that the only assumption we make is that each of the IPoIB,
6590 * IP and TCP headers will be contained in a single mblk fragment;
6591 * together, the headers may span multiple mblk fragments.
6592 */
6593 nmp = mp;
6594 ip_start = (uintptr_t)(nmp->b_rptr) + IPOIB_HDRSIZE;
6595 if (ip_start >= (uintptr_t)(nmp->b_wptr)) {
6596 ip_start = (uintptr_t)nmp->b_cont->b_rptr
6597 + (ip_start - (uintptr_t)(nmp->b_wptr));
6598 nmp = nmp->b_cont;
6599
6600 }
6601 iph_len = IPH_HDR_LENGTH((ipha_t *)ip_start);
6602
6603 tcp_start = ip_start + iph_len;
6604 if (tcp_start >= (uintptr_t)(nmp->b_wptr)) {
6605 tcp_start = (uintptr_t)nmp->b_cont->b_rptr
6606 + (tcp_start - (uintptr_t)(nmp->b_wptr));
6607 nmp = nmp->b_cont;
6608 }
6609 tcph_len = TCP_HDR_LENGTH((tcph_t *)tcp_start);
6610 lso->lso_hdr_sz = IPOIB_HDRSIZE + iph_len + tcph_len;
6611
6612 /*
6613 * If the lso header fits entirely within a single mblk fragment,
6614 * we'll avoid an additional copy of the lso header here and just
6615 * pass the b_rptr of the mblk directly.
6616 *
6617 * If this isn't true, we'd have to allocate for it explicitly.
6618 */
6619 if (lso->lso_hdr_sz <= MBLKL(mp)) {
6620 lso->lso_hdr = mp->b_rptr;
6621 } else {
6622 /* On work completion, remember to free this allocated hdr */
6623 lso->lso_hdr = kmem_zalloc(lso->lso_hdr_sz, KM_NOSLEEP);
6624 if (lso->lso_hdr == NULL) {
6625 DPRINT(10, "ibd_setup_lso: couldn't allocate lso hdr, "
6626 "sz = %d", lso->lso_hdr_sz);
6627 lso->lso_hdr_sz = 0;
6628 lso->lso_mss = 0;
6629 return (-1);
6630 }
6631 }
6632
6633 /*
6634 * Copy in the lso header only if we need to
6635 */
6636 if (lso->lso_hdr != mp->b_rptr) {
6637 dst = lso->lso_hdr;
6638 pending = lso->lso_hdr_sz;
6639
6640 for (nmp = mp; nmp && pending; nmp = nmp->b_cont) {
6641 mblen = MBLKL(nmp);
6642 if (pending > mblen) {
6643 bcopy(nmp->b_rptr, dst, mblen);
6644 dst += mblen;
6645 pending -= mblen;
6646 } else {
6647 bcopy(nmp->b_rptr, dst, pending);
6648 break;
6649 }
6650 }
6651 }
6652
6653 return (0);
6654 }
6655
6656 static void
6657 ibd_free_lsohdr(ibd_swqe_t *node, mblk_t *mp)
6658 {
6659 ibt_wr_lso_t *lso;
6660
6661 if ((!node) || (!mp))
6662 return;
6663
6664 /*
6665 * Free any header space that we might've allocated if we
6666 * did an LSO
6667 */
6668 if (node->w_swr.wr_opcode == IBT_WRC_SEND_LSO) {
6669 lso = &(node->w_swr.wr.ud_lso);
6670 if ((lso->lso_hdr) && (lso->lso_hdr != mp->b_rptr)) {
6671 kmem_free(lso->lso_hdr, lso->lso_hdr_sz);
6672 lso->lso_hdr = NULL;
6673 lso->lso_hdr_sz = 0;
6674 }
6675 }
6676 }
6677
6678 static void
6679 ibd_post_send(ibd_state_t *state, ibd_swqe_t *node)
6680 {
6681 uint_t i;
6682 uint_t num_posted;
6683 uint_t n_wrs;
6684 ibt_status_t ibt_status;
6685 ibt_send_wr_t wrs[IBD_MAX_TX_POST_MULTIPLE];
6686 ibd_swqe_t *tx_head, *elem;
6687 ibd_swqe_t *nodes[IBD_MAX_TX_POST_MULTIPLE];
6688
6689 /* post the one request, then check for more */
6690 ibt_status = ibt_post_send(state->id_chnl_hdl,
6691 &node->w_swr, 1, NULL);
6692 if (ibt_status != IBT_SUCCESS) {
6693 ibd_print_warn(state, "ibd_post_send: "
6694 "posting one wr failed: ret=%d", ibt_status);
6695 ibd_tx_cleanup(state, node);
6696 }
6697
6698 tx_head = NULL;
6699 for (;;) {
6700 if (tx_head == NULL) {
6701 mutex_enter(&state->id_txpost_lock);
6702 tx_head = state->id_tx_head;
6703 if (tx_head == NULL) {
6704 state->id_tx_busy = 0;
6705 mutex_exit(&state->id_txpost_lock);
6706 return;
6707 }
6708 state->id_tx_head = NULL;
6709 mutex_exit(&state->id_txpost_lock);
6710 }
6711
6712 /*
6713 * Collect pending requests, IBD_MAX_TX_POST_MULTIPLE wrs
6714 * at a time if possible, and keep posting them.
6715 */
6716 for (n_wrs = 0, elem = tx_head;
6717 (elem) && (n_wrs < IBD_MAX_TX_POST_MULTIPLE);
6718 elem = WQE_TO_SWQE(elem->swqe_next), n_wrs++) {
6719 nodes[n_wrs] = elem;
6720 wrs[n_wrs] = elem->w_swr;
6721 }
6722 tx_head = elem;
6723
6724 ASSERT(n_wrs != 0);
6725
6726 /*
6727 * If posting fails for some reason, we'll never receive
6728 * completion intimation, so we'll need to cleanup. But
6729 * we need to make sure we don't clean up nodes whose
6730 * wrs have been successfully posted. We assume that the
6731 * hca driver returns on the first failure to post and
6732 * therefore the first 'num_posted' entries don't need
6733 * cleanup here.
6734 */
6735 num_posted = 0;
6736 ibt_status = ibt_post_send(state->id_chnl_hdl,
6737 wrs, n_wrs, &num_posted);
6738 if (ibt_status != IBT_SUCCESS) {
6739 ibd_print_warn(state, "ibd_post_send: "
6740 "posting multiple wrs failed: "
6741 "requested=%d, done=%d, ret=%d",
6742 n_wrs, num_posted, ibt_status);
6743
6744 for (i = num_posted; i < n_wrs; i++)
6745 ibd_tx_cleanup(state, nodes[i]);
6746 }
6747 }
6748 }
6749
6750 static int
6751 ibd_prepare_sgl(ibd_state_t *state, mblk_t *mp, ibd_swqe_t *node,
6752 uint_t lsohdr_sz)
6753 {
6754 ibt_wr_ds_t *sgl;
6755 ibt_status_t ibt_status;
6756 mblk_t *nmp;
6757 mblk_t *data_mp;
6758 uchar_t *bufp;
6759 size_t blksize;
6760 size_t skip;
6761 size_t avail;
6762 uint_t pktsize;
6763 uint_t frag_len;
6764 uint_t pending_hdr;
6765 int nmblks;
6766 int i;
6767
6768 /*
6769 * Let's skip ahead to the data if this is LSO
6770 */
6771 data_mp = mp;
6772 pending_hdr = 0;
6773 if (lsohdr_sz) {
6774 pending_hdr = lsohdr_sz;
6775 for (nmp = mp; nmp; nmp = nmp->b_cont) {
6776 frag_len = nmp->b_wptr - nmp->b_rptr;
6777 if (frag_len > pending_hdr)
6778 break;
6779 pending_hdr -= frag_len;
6780 }
6781 data_mp = nmp; /* start of data past lso header */
6782 ASSERT(data_mp != NULL);
6783 }
6784
6785 /*
6786 * Calculate the size of message data and number of msg blocks
6787 */
6788 pktsize = 0;
6789 for (nmblks = 0, nmp = data_mp; nmp != NULL;
6790 nmp = nmp->b_cont, nmblks++) {
6791 pktsize += MBLKL(nmp);
6792 }
6793 pktsize -= pending_hdr;
6794
6795 /*
6796 * We only do ibt_map_mem_iov() if the pktsize is above the
6797 * "copy-threshold", and if the number of mp fragments is less than
6798 * the maximum acceptable.
6799 */
6800 if ((state->id_hca_res_lkey_capab) &&
6801 (pktsize > state->id_ud_tx_copy_thresh) &&
6802 (nmblks < state->id_max_sqseg_hiwm)) {
6803 ibt_iov_t iov_arr[IBD_MAX_SQSEG];
6804 ibt_iov_attr_t iov_attr;
6805
6806 iov_attr.iov_as = NULL;
6807 iov_attr.iov = iov_arr;
6808 iov_attr.iov_buf = NULL;
6809 iov_attr.iov_list_len = nmblks;
6810 iov_attr.iov_wr_nds = state->id_max_sqseg;
6811 iov_attr.iov_lso_hdr_sz = lsohdr_sz;
6812 iov_attr.iov_flags = IBT_IOV_SLEEP;
6813
6814 for (nmp = data_mp, i = 0; i < nmblks; i++, nmp = nmp->b_cont) {
6815 iov_arr[i].iov_addr = (caddr_t)(void *)nmp->b_rptr;
6816 iov_arr[i].iov_len = MBLKL(nmp);
6817 if (i == 0) {
6818 iov_arr[i].iov_addr += pending_hdr;
6819 iov_arr[i].iov_len -= pending_hdr;
6820 }
6821 }
6822
6823 node->w_buftype = IBD_WQE_MAPPED;
6824 node->w_swr.wr_sgl = node->w_sgl;
6825
6826 ibt_status = ibt_map_mem_iov(state->id_hca_hdl, &iov_attr,
6827 (ibt_all_wr_t *)&node->w_swr, &node->w_mi_hdl);
6828 if (ibt_status != IBT_SUCCESS) {
6829 ibd_print_warn(state, "ibd_send: ibt_map_mem_iov "
6830 "failed, nmblks=%d, ret=%d\n", nmblks, ibt_status);
6831 goto ibd_copy_path;
6832 }
6833
6834 return (0);
6835 }
6836
6837 ibd_copy_path:
6838 if (pktsize <= state->id_tx_buf_sz) {
6839 node->swqe_copybuf.ic_sgl.ds_len = pktsize;
6840 node->w_swr.wr_nds = 1;
6841 node->w_swr.wr_sgl = &node->swqe_copybuf.ic_sgl;
6842 node->w_buftype = IBD_WQE_TXBUF;
6843
6844 /*
6845 * Even though this is the copy path for transfers less than
6846 * id_tx_buf_sz, it could still be an LSO packet. If so, it
6847 * is possible the first data mblk fragment (data_mp) still
6848 * contains part of the LSO header that we need to skip.
6849 */
6850 bufp = (uchar_t *)(uintptr_t)node->w_swr.wr_sgl->ds_va;
6851 for (nmp = data_mp; nmp != NULL; nmp = nmp->b_cont) {
6852 blksize = MBLKL(nmp) - pending_hdr;
6853 bcopy(nmp->b_rptr + pending_hdr, bufp, blksize);
6854 bufp += blksize;
6855 pending_hdr = 0;
6856 }
6857
6858 return (0);
6859 }
6860
6861 /*
6862 * Copy path for transfers greater than id_tx_buf_sz
6863 */
6864 node->w_swr.wr_sgl = node->w_sgl;
6865 if (ibd_acquire_lsobufs(state, pktsize,
6866 node->w_swr.wr_sgl, &(node->w_swr.wr_nds)) != 0) {
6867 DPRINT(10, "ibd_prepare_sgl: lso bufs acquire failed");
6868 return (-1);
6869 }
6870 node->w_buftype = IBD_WQE_LSOBUF;
6871
6872 /*
6873 * Copy the larger-than-id_tx_buf_sz packet into a set of
6874 * fixed-sized, pre-mapped LSO buffers. Note that we might
6875 * need to skip part of the LSO header in the first fragment
6876 * as before.
6877 */
6878 nmp = data_mp;
6879 skip = pending_hdr;
6880 for (i = 0; i < node->w_swr.wr_nds; i++) {
6881 sgl = node->w_swr.wr_sgl + i;
6882 bufp = (uchar_t *)(uintptr_t)sgl->ds_va;
6883 avail = IBD_LSO_BUFSZ;
6884 while (nmp && avail) {
6885 blksize = MBLKL(nmp) - skip;
6886 if (blksize > avail) {
6887 bcopy(nmp->b_rptr + skip, bufp, avail);
6888 skip += avail;
6889 avail = 0;
6890 } else {
6891 bcopy(nmp->b_rptr + skip, bufp, blksize);
6892 skip = 0;
6893 avail -= blksize;
6894 bufp += blksize;
6895 nmp = nmp->b_cont;
6896 }
6897 }
6898 }
6899
6900 return (0);
6901 }
6902
6903 /*
6904 * Schedule a completion queue polling to reap the resource we're
6905 * short on. If we implement the change to reap tx completions
6906 * in a separate thread, we'll need to wake up that thread here.
6907 */
6908 static int
6909 ibd_sched_poll(ibd_state_t *state, int resource_type, int q_flag)
6910 {
6911 ibd_req_t *req;
6912
6913 mutex_enter(&state->id_sched_lock);
6914 state->id_sched_needed |= resource_type;
6915 mutex_exit(&state->id_sched_lock);
6916
6917 /*
6918 * If we are asked to queue a work entry, we need to do it
6919 */
6920 if (q_flag) {
6921 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
6922 if (req == NULL)
6923 return (-1);
6924
6925 ibd_queue_work_slot(state, req, IBD_ASYNC_SCHED);
6926 }
6927
6928 return (0);
6929 }
6930
6931 /*
6932 * The passed in packet has this format:
6933 * IPOIB_ADDRL b dest addr :: 2b sap :: 2b 0's :: data
6934 */
6935 static boolean_t
6936 ibd_send(ibd_state_t *state, mblk_t *mp)
6937 {
6938 ibd_ace_t *ace;
6939 ibd_swqe_t *node;
6940 ipoib_mac_t *dest;
6941 ib_header_info_t *ipibp;
6942 ip6_t *ip6h;
6943 uint_t pktsize;
6944 uint32_t mss;
6945 uint32_t hckflags;
6946 uint32_t lsoflags = 0;
6947 uint_t lsohdr_sz = 0;
6948 int ret, len;
6949 boolean_t dofree = B_FALSE;
6950 boolean_t rc;
6951 /* if (rc_chan == NULL) send by UD; else send by RC; */
6952 ibd_rc_chan_t *rc_chan;
6953 int nmblks;
6954 mblk_t *nmp;
6955
6956 /*
6957 * If we aren't done with the device initialization and start,
6958 * we shouldn't be here.
6959 */
6960 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6961 return (B_FALSE);
6962
6963 /*
6964 * Obtain an address handle for the destination.
6965 */
6966 ipibp = (ib_header_info_t *)mp->b_rptr;
6967 dest = (ipoib_mac_t *)&ipibp->ib_dst;
6968 if ((ntohl(dest->ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN)
6969 IBD_FILL_SCOPE_PKEY(dest, state->id_scope, state->id_pkey);
6970
6971 rc_chan = NULL;
6972 ace = ibd_acache_lookup(state, dest, &ret, 1);
6973 if (state->id_enable_rc && (ace != NULL) &&
6974 (ace->ac_mac.ipoib_qpn != htonl(IB_MC_QPN))) {
6975 if (ace->ac_chan == NULL) {
6976 state->rc_null_conn++;
6977 } else {
6978 if (ace->ac_chan->chan_state ==
6979 IBD_RC_STATE_ACT_ESTAB) {
6980 rc_chan = ace->ac_chan;
6981 rc_chan->is_used = B_TRUE;
6982 mutex_enter(&rc_chan->tx_wqe_list.dl_mutex);
6983 node = WQE_TO_SWQE(
6984 rc_chan->tx_wqe_list.dl_head);
6985 if (node != NULL) {
6986 rc_chan->tx_wqe_list.dl_cnt -= 1;
6987 rc_chan->tx_wqe_list.dl_head =
6988 node->swqe_next;
6989 } else {
6990 node = ibd_rc_acquire_swqes(rc_chan);
6991 }
6992 mutex_exit(&rc_chan->tx_wqe_list.dl_mutex);
6993
6994 if (node == NULL) {
6995 state->rc_swqe_short++;
6996 mutex_enter(&state->id_sched_lock);
6997 state->id_sched_needed |=
6998 IBD_RSRC_RC_SWQE;
6999 mutex_exit(&state->id_sched_lock);
7000 ibd_dec_ref_ace(state, ace);
7001 return (B_FALSE);
7002 }
7003 } else {
7004 state->rc_no_estab_conn++;
7005 }
7006 }
7007 }
7008
7009 if (rc_chan == NULL) {
7010 mutex_enter(&state->id_tx_list.dl_mutex);
7011 node = WQE_TO_SWQE(state->id_tx_list.dl_head);
7012 if (node != NULL) {
7013 state->id_tx_list.dl_cnt -= 1;
7014 state->id_tx_list.dl_head = node->swqe_next;
7015 } else {
7016 node = ibd_acquire_swqe(state);
7017 }
7018 mutex_exit(&state->id_tx_list.dl_mutex);
7019 if (node == NULL) {
7020 /*
7021 * If we don't have an swqe available, schedule a
7022 * transmit completion queue cleanup and hold off on
7023 * sending more packets until we have some free swqes
7024 */
7025 if (ibd_sched_poll(state, IBD_RSRC_SWQE, 0) == 0) {
7026 if (ace != NULL) {
7027 ibd_dec_ref_ace(state, ace);
7028 }
7029 return (B_FALSE);
7030 }
7031
7032 /*
7033 * If a poll cannot be scheduled, we have no choice but
7034 * to drop this packet
7035 */
7036 ibd_print_warn(state, "ibd_send: no swqe, pkt drop");
7037 if (ace != NULL) {
7038 ibd_dec_ref_ace(state, ace);
7039 }
7040 return (B_TRUE);
7041 }
7042 }
7043
7044 /*
7045 * Initialize the commonly used fields in swqe to NULL to protect
7046 * against ibd_tx_cleanup accidentally misinterpreting these on a
7047 * failure.
7048 */
7049 node->swqe_im_mblk = NULL;
7050 node->w_swr.wr_nds = 0;
7051 node->w_swr.wr_sgl = NULL;
7052 node->w_swr.wr_opcode = IBT_WRC_SEND;
7053
7054 /*
7055 * Calculate the size of message data and number of msg blocks
7056 */
7057 pktsize = 0;
7058 for (nmblks = 0, nmp = mp; nmp != NULL;
7059 nmp = nmp->b_cont, nmblks++) {
7060 pktsize += MBLKL(nmp);
7061 }
7062
7063 if (bcmp(&ipibp->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0)
7064 atomic_inc_64(&state->id_brd_xmt);
7065 else if ((ntohl(ipibp->ib_dst.ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN)
7066 atomic_inc_64(&state->id_multi_xmt);
7067
7068 if (ace != NULL) {
7069 node->w_ahandle = ace;
7070 node->w_swr.wr.ud.udwr_dest = ace->ac_dest;
7071 } else {
7072 DPRINT(5,
7073 "ibd_send: acache lookup %s for %08X:%08X:%08X:%08X:%08X",
7074 ((ret == EFAULT) ? "failed" : "queued"),
7075 htonl(dest->ipoib_qpn), htonl(dest->ipoib_gidpref[0]),
7076 htonl(dest->ipoib_gidpref[1]),
7077 htonl(dest->ipoib_gidsuff[0]),
7078 htonl(dest->ipoib_gidsuff[1]));
7079 state->rc_ace_not_found++;
7080 node->w_ahandle = NULL;
7081
7082 /*
7083 * Here if ibd_acache_lookup() returns EFAULT, it means ibd
7084 * can not find a path for the specific dest address. We
7085 * should get rid of this kind of packet. We also should get
7086 * rid of the packet if we cannot schedule a poll via the
7087 * async thread. For the normal case, ibd will return the
7088 * packet to upper layer and wait for AH creating.
7089 *
7090 * Note that we always queue a work slot entry for the async
7091 * thread when we fail AH lookup (even in intr mode); this is
7092 * due to the convoluted way the code currently looks for AH.
7093 */
7094 if (ret == EFAULT) {
7095 dofree = B_TRUE;
7096 rc = B_TRUE;
7097 } else if (ibd_sched_poll(state, IBD_RSRC_SWQE, 1) != 0) {
7098 dofree = B_TRUE;
7099 rc = B_TRUE;
7100 } else {
7101 dofree = B_FALSE;
7102 rc = B_FALSE;
7103 }
7104 goto ibd_send_fail;
7105 }
7106
7107 /*
7108 * For ND6 packets, padding is at the front of the source lladdr.
7109 * Insert the padding at front.
7110 */
7111 if (ntohs(ipibp->ipib_rhdr.ipoib_type) == ETHERTYPE_IPV6) {
7112 if (MBLKL(mp) < sizeof (ib_header_info_t) + IPV6_HDR_LEN) {
7113 if (!pullupmsg(mp, IPV6_HDR_LEN +
7114 sizeof (ib_header_info_t))) {
7115 DPRINT(10, "ibd_send: pullupmsg failure ");
7116 dofree = B_TRUE;
7117 rc = B_TRUE;
7118 goto ibd_send_fail;
7119 }
7120 ipibp = (ib_header_info_t *)mp->b_rptr;
7121 }
7122 ip6h = (ip6_t *)((uchar_t *)ipibp +
7123 sizeof (ib_header_info_t));
7124 len = ntohs(ip6h->ip6_plen);
7125 if (ip6h->ip6_nxt == IPPROTO_ICMPV6) {
7126 mblk_t *pad;
7127
7128 pad = allocb(4, 0);
7129 pad->b_wptr = (uchar_t *)pad->b_rptr + 4;
7130 linkb(mp, pad);
7131 if (MBLKL(mp) < sizeof (ib_header_info_t) +
7132 IPV6_HDR_LEN + len + 4) {
7133 if (!pullupmsg(mp, sizeof (ib_header_info_t) +
7134 IPV6_HDR_LEN + len + 4)) {
7135 DPRINT(10, "ibd_send: pullupmsg "
7136 "failure ");
7137 dofree = B_TRUE;
7138 rc = B_TRUE;
7139 goto ibd_send_fail;
7140 }
7141 ip6h = (ip6_t *)((uchar_t *)mp->b_rptr +
7142 sizeof (ib_header_info_t));
7143 }
7144
7145 /* LINTED: E_CONSTANT_CONDITION */
7146 IBD_PAD_NSNA(ip6h, len, IBD_SEND);
7147 }
7148 }
7149
7150 ASSERT(mp->b_wptr - mp->b_rptr >= sizeof (ib_addrs_t));
7151 mp->b_rptr += sizeof (ib_addrs_t);
7152 pktsize -= sizeof (ib_addrs_t);
7153
7154 if (rc_chan) { /* send in RC mode */
7155 ibt_iov_t iov_arr[IBD_MAX_SQSEG];
7156 ibt_iov_attr_t iov_attr;
7157 uint_t i;
7158 size_t blksize;
7159 uchar_t *bufp;
7160 ibd_rc_tx_largebuf_t *lbufp;
7161
7162 atomic_add_64(&state->rc_xmt_bytes, pktsize);
7163
7164 /*
7165 * Upper layer does Tx checksum, we don't need do any
7166 * checksum here.
7167 */
7168 ASSERT(node->w_swr.wr_trans == IBT_RC_SRV);
7169
7170 /*
7171 * We only do ibt_map_mem_iov() if the pktsize is above
7172 * the "copy-threshold", and if the number of mp
7173 * fragments is less than the maximum acceptable.
7174 */
7175 if (pktsize <= state->id_rc_tx_copy_thresh) {
7176 atomic_inc_64(&state->rc_xmt_small_pkt);
7177 /*
7178 * Only process unicast packet in Reliable Connected
7179 * mode.
7180 */
7181 node->swqe_copybuf.ic_sgl.ds_len = pktsize;
7182 node->w_swr.wr_nds = 1;
7183 node->w_swr.wr_sgl = &node->swqe_copybuf.ic_sgl;
7184 node->w_buftype = IBD_WQE_TXBUF;
7185
7186 bufp = (uchar_t *)(uintptr_t)node->w_swr.wr_sgl->ds_va;
7187 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
7188 blksize = MBLKL(nmp);
7189 bcopy(nmp->b_rptr, bufp, blksize);
7190 bufp += blksize;
7191 }
7192 freemsg(mp);
7193 ASSERT(node->swqe_im_mblk == NULL);
7194 } else {
7195 if ((state->rc_enable_iov_map) &&
7196 (nmblks < state->rc_max_sqseg_hiwm)) {
7197
7198 /* do ibt_map_mem_iov() */
7199 iov_attr.iov_as = NULL;
7200 iov_attr.iov = iov_arr;
7201 iov_attr.iov_buf = NULL;
7202 iov_attr.iov_wr_nds = state->rc_tx_max_sqseg;
7203 iov_attr.iov_lso_hdr_sz = 0;
7204 iov_attr.iov_flags = IBT_IOV_SLEEP;
7205
7206 i = 0;
7207 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
7208 iov_arr[i].iov_len = MBLKL(nmp);
7209 if (iov_arr[i].iov_len != 0) {
7210 iov_arr[i].iov_addr = (caddr_t)
7211 (void *)nmp->b_rptr;
7212 i++;
7213 }
7214 }
7215 iov_attr.iov_list_len = i;
7216 node->w_swr.wr_sgl = node->w_sgl;
7217
7218 ret = ibt_map_mem_iov(state->id_hca_hdl,
7219 &iov_attr, (ibt_all_wr_t *)&node->w_swr,
7220 &node->w_mi_hdl);
7221 if (ret != IBT_SUCCESS) {
7222 atomic_inc_64(
7223 &state->rc_xmt_map_fail_pkt);
7224 DPRINT(30, "ibd_send: ibt_map_mem_iov("
7225 ") failed, nmblks=%d, real_nmblks"
7226 "=%d, ret=0x%x", nmblks, i, ret);
7227 goto ibd_rc_large_copy;
7228 }
7229
7230 atomic_inc_64(&state->rc_xmt_map_succ_pkt);
7231 node->w_buftype = IBD_WQE_MAPPED;
7232 node->swqe_im_mblk = mp;
7233 } else {
7234 atomic_inc_64(&state->rc_xmt_fragmented_pkt);
7235 ibd_rc_large_copy:
7236 mutex_enter(&state->rc_tx_large_bufs_lock);
7237 if (state->rc_tx_largebuf_nfree == 0) {
7238 state->rc_xmt_buf_short++;
7239 mutex_exit
7240 (&state->rc_tx_large_bufs_lock);
7241 mutex_enter(&state->id_sched_lock);
7242 state->id_sched_needed |=
7243 IBD_RSRC_RC_TX_LARGEBUF;
7244 mutex_exit(&state->id_sched_lock);
7245 dofree = B_FALSE;
7246 rc = B_FALSE;
7247 /*
7248 * If we don't have Tx large bufs,
7249 * return failure. node->w_buftype
7250 * should not be IBD_WQE_RC_COPYBUF,
7251 * otherwise it will cause problem
7252 * in ibd_rc_tx_cleanup()
7253 */
7254 node->w_buftype = IBD_WQE_TXBUF;
7255 goto ibd_send_fail;
7256 }
7257
7258 lbufp = state->rc_tx_largebuf_free_head;
7259 ASSERT(lbufp->lb_buf != NULL);
7260 state->rc_tx_largebuf_free_head =
7261 lbufp->lb_next;
7262 lbufp->lb_next = NULL;
7263 /* Update nfree count */
7264 state->rc_tx_largebuf_nfree --;
7265 mutex_exit(&state->rc_tx_large_bufs_lock);
7266 bufp = lbufp->lb_buf;
7267 node->w_sgl[0].ds_va =
7268 (ib_vaddr_t)(uintptr_t)bufp;
7269 node->w_sgl[0].ds_key =
7270 state->rc_tx_mr_desc.md_lkey;
7271 node->w_sgl[0].ds_len = pktsize;
7272 node->w_swr.wr_sgl = node->w_sgl;
7273 node->w_swr.wr_nds = 1;
7274 node->w_buftype = IBD_WQE_RC_COPYBUF;
7275 node->w_rc_tx_largebuf = lbufp;
7276
7277 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
7278 blksize = MBLKL(nmp);
7279 if (blksize != 0) {
7280 bcopy(nmp->b_rptr, bufp,
7281 blksize);
7282 bufp += blksize;
7283 }
7284 }
7285 freemsg(mp);
7286 ASSERT(node->swqe_im_mblk == NULL);
7287 }
7288 }
7289
7290 node->swqe_next = NULL;
7291 mutex_enter(&rc_chan->tx_post_lock);
7292 if (rc_chan->tx_busy) {
7293 if (rc_chan->tx_head) {
7294 rc_chan->tx_tail->swqe_next =
7295 SWQE_TO_WQE(node);
7296 } else {
7297 rc_chan->tx_head = node;
7298 }
7299 rc_chan->tx_tail = node;
7300 mutex_exit(&rc_chan->tx_post_lock);
7301 } else {
7302 rc_chan->tx_busy = 1;
7303 mutex_exit(&rc_chan->tx_post_lock);
7304 ibd_rc_post_send(rc_chan, node);
7305 }
7306
7307 return (B_TRUE);
7308 } /* send by RC */
7309
7310 if ((state->id_enable_rc) && (pktsize > state->id_mtu)) {
7311 /*
7312 * Too long pktsize. The packet size from GLD should <=
7313 * state->id_mtu + sizeof (ib_addrs_t)
7314 */
7315 if (ace->ac_mac.ipoib_qpn != htonl(IB_MC_QPN)) {
7316 ibd_req_t *req;
7317
7318 mutex_enter(&ace->tx_too_big_mutex);
7319 if (ace->tx_too_big_ongoing) {
7320 mutex_exit(&ace->tx_too_big_mutex);
7321 state->rc_xmt_reenter_too_long_pkt++;
7322 dofree = B_TRUE;
7323 } else {
7324 ace->tx_too_big_ongoing = B_TRUE;
7325 mutex_exit(&ace->tx_too_big_mutex);
7326 state->rc_xmt_icmp_too_long_pkt++;
7327
7328 req = kmem_cache_alloc(state->id_req_kmc,
7329 KM_NOSLEEP);
7330 if (req == NULL) {
7331 ibd_print_warn(state, "ibd_send: alloc "
7332 "ibd_req_t fail");
7333 /* Drop it. */
7334 dofree = B_TRUE;
7335 } else {
7336 req->rq_ptr = mp;
7337 req->rq_ptr2 = ace;
7338 ibd_queue_work_slot(state, req,
7339 IBD_ASYNC_RC_TOO_BIG);
7340 dofree = B_FALSE;
7341 }
7342 }
7343 } else {
7344 ibd_print_warn(state, "Reliable Connected mode is on. "
7345 "Multicast packet length %d > %d is too long to "
7346 "send packet (%d > %d), drop it",
7347 pktsize, state->id_mtu);
7348 state->rc_xmt_drop_too_long_pkt++;
7349 /* Drop it. */
7350 dofree = B_TRUE;
7351 }
7352 rc = B_TRUE;
7353 goto ibd_send_fail;
7354 }
7355
7356 atomic_add_64(&state->id_xmt_bytes, pktsize);
7357 atomic_inc_64(&state->id_xmt_pkt);
7358
7359 /*
7360 * Do LSO and checksum related work here. For LSO send, adjust the
7361 * ud destination, the opcode and the LSO header information to the
7362 * work request.
7363 */
7364 mac_lso_get(mp, &mss, &lsoflags);
7365 if ((lsoflags & HW_LSO) != HW_LSO) {
7366 node->w_swr.wr_opcode = IBT_WRC_SEND;
7367 lsohdr_sz = 0;
7368 } else {
7369 if (ibd_setup_lso(node, mp, mss, ace->ac_dest) != 0) {
7370 /*
7371 * The routine can only fail if there's no memory; we
7372 * can only drop the packet if this happens
7373 */
7374 ibd_print_warn(state,
7375 "ibd_send: no memory, lso posting failed");
7376 dofree = B_TRUE;
7377 rc = B_TRUE;
7378 goto ibd_send_fail;
7379 }
7380
7381 node->w_swr.wr_opcode = IBT_WRC_SEND_LSO;
7382 lsohdr_sz = (node->w_swr.wr.ud_lso).lso_hdr_sz;
7383 }
7384
7385 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &hckflags);
7386 if ((hckflags & HCK_FULLCKSUM) == HCK_FULLCKSUM)
7387 node->w_swr.wr_flags |= IBT_WR_SEND_CKSUM;
7388 else
7389 node->w_swr.wr_flags &= ~IBT_WR_SEND_CKSUM;
7390
7391 /*
7392 * Prepare the sgl for posting; the routine can only fail if there's
7393 * no lso buf available for posting. If this is the case, we should
7394 * probably resched for lso bufs to become available and then try again.
7395 */
7396 if (ibd_prepare_sgl(state, mp, node, lsohdr_sz) != 0) {
7397 if (ibd_sched_poll(state, IBD_RSRC_LSOBUF, 1) != 0) {
7398 dofree = B_TRUE;
7399 rc = B_TRUE;
7400 } else {
7401 dofree = B_FALSE;
7402 rc = B_FALSE;
7403 }
7404 goto ibd_send_fail;
7405 }
7406 node->swqe_im_mblk = mp;
7407
7408 /*
7409 * Queue the wqe to hardware; since we can now simply queue a
7410 * post instead of doing it serially, we cannot assume anything
7411 * about the 'node' after ibd_post_send() returns.
7412 */
7413 node->swqe_next = NULL;
7414
7415 mutex_enter(&state->id_txpost_lock);
7416 if (state->id_tx_busy) {
7417 if (state->id_tx_head) {
7418 state->id_tx_tail->swqe_next =
7419 SWQE_TO_WQE(node);
7420 } else {
7421 state->id_tx_head = node;
7422 }
7423 state->id_tx_tail = node;
7424 mutex_exit(&state->id_txpost_lock);
7425 } else {
7426 state->id_tx_busy = 1;
7427 mutex_exit(&state->id_txpost_lock);
7428 ibd_post_send(state, node);
7429 }
7430
7431 return (B_TRUE);
7432
7433 ibd_send_fail:
7434 if (node && mp)
7435 ibd_free_lsohdr(node, mp);
7436
7437 if (dofree)
7438 freemsg(mp);
7439
7440 if (node != NULL) {
7441 if (rc_chan) {
7442 ibd_rc_tx_cleanup(node);
7443 } else {
7444 ibd_tx_cleanup(state, node);
7445 }
7446 }
7447
7448 return (rc);
7449 }
7450
7451 /*
7452 * GLDv3 entry point for transmitting datagram.
7453 */
7454 static mblk_t *
7455 ibd_m_tx(void *arg, mblk_t *mp)
7456 {
7457 ibd_state_t *state = (ibd_state_t *)arg;
7458 mblk_t *next;
7459
7460 if (state->id_type == IBD_PORT_DRIVER) {
7461 freemsgchain(mp);
7462 return (NULL);
7463 }
7464
7465 if ((state->id_link_state != LINK_STATE_UP) ||
7466 !(state->id_mac_state & IBD_DRV_STARTED)) {
7467 freemsgchain(mp);
7468 mp = NULL;
7469 }
7470
7471 while (mp != NULL) {
7472 next = mp->b_next;
7473 mp->b_next = NULL;
7474 if (ibd_send(state, mp) == B_FALSE) {
7475 /* Send fail */
7476 mp->b_next = next;
7477 break;
7478 }
7479 mp = next;
7480 }
7481
7482 return (mp);
7483 }
7484
7485 /*
7486 * this handles Tx and Rx completions. With separate CQs, this handles
7487 * only Rx completions.
7488 */
7489 static uint_t
7490 ibd_intr(caddr_t arg)
7491 {
7492 ibd_state_t *state = (ibd_state_t *)arg;
7493
7494 ibd_poll_rcq(state, state->id_rcq_hdl);
7495
7496 return (DDI_INTR_CLAIMED);
7497 }
7498
7499 /*
7500 * Poll and fully drain the send cq
7501 */
7502 static void
7503 ibd_drain_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
7504 {
7505 ibt_wc_t *wcs = state->id_txwcs;
7506 uint_t numwcs = state->id_txwcs_size;
7507 ibd_wqe_t *wqe;
7508 ibd_swqe_t *head, *tail;
7509 ibt_wc_t *wc;
7510 uint_t num_polled;
7511 int i;
7512
7513 while (ibt_poll_cq(cq_hdl, wcs, numwcs, &num_polled) == IBT_SUCCESS) {
7514 head = tail = NULL;
7515 for (i = 0, wc = wcs; i < num_polled; i++, wc++) {
7516 wqe = (ibd_wqe_t *)(uintptr_t)wc->wc_id;
7517 if (wc->wc_status != IBT_WC_SUCCESS) {
7518 /*
7519 * Channel being torn down.
7520 */
7521 if (wc->wc_status == IBT_WC_WR_FLUSHED_ERR) {
7522 DPRINT(5, "ibd_drain_scq: flush error");
7523 DPRINT(10, "ibd_drain_scq: Bad "
7524 "status %d", wc->wc_status);
7525 } else {
7526 DPRINT(10, "ibd_drain_scq: "
7527 "unexpected wc_status %d",
7528 wc->wc_status);
7529 }
7530 /*
7531 * Fallthrough to invoke the Tx handler to
7532 * release held resources, e.g., AH refcount.
7533 */
7534 }
7535 /*
7536 * Add this swqe to the list to be cleaned up.
7537 */
7538 if (head)
7539 tail->swqe_next = wqe;
7540 else
7541 head = WQE_TO_SWQE(wqe);
7542 tail = WQE_TO_SWQE(wqe);
7543 }
7544 tail->swqe_next = NULL;
7545 ibd_tx_cleanup_list(state, head, tail);
7546
7547 /*
7548 * Resume any blocked transmissions if possible
7549 */
7550 ibd_resume_transmission(state);
7551 }
7552 }
7553
7554 /*
7555 * Poll and fully drain the receive cq
7556 */
7557 static void
7558 ibd_drain_rcq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
7559 {
7560 ibt_wc_t *wcs = state->id_rxwcs;
7561 uint_t numwcs = state->id_rxwcs_size;
7562 ibd_rwqe_t *rwqe;
7563 ibt_wc_t *wc;
7564 uint_t num_polled;
7565 int i;
7566 mblk_t *head, *tail, *mp;
7567
7568 while (ibt_poll_cq(cq_hdl, wcs, numwcs, &num_polled) == IBT_SUCCESS) {
7569 head = tail = NULL;
7570 for (i = 0, wc = wcs; i < num_polled; i++, wc++) {
7571 rwqe = (ibd_rwqe_t *)(uintptr_t)wc->wc_id;
7572 if (wc->wc_status != IBT_WC_SUCCESS) {
7573 /*
7574 * Channel being torn down.
7575 */
7576 if (wc->wc_status == IBT_WC_WR_FLUSHED_ERR) {
7577 DPRINT(5, "ibd_drain_rcq: "
7578 "expected flushed rwqe");
7579 } else {
7580 DPRINT(5, "ibd_drain_rcq: "
7581 "unexpected wc_status %d",
7582 wc->wc_status);
7583 }
7584 atomic_inc_32(
7585 &state->id_rx_list.dl_bufs_outstanding);
7586 freemsg(rwqe->rwqe_im_mblk);
7587 continue;
7588 }
7589 mp = ibd_process_rx(state, rwqe, wc);
7590 if (mp == NULL)
7591 continue;
7592
7593 /*
7594 * Add this mp to the list to send to the nw layer.
7595 */
7596 if (head)
7597 tail->b_next = mp;
7598 else
7599 head = mp;
7600 tail = mp;
7601 }
7602 if (head)
7603 mac_rx(state->id_mh, state->id_rh, head);
7604
7605 /*
7606 * Account for #rwqes polled.
7607 * Post more here, if less than one fourth full.
7608 */
7609 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, -num_polled) <
7610 (state->id_ud_num_rwqe / 4))
7611 ibd_post_recv_intr(state);
7612 }
7613 }
7614
7615 /*
7616 * Common code for interrupt handling as well as for polling
7617 * for all completed wqe's while detaching.
7618 */
7619 static void
7620 ibd_poll_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
7621 {
7622 int flag, redo_flag;
7623 int redo = 1;
7624
7625 flag = IBD_CQ_POLLING;
7626 redo_flag = IBD_REDO_CQ_POLLING;
7627
7628 mutex_enter(&state->id_scq_poll_lock);
7629 if (state->id_scq_poll_busy & flag) {
7630 ibd_print_warn(state, "ibd_poll_scq: multiple polling threads");
7631 state->id_scq_poll_busy |= redo_flag;
7632 mutex_exit(&state->id_scq_poll_lock);
7633 return;
7634 }
7635 state->id_scq_poll_busy |= flag;
7636 mutex_exit(&state->id_scq_poll_lock);
7637
7638 /*
7639 * In some cases (eg detaching), this code can be invoked on
7640 * any cpu after disabling cq notification (thus no concurrency
7641 * exists). Apart from that, the following applies normally:
7642 * Transmit completion handling could be from any cpu if
7643 * Tx CQ is poll driven, but always on Tx interrupt cpu if Tx CQ
7644 * is interrupt driven.
7645 */
7646
7647 /*
7648 * Poll and drain the CQ
7649 */
7650 ibd_drain_scq(state, cq_hdl);
7651
7652 /*
7653 * Enable CQ notifications and redrain the cq to catch any
7654 * completions we might have missed after the ibd_drain_scq()
7655 * above and before the ibt_enable_cq_notify() that follows.
7656 * Finally, service any new requests to poll the cq that
7657 * could've come in after the ibt_enable_cq_notify().
7658 */
7659 do {
7660 if (ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION) !=
7661 IBT_SUCCESS) {
7662 DPRINT(10, "ibd_intr: ibt_enable_cq_notify() failed");
7663 }
7664
7665 ibd_drain_scq(state, cq_hdl);
7666
7667 mutex_enter(&state->id_scq_poll_lock);
7668 if (state->id_scq_poll_busy & redo_flag)
7669 state->id_scq_poll_busy &= ~redo_flag;
7670 else {
7671 state->id_scq_poll_busy &= ~flag;
7672 redo = 0;
7673 }
7674 mutex_exit(&state->id_scq_poll_lock);
7675
7676 } while (redo);
7677 }
7678
7679 /*
7680 * Common code for interrupt handling as well as for polling
7681 * for all completed wqe's while detaching.
7682 */
7683 static void
7684 ibd_poll_rcq(ibd_state_t *state, ibt_cq_hdl_t rcq)
7685 {
7686 int flag, redo_flag;
7687 int redo = 1;
7688
7689 flag = IBD_CQ_POLLING;
7690 redo_flag = IBD_REDO_CQ_POLLING;
7691
7692 mutex_enter(&state->id_rcq_poll_lock);
7693 if (state->id_rcq_poll_busy & flag) {
7694 ibd_print_warn(state, "ibd_poll_rcq: multiple polling threads");
7695 state->id_rcq_poll_busy |= redo_flag;
7696 mutex_exit(&state->id_rcq_poll_lock);
7697 return;
7698 }
7699 state->id_rcq_poll_busy |= flag;
7700 mutex_exit(&state->id_rcq_poll_lock);
7701
7702 /*
7703 * Poll and drain the CQ
7704 */
7705 ibd_drain_rcq(state, rcq);
7706
7707 /*
7708 * Enable CQ notifications and redrain the cq to catch any
7709 * completions we might have missed after the ibd_drain_cq()
7710 * above and before the ibt_enable_cq_notify() that follows.
7711 * Finally, service any new requests to poll the cq that
7712 * could've come in after the ibt_enable_cq_notify().
7713 */
7714 do {
7715 if (ibt_enable_cq_notify(rcq, IBT_NEXT_COMPLETION) !=
7716 IBT_SUCCESS) {
7717 DPRINT(10, "ibd_intr: ibt_enable_cq_notify() failed");
7718 }
7719
7720 ibd_drain_rcq(state, rcq);
7721
7722 mutex_enter(&state->id_rcq_poll_lock);
7723 if (state->id_rcq_poll_busy & redo_flag)
7724 state->id_rcq_poll_busy &= ~redo_flag;
7725 else {
7726 state->id_rcq_poll_busy &= ~flag;
7727 redo = 0;
7728 }
7729 mutex_exit(&state->id_rcq_poll_lock);
7730
7731 } while (redo);
7732 }
7733
7734 /*
7735 * Unmap the memory area associated with a given swqe.
7736 */
7737 void
7738 ibd_unmap_mem(ibd_state_t *state, ibd_swqe_t *swqe)
7739 {
7740 ibt_status_t stat;
7741
7742 DPRINT(20, "ibd_unmap_mem: wqe=%p, seg=%d\n", swqe, swqe->w_swr.wr_nds);
7743
7744 if (swqe->w_mi_hdl) {
7745 if ((stat = ibt_unmap_mem_iov(state->id_hca_hdl,
7746 swqe->w_mi_hdl)) != IBT_SUCCESS) {
7747 DPRINT(10,
7748 "failed in ibt_unmap_mem_iov, ret=%d\n", stat);
7749 }
7750 swqe->w_mi_hdl = NULL;
7751 }
7752 swqe->w_swr.wr_nds = 0;
7753 }
7754
7755 void
7756 ibd_dec_ref_ace(ibd_state_t *state, ibd_ace_t *ace)
7757 {
7758 /*
7759 * The recycling logic can be eliminated from here
7760 * and put into the async thread if we create another
7761 * list to hold ACE's for unjoined mcg's.
7762 */
7763 if (DEC_REF_DO_CYCLE(ace)) {
7764 ibd_mce_t *mce;
7765
7766 /*
7767 * Check with the lock taken: we decremented
7768 * reference count without the lock, and some
7769 * transmitter might already have bumped the
7770 * reference count (possible in case of multicast
7771 * disable when we leave the AH on the active
7772 * list). If not still 0, get out, leaving the
7773 * recycle bit intact.
7774 *
7775 * Atomically transition the AH from active
7776 * to free list, and queue a work request to
7777 * leave the group and destroy the mce. No
7778 * transmitter can be looking at the AH or
7779 * the MCE in between, since we have the
7780 * ac_mutex lock. In the SendOnly reap case,
7781 * it is not necessary to hold the ac_mutex
7782 * and recheck the ref count (since the AH was
7783 * taken off the active list), we just do it
7784 * to have uniform processing with the Full
7785 * reap case.
7786 */
7787 mutex_enter(&state->id_ac_mutex);
7788 mce = ace->ac_mce;
7789 if (GET_REF_CYCLE(ace) == 0) {
7790 CLEAR_REFCYCLE(ace);
7791 /*
7792 * Identify the case of fullmember reap as
7793 * opposed to mcg trap reap. Also, port up
7794 * might set ac_mce to NULL to indicate Tx
7795 * cleanup should do no more than put the
7796 * AH in the free list (see ibd_async_link).
7797 */
7798 if (mce != NULL) {
7799 ace->ac_mce = NULL;
7800 IBD_ACACHE_PULLOUT_ACTIVE(state, ace);
7801 /*
7802 * mc_req was initialized at mce
7803 * creation time.
7804 */
7805 ibd_queue_work_slot(state,
7806 &mce->mc_req, IBD_ASYNC_REAP);
7807 }
7808 IBD_ACACHE_INSERT_FREE(state, ace);
7809 }
7810 mutex_exit(&state->id_ac_mutex);
7811 }
7812 }
7813
7814 /*
7815 * Common code that deals with clean ups after a successful or
7816 * erroneous transmission attempt.
7817 */
7818 static void
7819 ibd_tx_cleanup(ibd_state_t *state, ibd_swqe_t *swqe)
7820 {
7821 ibd_ace_t *ace = swqe->w_ahandle;
7822
7823 DPRINT(20, "ibd_tx_cleanup %p\n", swqe);
7824
7825 /*
7826 * If this was a dynamic mapping in ibd_send(), we need to
7827 * unmap here. If this was an lso buffer we'd used for sending,
7828 * we need to release the lso buf to the pool, since the resource
7829 * is scarce. However, if this was simply a normal send using
7830 * the copybuf (present in each swqe), we don't need to release it.
7831 */
7832 if (swqe->swqe_im_mblk != NULL) {
7833 if (swqe->w_buftype == IBD_WQE_MAPPED) {
7834 ibd_unmap_mem(state, swqe);
7835 } else if (swqe->w_buftype == IBD_WQE_LSOBUF) {
7836 ibd_release_lsobufs(state,
7837 swqe->w_swr.wr_sgl, swqe->w_swr.wr_nds);
7838 }
7839 ibd_free_lsohdr(swqe, swqe->swqe_im_mblk);
7840 freemsg(swqe->swqe_im_mblk);
7841 swqe->swqe_im_mblk = NULL;
7842 }
7843
7844 /*
7845 * Drop the reference count on the AH; it can be reused
7846 * now for a different destination if there are no more
7847 * posted sends that will use it. This can be eliminated
7848 * if we can always associate each Tx buffer with an AH.
7849 * The ace can be null if we are cleaning up from the
7850 * ibd_send() error path.
7851 */
7852 if (ace != NULL) {
7853 ibd_dec_ref_ace(state, ace);
7854 }
7855
7856 /*
7857 * Release the send wqe for reuse.
7858 */
7859 swqe->swqe_next = NULL;
7860 ibd_release_swqe(state, swqe, swqe, 1);
7861 }
7862
7863 static void
7864 ibd_tx_cleanup_list(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail)
7865 {
7866 ibd_ace_t *ace;
7867 ibd_swqe_t *swqe;
7868 int n = 0;
7869
7870 DPRINT(20, "ibd_tx_cleanup_list %p %p\n", head, tail);
7871
7872 for (swqe = head; swqe != NULL; swqe = WQE_TO_SWQE(swqe->swqe_next)) {
7873
7874 /*
7875 * If this was a dynamic mapping in ibd_send(), we need to
7876 * unmap here. If this was an lso buffer we'd used for sending,
7877 * we need to release the lso buf to the pool, since the
7878 * resource is scarce. However, if this was simply a normal
7879 * send using the copybuf (present in each swqe), we don't need
7880 * to release it.
7881 */
7882 if (swqe->swqe_im_mblk != NULL) {
7883 if (swqe->w_buftype == IBD_WQE_MAPPED) {
7884 ibd_unmap_mem(state, swqe);
7885 } else if (swqe->w_buftype == IBD_WQE_LSOBUF) {
7886 ibd_release_lsobufs(state,
7887 swqe->w_swr.wr_sgl, swqe->w_swr.wr_nds);
7888 }
7889 ibd_free_lsohdr(swqe, swqe->swqe_im_mblk);
7890 freemsg(swqe->swqe_im_mblk);
7891 swqe->swqe_im_mblk = NULL;
7892 }
7893
7894 /*
7895 * Drop the reference count on the AH; it can be reused
7896 * now for a different destination if there are no more
7897 * posted sends that will use it. This can be eliminated
7898 * if we can always associate each Tx buffer with an AH.
7899 * The ace can be null if we are cleaning up from the
7900 * ibd_send() error path.
7901 */
7902 ace = swqe->w_ahandle;
7903 if (ace != NULL) {
7904 ibd_dec_ref_ace(state, ace);
7905 }
7906 n++;
7907 }
7908
7909 /*
7910 * Release the send wqes for reuse.
7911 */
7912 ibd_release_swqe(state, head, tail, n);
7913 }
7914
7915 /*
7916 * Processing to be done after receipt of a packet; hand off to GLD
7917 * in the format expected by GLD. The received packet has this
7918 * format: 2b sap :: 00 :: data.
7919 */
7920 static mblk_t *
7921 ibd_process_rx(ibd_state_t *state, ibd_rwqe_t *rwqe, ibt_wc_t *wc)
7922 {
7923 ib_header_info_t *phdr;
7924 mblk_t *mp;
7925 ipoib_hdr_t *ipibp;
7926 ipha_t *iphap;
7927 ip6_t *ip6h;
7928 int len;
7929 ib_msglen_t pkt_len = wc->wc_bytes_xfer;
7930 uint32_t bufs;
7931
7932 /*
7933 * Track number handed to upper layer that need to be returned.
7934 */
7935 bufs = atomic_inc_32_nv(&state->id_rx_list.dl_bufs_outstanding);
7936
7937 /* Never run out of rwqes, use allocb when running low */
7938 if (bufs >= state->id_rx_bufs_outstanding_limit) {
7939 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding);
7940 atomic_inc_32(&state->id_rx_allocb);
7941 mp = allocb(pkt_len, BPRI_HI);
7942 if (mp) {
7943 bcopy(rwqe->rwqe_im_mblk->b_rptr, mp->b_rptr, pkt_len);
7944 ibd_post_recv(state, rwqe);
7945 } else { /* no memory */
7946 atomic_inc_32(&state->id_rx_allocb_failed);
7947 ibd_post_recv(state, rwqe);
7948 return (NULL);
7949 }
7950 } else {
7951 mp = rwqe->rwqe_im_mblk;
7952 }
7953
7954
7955 /*
7956 * Adjust write pointer depending on how much data came in.
7957 */
7958 mp->b_wptr = mp->b_rptr + pkt_len;
7959
7960 /*
7961 * Make sure this is NULL or we're in trouble.
7962 */
7963 if (mp->b_next != NULL) {
7964 ibd_print_warn(state,
7965 "ibd_process_rx: got duplicate mp from rcq?");
7966 mp->b_next = NULL;
7967 }
7968
7969 /*
7970 * the IB link will deliver one of the IB link layer
7971 * headers called, the Global Routing Header (GRH).
7972 * ibd driver uses the information in GRH to build the
7973 * Header_info structure and pass it with the datagram up
7974 * to GLDv3.
7975 * If the GRH is not valid, indicate to GLDv3 by setting
7976 * the VerTcFlow field to 0.
7977 */
7978 phdr = (ib_header_info_t *)mp->b_rptr;
7979 if (wc->wc_flags & IBT_WC_GRH_PRESENT) {
7980 phdr->ib_grh.ipoib_sqpn = htonl(wc->wc_qpn);
7981
7982 /* if it is loop back packet, just drop it. */
7983 if (state->id_enable_rc) {
7984 if (bcmp(&phdr->ib_grh.ipoib_sqpn,
7985 &state->rc_macaddr_loopback,
7986 IPOIB_ADDRL) == 0) {
7987 freemsg(mp);
7988 return (NULL);
7989 }
7990 } else {
7991 if (bcmp(&phdr->ib_grh.ipoib_sqpn, &state->id_macaddr,
7992 IPOIB_ADDRL) == 0) {
7993 freemsg(mp);
7994 return (NULL);
7995 }
7996 }
7997
7998 ovbcopy(&phdr->ib_grh.ipoib_sqpn, &phdr->ib_src,
7999 sizeof (ipoib_mac_t));
8000 if (*(uint8_t *)(phdr->ib_grh.ipoib_dgid_pref) == 0xFF) {
8001 phdr->ib_dst.ipoib_qpn = htonl(IB_MC_QPN);
8002 IBD_CLEAR_SCOPE_PKEY(&phdr->ib_dst);
8003 } else {
8004 phdr->ib_dst.ipoib_qpn = state->id_macaddr.ipoib_qpn;
8005 }
8006 } else {
8007 /*
8008 * It can not be a IBA multicast packet. Must have been
8009 * unicast for us. Just copy the interface address to dst.
8010 */
8011 phdr->ib_grh.ipoib_vertcflow = 0;
8012 ovbcopy(&state->id_macaddr, &phdr->ib_dst,
8013 sizeof (ipoib_mac_t));
8014 }
8015
8016 /*
8017 * For ND6 packets, padding is at the front of the source/target
8018 * lladdr. However the inet6 layer is not aware of it, hence remove
8019 * the padding from such packets.
8020 */
8021 ipibp = (ipoib_hdr_t *)((uchar_t *)mp->b_rptr + sizeof (ipoib_pgrh_t));
8022 if (ntohs(ipibp->ipoib_type) == ETHERTYPE_IPV6) {
8023 ip6h = (ip6_t *)((uchar_t *)ipibp + sizeof (ipoib_hdr_t));
8024 len = ntohs(ip6h->ip6_plen);
8025 if (ip6h->ip6_nxt == IPPROTO_ICMPV6) {
8026 /* LINTED: E_CONSTANT_CONDITION */
8027 IBD_PAD_NSNA(ip6h, len, IBD_RECV);
8028 }
8029 }
8030
8031 /*
8032 * Update statistics
8033 */
8034 atomic_add_64(&state->id_rcv_bytes, pkt_len);
8035 atomic_inc_64(&state->id_rcv_pkt);
8036 if (bcmp(&phdr->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0)
8037 atomic_inc_64(&state->id_brd_rcv);
8038 else if ((ntohl(phdr->ib_dst.ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN)
8039 atomic_inc_64(&state->id_multi_rcv);
8040
8041 iphap = (ipha_t *)((uchar_t *)ipibp + sizeof (ipoib_hdr_t));
8042 /*
8043 * Set receive checksum status in mp
8044 * Hardware checksumming can be considered valid only if:
8045 * 1. CQE.IP_OK bit is set
8046 * 2. CQE.CKSUM = 0xffff
8047 * 3. IPv6 routing header is not present in the packet
8048 * 4. If there are no IP_OPTIONS in the IP HEADER
8049 */
8050
8051 if (((wc->wc_flags & IBT_WC_CKSUM_OK) == IBT_WC_CKSUM_OK) &&
8052 (wc->wc_cksum == 0xFFFF) &&
8053 (iphap->ipha_version_and_hdr_length == IP_SIMPLE_HDR_VERSION)) {
8054 mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM_OK);
8055 }
8056
8057 return (mp);
8058 }
8059
8060 /*
8061 * Callback code invoked from STREAMs when the receive data buffer is
8062 * free for recycling.
8063 */
8064 static void
8065 ibd_freemsg_cb(char *arg)
8066 {
8067 ibd_rwqe_t *rwqe = (ibd_rwqe_t *)arg;
8068 ibd_state_t *state = rwqe->w_state;
8069
8070 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding);
8071
8072 /*
8073 * If the driver is stopped, just free the rwqe.
8074 */
8075 if (atomic_add_32_nv(&state->id_running, 0) == 0) {
8076 DPRINT(6, "ibd_freemsg: wqe being freed");
8077 rwqe->rwqe_im_mblk = NULL;
8078 ibd_free_rwqe(state, rwqe);
8079 return;
8080 }
8081
8082 rwqe->rwqe_im_mblk = desballoc(rwqe->rwqe_copybuf.ic_bufaddr,
8083 state->id_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb);
8084 if (rwqe->rwqe_im_mblk == NULL) {
8085 ibd_free_rwqe(state, rwqe);
8086 DPRINT(6, "ibd_freemsg: desballoc failed");
8087 return;
8088 }
8089
8090 ibd_post_recv(state, rwqe);
8091 }
8092
8093 static uint_t
8094 ibd_tx_recycle(caddr_t arg)
8095 {
8096 ibd_state_t *state = (ibd_state_t *)arg;
8097
8098 /*
8099 * Poll for completed entries
8100 */
8101 ibd_poll_scq(state, state->id_scq_hdl);
8102
8103 return (DDI_INTR_CLAIMED);
8104 }
8105
8106 #ifdef IBD_LOGGING
8107 static void
8108 ibd_log_init(void)
8109 {
8110 ibd_lbuf = kmem_zalloc(IBD_LOG_SZ, KM_SLEEP);
8111 ibd_lbuf_ndx = 0;
8112
8113 mutex_init(&ibd_lbuf_lock, NULL, MUTEX_DRIVER, NULL);
8114 }
8115
8116 static void
8117 ibd_log_fini(void)
8118 {
8119 if (ibd_lbuf)
8120 kmem_free(ibd_lbuf, IBD_LOG_SZ);
8121 ibd_lbuf_ndx = 0;
8122 ibd_lbuf = NULL;
8123
8124 mutex_destroy(&ibd_lbuf_lock);
8125 }
8126
8127 static void
8128 ibd_log(const char *fmt, ...)
8129 {
8130 va_list ap;
8131 uint32_t off;
8132 uint32_t msglen;
8133 char tmpbuf[IBD_DMAX_LINE];
8134
8135 if (ibd_lbuf == NULL)
8136 return;
8137
8138 va_start(ap, fmt);
8139 msglen = vsnprintf(tmpbuf, IBD_DMAX_LINE, fmt, ap);
8140 va_end(ap);
8141
8142 if (msglen >= IBD_DMAX_LINE)
8143 msglen = IBD_DMAX_LINE - 1;
8144
8145 mutex_enter(&ibd_lbuf_lock);
8146
8147 off = ibd_lbuf_ndx; /* current msg should go here */
8148 if ((ibd_lbuf_ndx) && (ibd_lbuf[ibd_lbuf_ndx-1] != '\n'))
8149 ibd_lbuf[ibd_lbuf_ndx-1] = '\n';
8150
8151 ibd_lbuf_ndx += msglen; /* place where next msg should start */
8152 ibd_lbuf[ibd_lbuf_ndx] = 0; /* current msg should terminate */
8153
8154 if (ibd_lbuf_ndx >= (IBD_LOG_SZ - 2 * IBD_DMAX_LINE))
8155 ibd_lbuf_ndx = 0;
8156
8157 mutex_exit(&ibd_lbuf_lock);
8158
8159 bcopy(tmpbuf, ibd_lbuf+off, msglen); /* no lock needed for this */
8160 }
8161 #endif
8162
8163 /* ARGSUSED */
8164 static int
8165 ibd_create_partition(void *karg, intptr_t arg, int mode, cred_t *credp,
8166 int *rvalp)
8167 {
8168 ibd_create_ioctl_t *cmd = karg;
8169 ibd_state_t *state, *port_state, *p;
8170 int i, err, rval = 0;
8171 mac_register_t *macp;
8172 ibt_hca_portinfo_t *pinfop = NULL;
8173 ibt_status_t ibt_status;
8174 uint_t psize, pinfosz;
8175 boolean_t force_create = B_FALSE;
8176
8177 cmd->ibdioc.ioc_status = 0;
8178
8179 if (cmd->ibdioc.ioc_port_inst < 0) {
8180 cmd->ibdioc.ioc_status = IBD_INVALID_PORT_INST;
8181 return (EINVAL);
8182 }
8183 port_state = ddi_get_soft_state(ibd_list, cmd->ibdioc.ioc_port_inst);
8184 if (port_state == NULL) {
8185 DPRINT(10, "ibd_create_partition: failed to get state %d",
8186 cmd->ibdioc.ioc_port_inst);
8187 cmd->ibdioc.ioc_status = IBD_INVALID_PORT_INST;
8188 return (EINVAL);
8189 }
8190
8191 /* Limited PKeys not supported */
8192 if (cmd->ioc_pkey <= IB_PKEY_INVALID_FULL) {
8193 rval = EINVAL;
8194 goto part_create_return;
8195 }
8196
8197 if (cmd->ioc_force_create == 0) {
8198 /*
8199 * Check if the port pkey table contains the pkey for which
8200 * this partition is being created.
8201 */
8202 ibt_status = ibt_query_hca_ports(port_state->id_hca_hdl,
8203 port_state->id_port, &pinfop, &psize, &pinfosz);
8204
8205 if ((ibt_status != IBT_SUCCESS) || (psize != 1)) {
8206 rval = EINVAL;
8207 goto part_create_return;
8208 }
8209
8210 if (pinfop->p_linkstate != IBT_PORT_ACTIVE) {
8211 rval = ENETDOWN;
8212 cmd->ibdioc.ioc_status = IBD_PORT_IS_DOWN;
8213 goto part_create_return;
8214 }
8215
8216 for (i = 0; i < pinfop->p_pkey_tbl_sz; i++) {
8217 if (pinfop->p_pkey_tbl[i] == cmd->ioc_pkey) {
8218 break;
8219 }
8220 }
8221 if (i == pinfop->p_pkey_tbl_sz) {
8222 rval = EINVAL;
8223 cmd->ibdioc.ioc_status = IBD_PKEY_NOT_PRESENT;
8224 goto part_create_return;
8225 }
8226 } else {
8227 force_create = B_TRUE;
8228 }
8229
8230 mutex_enter(&ibd_objlist_lock);
8231 for (p = ibd_objlist_head; p; p = p->id_next) {
8232 if ((p->id_port_inst == cmd->ibdioc.ioc_port_inst) &&
8233 (p->id_pkey == cmd->ioc_pkey) &&
8234 (p->id_plinkid == cmd->ioc_partid)) {
8235 mutex_exit(&ibd_objlist_lock);
8236 rval = EEXIST;
8237 cmd->ibdioc.ioc_status = IBD_PARTITION_EXISTS;
8238 goto part_create_return;
8239 }
8240 }
8241 mutex_exit(&ibd_objlist_lock);
8242
8243 state = kmem_zalloc(sizeof (ibd_state_t), KM_SLEEP);
8244
8245 state->id_type = IBD_PARTITION_OBJ;
8246
8247 state->id_plinkid = cmd->ioc_partid;
8248 state->id_dlinkid = cmd->ibdioc.ioc_linkid;
8249 state->id_port_inst = cmd->ibdioc.ioc_port_inst;
8250
8251 state->id_dip = port_state->id_dip;
8252 state->id_port = port_state->id_port;
8253 state->id_pkey = cmd->ioc_pkey;
8254 state->id_hca_guid = port_state->id_hca_guid;
8255 state->id_port_guid = port_state->id_port_guid;
8256 state->id_force_create = force_create;
8257
8258 mutex_init(&state->id_macst_lock, NULL, MUTEX_DRIVER, NULL);
8259 cv_init(&state->id_macst_cv, NULL, CV_DEFAULT, NULL);
8260
8261 if (ibd_part_attach(state, state->id_dip) != DDI_SUCCESS) {
8262 rval = EIO;
8263 cmd->ibdioc.ioc_status = IBD_NO_HW_RESOURCE;
8264 goto fail;
8265 }
8266
8267 if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
8268 rval = EAGAIN;
8269 goto fail;
8270 }
8271
8272 macp->m_type_ident = MAC_PLUGIN_IDENT_IB;
8273 macp->m_dip = port_state->id_dip;
8274 macp->m_instance = (uint_t)-1;
8275 macp->m_driver = state;
8276 macp->m_src_addr = (uint8_t *)&state->id_macaddr;
8277 macp->m_callbacks = &ibd_m_callbacks;
8278 macp->m_min_sdu = 0;
8279 macp->m_multicast_sdu = IBD_DEF_MAX_SDU;
8280 if (state->id_enable_rc) {
8281 macp->m_max_sdu = IBD_DEF_RC_MAX_SDU;
8282 } else {
8283 macp->m_max_sdu = IBD_DEF_MAX_SDU;
8284 }
8285 macp->m_priv_props = ibd_priv_props;
8286
8287 err = mac_register(macp, &state->id_mh);
8288 mac_free(macp);
8289
8290 if (err != 0) {
8291 DPRINT(10, "ibd_create_partition: mac_register() failed %d",
8292 err);
8293 rval = err;
8294 goto fail;
8295 }
8296
8297 err = dls_devnet_create(state->id_mh,
8298 cmd->ioc_partid, crgetzoneid(credp));
8299 if (err != 0) {
8300 DPRINT(10, "ibd_create_partition: dls_devnet_create() failed "
8301 "%d", err);
8302 rval = err;
8303 (void) mac_unregister(state->id_mh);
8304 goto fail;
8305 }
8306
8307 /*
8308 * Add the new partition state structure to the list
8309 */
8310 mutex_enter(&ibd_objlist_lock);
8311 if (ibd_objlist_head)
8312 state->id_next = ibd_objlist_head;
8313
8314 ibd_objlist_head = state;
8315 mutex_exit(&ibd_objlist_lock);
8316
8317 part_create_return:
8318 if (pinfop) {
8319 ibt_free_portinfo(pinfop, pinfosz);
8320 }
8321 return (rval);
8322
8323 fail:
8324 if (pinfop) {
8325 ibt_free_portinfo(pinfop, pinfosz);
8326 }
8327 ibd_part_unattach(state);
8328 kmem_free(state, sizeof (ibd_state_t));
8329 return (rval);
8330 }
8331
8332 /* ARGSUSED */
8333 static int
8334 ibd_delete_partition(void *karg, intptr_t arg, int mode, cred_t *credp,
8335 int *rvalp)
8336 {
8337 int err;
8338 datalink_id_t tmpid;
8339 ibd_state_t *node, *prev;
8340 ibd_delete_ioctl_t *cmd = karg;
8341
8342 prev = NULL;
8343
8344 mutex_enter(&ibd_objlist_lock);
8345 node = ibd_objlist_head;
8346
8347 /* Find the ibd state structure corresponding to the partition */
8348 while (node != NULL) {
8349 if (node->id_plinkid == cmd->ioc_partid)
8350 break;
8351 prev = node;
8352 node = node->id_next;
8353 }
8354
8355 if (node == NULL) {
8356 mutex_exit(&ibd_objlist_lock);
8357 return (ENOENT);
8358 }
8359
8360 if ((err = dls_devnet_destroy(node->id_mh, &tmpid, B_TRUE)) != 0) {
8361 DPRINT(10, "ibd_delete_partition: dls_devnet_destroy() failed "
8362 "%d", err);
8363 mutex_exit(&ibd_objlist_lock);
8364 return (err);
8365 }
8366
8367 /*
8368 * Call ibd_part_unattach() only after making sure that the instance has
8369 * not been started yet and is also not in late hca init mode.
8370 */
8371 ibd_set_mac_progress(node, IBD_DRV_DELETE_IN_PROGRESS);
8372
8373 err = 0;
8374 if ((node->id_mac_state & IBD_DRV_STARTED) ||
8375 (node->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ||
8376 (ibd_part_busy(node) != DDI_SUCCESS) ||
8377 ((err = mac_disable(node->id_mh)) != 0)) {
8378 (void) dls_devnet_create(node->id_mh, cmd->ioc_partid,
8379 crgetzoneid(credp));
8380 ibd_clr_mac_progress(node, IBD_DRV_DELETE_IN_PROGRESS);
8381 mutex_exit(&ibd_objlist_lock);
8382 return (err != 0 ? err : EBUSY);
8383 }
8384
8385 node->id_mac_state |= IBD_DRV_IN_DELETION;
8386
8387 ibd_part_unattach(node);
8388
8389 ibd_clr_mac_progress(node, IBD_DRV_DELETE_IN_PROGRESS);
8390
8391 /* Remove the partition state structure from the linked list */
8392 if (prev == NULL)
8393 ibd_objlist_head = node->id_next;
8394 else
8395 prev->id_next = node->id_next;
8396 mutex_exit(&ibd_objlist_lock);
8397
8398 if ((err = mac_unregister(node->id_mh)) != 0) {
8399 DPRINT(10, "ibd_delete_partition: mac_unregister() failed %d",
8400 err);
8401 }
8402
8403 cv_destroy(&node->id_macst_cv);
8404 mutex_destroy(&node->id_macst_lock);
8405
8406 kmem_free(node, sizeof (ibd_state_t));
8407
8408 return (0);
8409 }
8410
8411 /* ARGSUSED */
8412 static int
8413 ibd_get_partition_info(void *karg, intptr_t arg, int mode, cred_t *cred,
8414 int *rvalp)
8415 {
8416 ibd_ioctl_t cmd;
8417 ibpart_ioctl_t partioc;
8418 ibport_ioctl_t portioc;
8419 #ifdef _MULTI_DATAMODEL
8420 ibport_ioctl32_t portioc32;
8421 #endif
8422 ibd_state_t *state, *port_state;
8423 int size;
8424 ibt_hca_portinfo_t *pinfop = NULL;
8425 ibt_status_t ibt_status;
8426 uint_t psize, pinfosz;
8427 int rval = 0;
8428
8429 size = sizeof (ibd_ioctl_t);
8430 if (ddi_copyin((void *)arg, &cmd, size, mode)) {
8431 return (EFAULT);
8432 }
8433 cmd.ioc_status = 0;
8434 switch (cmd.ioc_info_cmd) {
8435 case IBD_INFO_CMD_IBPART:
8436 size = sizeof (ibpart_ioctl_t);
8437 if (ddi_copyin((void *)arg, &partioc, size, mode)) {
8438 return (EFAULT);
8439 }
8440
8441 mutex_enter(&ibd_objlist_lock);
8442 /* Find the ibd state structure corresponding the partition */
8443 for (state = ibd_objlist_head; state; state = state->id_next) {
8444 if (state->id_plinkid == cmd.ioc_linkid) {
8445 break;
8446 }
8447 }
8448
8449 if (state == NULL) {
8450 mutex_exit(&ibd_objlist_lock);
8451 return (ENOENT);
8452 }
8453
8454 partioc.ibdioc.ioc_linkid = state->id_dlinkid;
8455 partioc.ibdioc.ioc_port_inst = state->id_port_inst;
8456 partioc.ibdioc.ioc_portnum = state->id_port;
8457 partioc.ibdioc.ioc_hcaguid = state->id_hca_guid;
8458 partioc.ibdioc.ioc_portguid = state->id_port_guid;
8459 partioc.ibdioc.ioc_status = 0;
8460 partioc.ioc_partid = state->id_plinkid;
8461 partioc.ioc_pkey = state->id_pkey;
8462 partioc.ioc_force_create = state->id_force_create;
8463 if (ddi_copyout((void *)&partioc, (void *)arg, size, mode)) {
8464 mutex_exit(&ibd_objlist_lock);
8465 return (EFAULT);
8466 }
8467 mutex_exit(&ibd_objlist_lock);
8468
8469 break;
8470
8471 case IBD_INFO_CMD_IBPORT:
8472 if ((cmd.ioc_port_inst < 0) || ((port_state =
8473 ddi_get_soft_state(ibd_list, cmd.ioc_port_inst)) == NULL)) {
8474 DPRINT(10, "ibd_create_partition: failed to get"
8475 " state %d", cmd.ioc_port_inst);
8476 size = sizeof (ibd_ioctl_t);
8477 cmd.ioc_status = IBD_INVALID_PORT_INST;
8478 if (ddi_copyout((void *)&cmd, (void *)arg, size,
8479 mode)) {
8480 return (EFAULT);
8481 }
8482 return (EINVAL);
8483 }
8484 ibt_status = ibt_query_hca_ports(port_state->id_hca_hdl,
8485 port_state->id_port, &pinfop, &psize, &pinfosz);
8486 if ((ibt_status != IBT_SUCCESS) || (psize != 1)) {
8487 return (EINVAL);
8488 }
8489 #ifdef _MULTI_DATAMODEL
8490 switch (ddi_model_convert_from(mode & FMODELS)) {
8491 case DDI_MODEL_ILP32: {
8492 size = sizeof (ibport_ioctl32_t);
8493 if (ddi_copyin((void *)arg, &portioc32, size, mode)) {
8494 rval = EFAULT;
8495 goto fail;
8496 }
8497 portioc32.ibdioc.ioc_status = 0;
8498 portioc32.ibdioc.ioc_portnum = port_state->id_port;
8499 portioc32.ibdioc.ioc_hcaguid =
8500 port_state->id_hca_guid;
8501 portioc32.ibdioc.ioc_portguid =
8502 port_state->id_port_guid;
8503 if (portioc32.ioc_pkey_tbl_sz !=
8504 pinfop->p_pkey_tbl_sz) {
8505 rval = EINVAL;
8506 size = sizeof (ibd_ioctl_t);
8507 portioc32.ibdioc.ioc_status =
8508 IBD_INVALID_PKEY_TBL_SIZE;
8509 if (ddi_copyout((void *)&portioc32.ibdioc,
8510 (void *)arg, size, mode)) {
8511 rval = EFAULT;
8512 goto fail;
8513 }
8514 goto fail;
8515 }
8516 size = pinfop->p_pkey_tbl_sz * sizeof (ib_pkey_t);
8517 if (ddi_copyout((void *)pinfop->p_pkey_tbl,
8518 (void *)(uintptr_t)portioc32.ioc_pkeys, size,
8519 mode)) {
8520 rval = EFAULT;
8521 goto fail;
8522 }
8523 size = sizeof (ibport_ioctl32_t);
8524 if (ddi_copyout((void *)&portioc32, (void *)arg, size,
8525 mode)) {
8526 rval = EFAULT;
8527 goto fail;
8528 }
8529 break;
8530 }
8531 case DDI_MODEL_NONE:
8532 size = sizeof (ibport_ioctl_t);
8533 if (ddi_copyin((void *)arg, &portioc, size, mode)) {
8534 rval = EFAULT;
8535 goto fail;
8536 }
8537 portioc.ibdioc.ioc_status = 0;
8538 portioc.ibdioc.ioc_portnum = port_state->id_port;
8539 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid;
8540 portioc.ibdioc.ioc_portguid = port_state->id_port_guid;
8541 if (portioc.ioc_pkey_tbl_sz != pinfop->p_pkey_tbl_sz) {
8542 rval = EINVAL;
8543 size = sizeof (ibd_ioctl_t);
8544 portioc.ibdioc.ioc_status =
8545 IBD_INVALID_PKEY_TBL_SIZE;
8546 if (ddi_copyout((void *)&portioc.ibdioc,
8547 (void *)arg, size, mode)) {
8548 rval = EFAULT;
8549 goto fail;
8550 }
8551 goto fail;
8552 }
8553 size = pinfop->p_pkey_tbl_sz * sizeof (ib_pkey_t);
8554 if (ddi_copyout((void *)pinfop->p_pkey_tbl,
8555 (void *)(portioc.ioc_pkeys), size, mode)) {
8556 rval = EFAULT;
8557 goto fail;
8558 }
8559 size = sizeof (ibport_ioctl_t);
8560 if (ddi_copyout((void *)&portioc, (void *)arg, size,
8561 mode)) {
8562 rval = EFAULT;
8563 goto fail;
8564 }
8565 break;
8566 }
8567 #else /* ! _MULTI_DATAMODEL */
8568 size = sizeof (ibport_ioctl_t);
8569 if (ddi_copyin((void *)arg, &portioc, size, mode)) {
8570 rval = EFAULT;
8571 goto fail;
8572 }
8573 portioc.ibdioc.ioc_status = 0;
8574 portioc.ibdioc.ioc_portnum = port_state->id_port;
8575 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid;
8576 portioc.ibdioc.ioc_portguid = port_state->id_port_guid;
8577 if (portioc.ioc_pkey_tbl_sz != pinfop->p_pkey_tbl_sz) {
8578 rval = EINVAL;
8579 size = sizeof (ibd_ioctl_t);
8580 portioc.ibdioc.ioc_status = IBD_INVALID_PKEY_TBL_SIZE;
8581 if (ddi_copyout((void *)&portioc.ibdioc, (void *)arg,
8582 size, mode)) {
8583 rval = EFAULT;
8584 goto fail;
8585 }
8586 goto fail;
8587 }
8588 size = pinfop->p_pkey_tbl_sz * sizeof (ib_pkey_t);
8589 if (ddi_copyout((void *)pinfop->p_pkey_tbl,
8590 (void *)(portioc.ioc_pkeys), size, mode)) {
8591 rval = EFAULT;
8592 goto fail;
8593 }
8594 size = sizeof (ibport_ioctl_t);
8595 if (ddi_copyout((void *)&portioc, (void *)arg, size,
8596 mode)) {
8597 rval = EFAULT;
8598 goto fail;
8599 }
8600 #endif /* _MULTI_DATAMODEL */
8601
8602 break;
8603
8604 case IBD_INFO_CMD_PKEYTBLSZ:
8605 if ((cmd.ioc_port_inst < 0) || ((port_state =
8606 ddi_get_soft_state(ibd_list, cmd.ioc_port_inst)) == NULL)) {
8607 DPRINT(10, "ibd_create_partition: failed to get"
8608 " state %d", cmd.ioc_port_inst);
8609 size = sizeof (ibd_ioctl_t);
8610 cmd.ioc_status = IBD_INVALID_PORT_INST;
8611 if (ddi_copyout((void *)&cmd, (void *)arg, size,
8612 mode)) {
8613 return (EFAULT);
8614 }
8615 return (EINVAL);
8616 }
8617 ibt_status = ibt_query_hca_ports(port_state->id_hca_hdl,
8618 port_state->id_port, &pinfop, &psize, &pinfosz);
8619 if ((ibt_status != IBT_SUCCESS) || (psize != 1)) {
8620 return (EINVAL);
8621 }
8622 #ifdef _MULTI_DATAMODEL
8623 switch (ddi_model_convert_from(mode & FMODELS)) {
8624 case DDI_MODEL_ILP32: {
8625 size = sizeof (ibport_ioctl32_t);
8626 if (ddi_copyin((void *)arg, &portioc32, size, mode)) {
8627 rval = EFAULT;
8628 goto fail;
8629 }
8630 portioc32.ibdioc.ioc_status = 0;
8631 portioc32.ibdioc.ioc_portnum = port_state->id_port;
8632 portioc32.ibdioc.ioc_hcaguid =
8633 port_state->id_hca_guid;
8634 portioc32.ibdioc.ioc_portguid =
8635 port_state->id_port_guid;
8636 portioc32.ioc_pkey_tbl_sz = pinfop->p_pkey_tbl_sz;
8637 if (ddi_copyout((void *)&portioc32, (void *)arg, size,
8638 mode)) {
8639 rval = EFAULT;
8640 goto fail;
8641 }
8642 break;
8643 }
8644 case DDI_MODEL_NONE:
8645 size = sizeof (ibport_ioctl_t);
8646 if (ddi_copyin((void *)arg, &portioc, size, mode)) {
8647 rval = EFAULT;
8648 goto fail;
8649 }
8650 portioc.ibdioc.ioc_status = 0;
8651 portioc.ibdioc.ioc_portnum = port_state->id_port;
8652 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid;
8653 portioc.ibdioc.ioc_portguid = port_state->id_port_guid;
8654 portioc.ioc_pkey_tbl_sz = pinfop->p_pkey_tbl_sz;
8655 if (ddi_copyout((void *)&portioc, (void *)arg, size,
8656 mode)) {
8657 rval = EFAULT;
8658 goto fail;
8659 }
8660 break;
8661 }
8662 #else /* ! _MULTI_DATAMODEL */
8663 size = sizeof (ibport_ioctl_t);
8664 if (ddi_copyin((void *)arg, &portioc, size, mode)) {
8665 rval = EFAULT;
8666 goto fail;
8667 }
8668 portioc.ibdioc.ioc_status = 0;
8669 portioc.ibdioc.ioc_portnum = port_state->id_port;
8670 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid;
8671 portioc.ibdioc.ioc_portguid = port_state->id_port_guid;
8672 portioc.ioc_pkey_tbl_sz = pinfop->p_pkey_tbl_sz;
8673 if (ddi_copyout((void *)&portioc, (void *)arg, size,
8674 mode)) {
8675 rval = EFAULT;
8676 goto fail;
8677 }
8678 #endif /* _MULTI_DATAMODEL */
8679 break;
8680
8681 default:
8682 return (EINVAL);
8683
8684 } /* switch (cmd.ioc_info_cmd) */
8685 fail:
8686 if (pinfop) {
8687 ibt_free_portinfo(pinfop, pinfosz);
8688 }
8689 return (rval);
8690 }
8691
8692 /* ARGSUSED */
8693 static void
8694 ibdpd_async_handler(void *arg, ibt_hca_hdl_t hca_hdl,
8695 ibt_async_code_t code, ibt_async_event_t *event)
8696 {
8697 ibd_state_t *state = (ibd_state_t *)arg;
8698 link_state_t lstate;
8699
8700 switch (code) {
8701 case IBT_EVENT_PORT_UP:
8702 case IBT_ERROR_PORT_DOWN:
8703 if (ibd_get_port_state(state, &lstate) != 0)
8704 break;
8705
8706 if (state->id_link_state != lstate) {
8707 state->id_link_state = lstate;
8708 mac_link_update(state->id_mh, lstate);
8709 }
8710 break;
8711 default:
8712 break;
8713 }
8714 }
8715
8716 static int
8717 ibd_get_port_state(ibd_state_t *state, link_state_t *lstate)
8718 {
8719 ibt_hca_portinfo_t *port_infop;
8720 uint_t psize, port_infosz;
8721 ibt_status_t ret;
8722
8723 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
8724 &port_infop, &psize, &port_infosz);
8725 if ((ret != IBT_SUCCESS) || (psize != 1))
8726 return (-1);
8727
8728 state->id_sgid = *port_infop->p_sgid_tbl;
8729 state->id_link_speed = ibd_get_portspeed(state);
8730
8731 if (port_infop->p_linkstate == IBT_PORT_ACTIVE)
8732 *lstate = LINK_STATE_UP;
8733 else
8734 *lstate = LINK_STATE_DOWN;
8735
8736 ibt_free_portinfo(port_infop, port_infosz);
8737 return (0);
8738 }
8739
8740 static int
8741 ibd_port_attach(dev_info_t *dip)
8742 {
8743 ibd_state_t *state;
8744 link_state_t lstate;
8745 int instance;
8746 ibt_status_t ret;
8747
8748 /*
8749 * Allocate softstate structure
8750 */
8751 instance = ddi_get_instance(dip);
8752 if (ddi_soft_state_zalloc(ibd_list, instance) == DDI_FAILURE) {
8753 DPRINT(10, "ibd_port_attach: ddi_soft_state_zalloc() failed");
8754 return (DDI_FAILURE);
8755 }
8756
8757 state = ddi_get_soft_state(ibd_list, instance);
8758
8759 state->id_dip = dip;
8760 state->id_type = IBD_PORT_DRIVER;
8761
8762 if ((state->id_port = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
8763 "port-number", 0)) == 0) {
8764 DPRINT(10, "ibd_port_attach: invalid port number (%d)",
8765 state->id_port);
8766 return (DDI_FAILURE);
8767 }
8768 if ((state->id_hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0,
8769 "hca-guid", 0)) == 0) {
8770 DPRINT(10, "ibd_port_attach: hca has invalid guid (0x%llx)",
8771 state->id_hca_guid);
8772 return (DDI_FAILURE);
8773 }
8774 if ((state->id_port_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0,
8775 "port-guid", 0)) == 0) {
8776 DPRINT(10, "ibd_port_attach: port has invalid guid (0x%llx)",
8777 state->id_port_guid);
8778 return (DDI_FAILURE);
8779 }
8780
8781 /*
8782 * Attach to IBTL
8783 */
8784 if ((ret = ibt_attach(&ibdpd_clnt_modinfo, dip, state,
8785 &state->id_ibt_hdl)) != IBT_SUCCESS) {
8786 DPRINT(10, "ibd_port_attach: failed in ibt_attach(), ret=%d",
8787 ret);
8788 goto done;
8789 }
8790
8791 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE;
8792
8793 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid,
8794 &state->id_hca_hdl)) != IBT_SUCCESS) {
8795 DPRINT(10, "ibd_port_attach: ibt_open_hca() failed, ret=%d",
8796 ret);
8797 goto done;
8798 }
8799 state->id_mac_state |= IBD_DRV_HCA_OPENED;
8800
8801 /* Update link status */
8802
8803 if (ibd_get_port_state(state, &lstate) != 0) {
8804 DPRINT(10, "ibd_port_attach: ibt_open_hca() failed, ret=%d",
8805 ret);
8806 goto done;
8807 }
8808 state->id_link_state = lstate;
8809 /*
8810 * Register ibd interfaces with the Nemo framework
8811 */
8812 if (ibd_register_mac(state, dip) != IBT_SUCCESS) {
8813 DPRINT(10, "ibd_port_attach: failed in ibd_register_mac()");
8814 goto done;
8815 }
8816 state->id_mac_state |= IBD_DRV_MAC_REGISTERED;
8817
8818 mac_link_update(state->id_mh, lstate);
8819
8820 return (DDI_SUCCESS);
8821 done:
8822 (void) ibd_port_unattach(state, dip);
8823 return (DDI_FAILURE);
8824 }
8825
8826 static int
8827 ibd_port_unattach(ibd_state_t *state, dev_info_t *dip)
8828 {
8829 int instance;
8830 uint32_t progress = state->id_mac_state;
8831 ibt_status_t ret;
8832
8833 if (progress & IBD_DRV_MAC_REGISTERED) {
8834 (void) mac_unregister(state->id_mh);
8835 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED);
8836 }
8837
8838 if (progress & IBD_DRV_HCA_OPENED) {
8839 if ((ret = ibt_close_hca(state->id_hca_hdl)) !=
8840 IBT_SUCCESS) {
8841 ibd_print_warn(state, "failed to close "
8842 "HCA device, ret=%d", ret);
8843 }
8844 state->id_hca_hdl = NULL;
8845 state->id_mac_state &= (~IBD_DRV_HCA_OPENED);
8846 }
8847
8848 if (progress & IBD_DRV_IBTL_ATTACH_DONE) {
8849 if ((ret = ibt_detach(state->id_ibt_hdl)) != IBT_SUCCESS) {
8850 ibd_print_warn(state,
8851 "ibt_detach() failed, ret=%d", ret);
8852 }
8853 state->id_ibt_hdl = NULL;
8854 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE);
8855 }
8856 instance = ddi_get_instance(dip);
8857 ddi_soft_state_free(ibd_list, instance);
8858
8859 return (DDI_SUCCESS);
8860 }
8861
8862 ibt_status_t
8863 ibd_get_part_attr(datalink_id_t linkid, ibt_part_attr_t *attr)
8864 {
8865 ibd_state_t *state;
8866
8867 mutex_enter(&ibd_objlist_lock);
8868
8869 /* Find the ibd state structure corresponding the partition */
8870 for (state = ibd_objlist_head; state; state = state->id_next) {
8871 if (state->id_plinkid == linkid) {
8872 break;
8873 }
8874 }
8875
8876 if (state == NULL) {
8877 mutex_exit(&ibd_objlist_lock);
8878 return (IBT_NO_SUCH_OBJECT);
8879 }
8880
8881 attr->pa_dlinkid = state->id_dlinkid;
8882 attr->pa_plinkid = state->id_plinkid;
8883 attr->pa_port = state->id_port;
8884 attr->pa_hca_guid = state->id_hca_guid;
8885 attr->pa_port_guid = state->id_port_guid;
8886 attr->pa_pkey = state->id_pkey;
8887
8888 mutex_exit(&ibd_objlist_lock);
8889
8890 return (IBT_SUCCESS);
8891 }
8892
8893 ibt_status_t
8894 ibd_get_all_part_attr(ibt_part_attr_t **attr_list, int *nparts)
8895 {
8896 ibd_state_t *state;
8897 int n = 0;
8898 ibt_part_attr_t *attr;
8899
8900 mutex_enter(&ibd_objlist_lock);
8901
8902 for (state = ibd_objlist_head; state; state = state->id_next)
8903 n++;
8904
8905 *nparts = n;
8906 if (n == 0) {
8907 *attr_list = NULL;
8908 mutex_exit(&ibd_objlist_lock);
8909 return (IBT_SUCCESS);
8910 }
8911
8912 *attr_list = kmem_alloc(sizeof (ibt_part_attr_t) * n, KM_SLEEP);
8913 attr = *attr_list;
8914 for (state = ibd_objlist_head; state; state = state->id_next) {
8915 #ifdef DEBUG
8916 ASSERT(n > 0);
8917 n--;
8918 #endif
8919 attr->pa_dlinkid = state->id_dlinkid;
8920 attr->pa_plinkid = state->id_plinkid;
8921 attr->pa_port = state->id_port;
8922 attr->pa_hca_guid = state->id_hca_guid;
8923 attr->pa_port_guid = state->id_port_guid;
8924 attr->pa_pkey = state->id_pkey;
8925 attr++;
8926 }
8927
8928 mutex_exit(&ibd_objlist_lock);
8929 return (IBT_SUCCESS);
8930 }