1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Copyright 2019, Joyent, Inc.
28 */
29
30 /*
31 * Softmac data-path switching:
32 *
33 * - Fast-path model
34 *
35 * When the softmac fast-path is used, a dedicated lower-stream
36 * will be opened over the legacy device for each IP/ARP (upper-)stream
37 * over the softMAC, and all DLPI messages (including control messages
38 * and data messages) will be exchanged between the upper-stream and
39 * the corresponding lower-stream directly. Therefore, the data
40 * demultiplexing, filtering and classification processing will be done
41 * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
42 * no longer needed.
43 *
44 * - Slow-path model
45 *
46 * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
47 * not be bypassed to assure its function correctness. For example,
48 * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
49 * In this case, a shared lower-stream will be opened over the legacy
50 * device, which is responsible for implementing the GLDv3 callbacks
51 * and passing RAW data messages between the legacy devices and the GLDv3
52 * framework.
53 *
54 * By default, the softmac fast-path mode will be used to assure the
55 * performance; MAC clients will be able to request to disable the softmac
56 * fast-path mode to support certain features, and if that succeeds,
57 * the system will fallback to the slow-path softmac data-path model.
58 *
59 *
60 * The details of the softmac data fast-path model is stated as below
61 *
62 * 1. When a stream is opened on a softMAC, the softmac module will takes
63 * over the DLPI processing on this stream;
64 *
65 * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
66 * used by default, unless fast-path is disabled by any MAC client
67 * explicitly. The softmac module first identifies an IP/ARP stream
68 * by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
69 * if there is one, this stream is either an IP or an ARP stream
70 * and will use fast-path potentially;
71 *
72 * 3. When the softmac fast-path is used, an dedicated lower-stream will
73 * be setup for each IP/ARP stream (1-1 mapping). From that point on,
74 * all control and data messages will be exchanged between the IP/ARP
75 * upper-stream and the legacy device through this dedicated
76 * lower-stream. As a result, the DLS/MAC layer processing in GLDv3
77 * will be skipped, and this greatly improves the performance;
78 *
79 * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
80 * by a VNIC), all the IP/ARP upper streams will try to switch from
81 * the fast-path to the slow-path. The dedicated lower-stream will be
82 * destroyed, and all the control and data-messages will go through the
83 * existing GLDv3 code path and (in the end) the shared lower-stream;
84 *
85 * 5. On the other hand, when the last MAC client cancels its fast-path
86 * disable request, all the IP/ARP streams will try to switch back to
87 * the fast-path mode;
88 *
89 * Step 5 and 6 both rely on the data-path mode switching process
90 * described below:
91 *
92 * 1) To switch the softmac data-path mode (between fast-path and slow-path),
93 * softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
94 * upstream over each IP/ARP streams that needs data-path mode switching;
95 *
96 * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
97 * all the IP interfaces on the corresponding ill (IP Lower level
98 * structure), and bring up those interfaces over again; this will in
99 * turn cause the ARP to "replumb" the interface.
100 *
101 * During the replumb process, both IP and ARP will send downstream the
102 * necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
103 * the old state of the underlying softMAC, following with the necessary
104 * DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
105 * Between the cleanup and re-setup process, IP/ARP will also send down
106 * a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
107 * indicate the *switching point*;
108 *
109 * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
110 * creates or destroys the dedicated lower-stream (depending on which
111 * data-path mode the softMAC switches to), and change the softmac
112 * data-path mode. From then on, softmac will process all the succeeding
113 * control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
114 * messages) and data messages based on new data-path mode.
115 */
116
117 #include <sys/types.h>
118 #include <sys/disp.h>
119 #include <sys/callb.h>
120 #include <sys/sysmacros.h>
121 #include <sys/file.h>
122 #include <sys/vlan.h>
123 #include <sys/dld.h>
124 #include <sys/sockio.h>
125 #include <sys/softmac_impl.h>
126 #include <net/if.h>
127
128 static kmutex_t softmac_taskq_lock;
129 static kcondvar_t softmac_taskq_cv;
130 static list_t softmac_taskq_list; /* List of softmac_upper_t */
131 boolean_t softmac_taskq_quit;
132 boolean_t softmac_taskq_done;
133
134 static void softmac_taskq_dispatch();
135 static int softmac_fastpath_setup(softmac_upper_t *);
136 static mac_tx_cookie_t softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
137 uintptr_t, uint16_t);
138 static void softmac_datapath_switch_done(softmac_upper_t *);
139
140 void
141 softmac_fp_init()
142 {
143 mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
144 cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
145
146 softmac_taskq_quit = B_FALSE;
147 softmac_taskq_done = B_FALSE;
148 list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
149 offsetof(softmac_upper_t, su_taskq_list_node));
150 (void) thread_create(NULL, 0, softmac_taskq_dispatch, NULL, 0,
151 &p0, TS_RUN, minclsyspri);
152 }
153
154 void
155 softmac_fp_fini()
156 {
157 /*
158 * Request the softmac_taskq thread to quit and wait for it to be done.
159 */
160 mutex_enter(&softmac_taskq_lock);
161 softmac_taskq_quit = B_TRUE;
162 cv_signal(&softmac_taskq_cv);
163 while (!softmac_taskq_done)
164 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
165 mutex_exit(&softmac_taskq_lock);
166 list_destroy(&softmac_taskq_list);
167
168 mutex_destroy(&softmac_taskq_lock);
169 cv_destroy(&softmac_taskq_cv);
170 }
171
172 static boolean_t
173 check_ip_above(queue_t *q)
174 {
175 queue_t *next_q;
176 boolean_t ret = B_TRUE;
177
178 claimstr(q);
179 next_q = q->q_next;
180 if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
181 ret = B_FALSE;
182 releasestr(q);
183 return (ret);
184 }
185
186 /* ARGSUSED */
187 static int
188 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
189 {
190 switch (flags) {
191 case DLD_ENABLE:
192 mutex_enter(&sup->su_mutex);
193 break;
194 case DLD_DISABLE:
195 mutex_exit(&sup->su_mutex);
196 break;
197 case DLD_QUERY:
198 return (MUTEX_HELD(&sup->su_mutex));
199 }
200 return (0);
201 }
202
203 static mac_tx_notify_handle_t
204 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
205 {
206 ASSERT(MUTEX_HELD(&sup->su_mutex));
207
208 if (func != NULL) {
209 sup->su_tx_notify_func = func;
210 sup->su_tx_notify_arg = arg;
211 } else {
212 /*
213 * Wait for all tx_notify_func call to be done.
214 */
215 while (sup->su_tx_inprocess != 0)
216 cv_wait(&sup->su_cv, &sup->su_mutex);
217
218 sup->su_tx_notify_func = NULL;
219 sup->su_tx_notify_arg = NULL;
220 }
221 return ((mac_tx_notify_handle_t)sup);
222 }
223
224 static boolean_t
225 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
226 {
227 ASSERT(cookie == (mac_tx_cookie_t)sup);
228 return (sup->su_tx_busy);
229 }
230
231 static int
232 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
233 {
234 dld_capab_direct_t *direct = data;
235 softmac_lower_t *slp = sup->su_slp;
236
237 ASSERT(MUTEX_HELD(&sup->su_mutex));
238
239 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
240
241 switch (flags) {
242 case DLD_ENABLE:
243 if (sup->su_direct)
244 return (0);
245
246 sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
247 sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
248 slp->sl_rxinfo = &sup->su_direct_rxinfo;
249 direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
250 direct->di_tx_dh = sup;
251 direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
252 direct->di_tx_fctl_dh = sup;
253 direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
254 direct->di_tx_cb_dh = sup;
255 sup->su_direct = B_TRUE;
256 return (0);
257
258 case DLD_DISABLE:
259 if (!sup->su_direct)
260 return (0);
261
262 slp->sl_rxinfo = &sup->su_rxinfo;
263 sup->su_direct = B_FALSE;
264 return (0);
265 }
266 return (ENOTSUP);
267 }
268
269 static int
270 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
271 {
272 int err;
273
274 /*
275 * Don't enable direct callback capabilities unless the caller is
276 * the IP client. When a module is inserted in a stream (_I_INSERT)
277 * the stack initiates capability disable, but due to races, the
278 * module insertion may complete before the capability disable
279 * completes. So we limit the check to DLD_ENABLE case.
280 */
281 if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
282 !check_ip_above(sup->su_rq)) {
283 return (ENOTSUP);
284 }
285
286 switch (type) {
287 case DLD_CAPAB_DIRECT:
288 err = softmac_capab_direct(sup, data, flags);
289 break;
290
291 case DLD_CAPAB_PERIM:
292 err = softmac_capab_perim(sup, data, flags);
293 break;
294
295 default:
296 err = ENOTSUP;
297 break;
298 }
299 return (err);
300 }
301
302 static void
303 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
304 {
305 dl_capability_ack_t *dlap;
306 dl_capability_sub_t *dlsp;
307 t_uscalar_t subsize;
308 uint8_t *ptr;
309 queue_t *q = sup->su_wq;
310 mblk_t *mp1;
311 softmac_t *softmac = sup->su_softmac;
312 boolean_t dld_capable = B_FALSE;
313 boolean_t hcksum_capable = B_FALSE;
314 boolean_t zcopy_capable = B_FALSE;
315 boolean_t mdt_capable = B_FALSE;
316
317 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
318
319 /*
320 * Initially assume no capabilities.
321 */
322 subsize = 0;
323
324 /*
325 * Direct capability negotiation interface between IP and softmac
326 */
327 if (check_ip_above(sup->su_rq)) {
328 dld_capable = B_TRUE;
329 subsize += sizeof (dl_capability_sub_t) +
330 sizeof (dl_capab_dld_t);
331 }
332
333 /*
334 * Check if checksum offload is supported on this MAC.
335 */
336 if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
337 hcksum_capable = B_TRUE;
338 subsize += sizeof (dl_capability_sub_t) +
339 sizeof (dl_capab_hcksum_t);
340 }
341
342 /*
343 * Check if zerocopy is supported on this interface.
344 */
345 if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
346 zcopy_capable = B_TRUE;
347 subsize += sizeof (dl_capability_sub_t) +
348 sizeof (dl_capab_zerocopy_t);
349 }
350
351 if (softmac->smac_mdt) {
352 mdt_capable = B_TRUE;
353 subsize += sizeof (dl_capability_sub_t) +
354 sizeof (dl_capab_mdt_t);
355 }
356
357 /*
358 * If there are no capabilities to advertise or if we
359 * can't allocate a response, send a DL_ERROR_ACK.
360 */
361 if ((subsize == 0) || (mp1 = reallocb(mp,
362 sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
363 dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
364 return;
365 }
366
367 mp = mp1;
368 DB_TYPE(mp) = M_PROTO;
369 mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
370 bzero(mp->b_rptr, MBLKL(mp));
371 dlap = (dl_capability_ack_t *)mp->b_rptr;
372 dlap->dl_primitive = DL_CAPABILITY_ACK;
373 dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
374 dlap->dl_sub_length = subsize;
375 ptr = (uint8_t *)&dlap[1];
376
377 /*
378 * IP polling interface.
379 */
380 if (dld_capable) {
381 dl_capab_dld_t dld;
382
383 dlsp = (dl_capability_sub_t *)ptr;
384 dlsp->dl_cap = DL_CAPAB_DLD;
385 dlsp->dl_length = sizeof (dl_capab_dld_t);
386 ptr += sizeof (dl_capability_sub_t);
387
388 bzero(&dld, sizeof (dl_capab_dld_t));
389 dld.dld_version = DLD_CURRENT_VERSION;
390 dld.dld_capab = (uintptr_t)softmac_dld_capab;
391 dld.dld_capab_handle = (uintptr_t)sup;
392
393 dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
394 bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
395 ptr += sizeof (dl_capab_dld_t);
396 }
397
398 /*
399 * TCP/IP checksum offload.
400 */
401 if (hcksum_capable) {
402 dl_capab_hcksum_t hcksum;
403
404 dlsp = (dl_capability_sub_t *)ptr;
405
406 dlsp->dl_cap = DL_CAPAB_HCKSUM;
407 dlsp->dl_length = sizeof (dl_capab_hcksum_t);
408 ptr += sizeof (dl_capability_sub_t);
409
410 bzero(&hcksum, sizeof (dl_capab_hcksum_t));
411 hcksum.hcksum_version = HCKSUM_VERSION_1;
412 hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
413 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
414 bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
415 ptr += sizeof (dl_capab_hcksum_t);
416 }
417
418 /*
419 * Zero copy
420 */
421 if (zcopy_capable) {
422 dl_capab_zerocopy_t zcopy;
423
424 dlsp = (dl_capability_sub_t *)ptr;
425
426 dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
427 dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
428 ptr += sizeof (dl_capability_sub_t);
429
430 bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
431 zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
432 zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
433 dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
434 bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
435 ptr += sizeof (dl_capab_zerocopy_t);
436 }
437
438 /*
439 * MDT
440 */
441 if (mdt_capable) {
442 dl_capab_mdt_t mdt;
443
444 dlsp = (dl_capability_sub_t *)ptr;
445
446 dlsp->dl_cap = DL_CAPAB_MDT;
447 dlsp->dl_length = sizeof (dl_capab_mdt_t);
448 ptr += sizeof (dl_capability_sub_t);
449
450 bzero(&mdt, sizeof (dl_capab_mdt_t));
451 mdt.mdt_version = MDT_VERSION_2;
452 mdt.mdt_flags = DL_CAPAB_MDT_ENABLE;
453 mdt.mdt_hdr_head = softmac->smac_mdt_capab.mdt_hdr_head;
454 mdt.mdt_hdr_tail = softmac->smac_mdt_capab.mdt_hdr_tail;
455 mdt.mdt_max_pld = softmac->smac_mdt_capab.mdt_max_pld;
456 mdt.mdt_span_limit = softmac->smac_mdt_capab.mdt_span_limit;
457 dlcapabsetqid(&(mdt.mdt_mid), sup->su_rq);
458 bcopy(&mdt, ptr, sizeof (dl_capab_mdt_t));
459 ptr += sizeof (dl_capab_mdt_t);
460 }
461
462 ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
463 qreply(q, mp);
464 }
465
466 static void
467 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
468 {
469 dl_capability_req_t *dlp = (dl_capability_req_t *)mp->b_rptr;
470 dl_capability_sub_t *sp;
471 size_t size, len;
472 offset_t off, end;
473 t_uscalar_t dl_err;
474 queue_t *q = sup->su_wq;
475
476 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
477 if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
478 dl_err = DL_BADPRIM;
479 goto failed;
480 }
481
482 if (!sup->su_bound) {
483 dl_err = DL_OUTSTATE;
484 goto failed;
485 }
486
487 /*
488 * This request is overloaded. If there are no requested capabilities
489 * then we just want to acknowledge with all the capabilities we
490 * support. Otherwise we enable the set of capabilities requested.
491 */
492 if (dlp->dl_sub_length == 0) {
493 softmac_capability_advertise(sup, mp);
494 return;
495 }
496
497 if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
498 dl_err = DL_BADPRIM;
499 goto failed;
500 }
501
502 dlp->dl_primitive = DL_CAPABILITY_ACK;
503
504 off = dlp->dl_sub_offset;
505 len = dlp->dl_sub_length;
506
507 /*
508 * Walk the list of capabilities to be enabled.
509 */
510 for (end = off + len; off < end; ) {
511 sp = (dl_capability_sub_t *)(mp->b_rptr + off);
512 size = sizeof (dl_capability_sub_t) + sp->dl_length;
513
514 if (off + size > end ||
515 !IS_P2ALIGNED(off, sizeof (uint32_t))) {
516 dl_err = DL_BADPRIM;
517 goto failed;
518 }
519
520 switch (sp->dl_cap) {
521 /*
522 * TCP/IP checksum offload to hardware.
523 */
524 case DL_CAPAB_HCKSUM: {
525 dl_capab_hcksum_t *hcksump;
526 dl_capab_hcksum_t hcksum;
527
528 hcksump = (dl_capab_hcksum_t *)&sp[1];
529 /*
530 * Copy for alignment.
531 */
532 bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
533 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
534 bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
535 break;
536 }
537
538 default:
539 break;
540 }
541
542 off += size;
543 }
544 qreply(q, mp);
545 return;
546 failed:
547 dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
548 }
549
550 static void
551 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
552 {
553 softmac_lower_t *slp = sup->su_slp;
554 softmac_t *softmac = sup->su_softmac;
555 mblk_t *ackmp, *mp1;
556 int err;
557
558 if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
559 freemsg(mp);
560 return;
561 }
562
563 /*
564 * Allocate ackmp incase the underlying driver does not ack timely.
565 */
566 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
567 dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
568 return;
569 }
570
571 err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
572 if (ackmp != NULL) {
573 freemsg(mp1);
574 } else {
575 /*
576 * The driver does not ack timely.
577 */
578 ASSERT(err == ENOMSG);
579 ackmp = mp1;
580 }
581 if (err != 0)
582 goto failed;
583
584 /*
585 * Enable capabilities the underlying driver claims to support.
586 */
587 if ((err = softmac_capab_enable(slp)) != 0)
588 goto failed;
589
590 /*
591 * Check whether this softmac is already marked as exclusively used,
592 * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
593 */
594 mutex_enter(&softmac->smac_active_mutex);
595 if (softmac->smac_active) {
596 mutex_exit(&softmac->smac_active_mutex);
597 err = EBUSY;
598 goto failed;
599 }
600 softmac->smac_nactive++;
601 sup->su_active = B_TRUE;
602 mutex_exit(&softmac->smac_active_mutex);
603 sup->su_bound = B_TRUE;
604
605 qreply(sup->su_wq, ackmp);
606 return;
607 failed:
608 if (err != 0) {
609 dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
610 return;
611 }
612 }
613
614 static void
615 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
616 {
617 softmac_lower_t *slp = sup->su_slp;
618 softmac_t *softmac = sup->su_softmac;
619 mblk_t *ackmp, *mp1;
620 int err;
621
622 if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
623 freemsg(mp);
624 return;
625 }
626
627 if (!sup->su_bound) {
628 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
629 return;
630 }
631
632 /*
633 * Allocate ackmp incase the underlying driver does not ack timely.
634 */
635 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
636 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
637 return;
638 }
639
640 err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
641 if (ackmp != NULL) {
642 freemsg(mp1);
643 } else {
644 /*
645 * The driver does not ack timely.
646 */
647 ASSERT(err == ENOMSG);
648 ackmp = mp1;
649 }
650 if (err != 0) {
651 dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
652 return;
653 }
654
655 sup->su_bound = B_FALSE;
656
657 mutex_enter(&softmac->smac_active_mutex);
658 if (sup->su_active) {
659 ASSERT(!softmac->smac_active);
660 softmac->smac_nactive--;
661 sup->su_active = B_FALSE;
662 }
663 mutex_exit(&softmac->smac_active_mutex);
664
665 done:
666 qreply(sup->su_wq, ackmp);
667 }
668
669 /*
670 * Process the non-data mblk.
671 */
672 static void
673 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
674 {
675 softmac_t *softmac = sup->su_softmac;
676 softmac_lower_t *slp = sup->su_slp;
677 unsigned char dbtype;
678 t_uscalar_t prim;
679
680 dbtype = DB_TYPE(mp);
681 sup->su_is_arp = 0;
682 switch (dbtype) {
683 case M_CTL:
684 sup->su_is_arp = 1;
685 /* FALLTHROUGH */
686 case M_IOCTL: {
687 uint32_t expected_mode;
688
689 if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
690 break;
691
692 /*
693 * Nak the M_IOCTL based on the STREAMS specification.
694 */
695 if (dbtype == M_IOCTL)
696 miocnak(sup->su_wq, mp, 0, EINVAL);
697 else
698 freemsg(mp);
699
700 /*
701 * This stream is either IP or ARP. See whether
702 * we need to setup a dedicated-lower-stream for it.
703 */
704 mutex_enter(&softmac->smac_fp_mutex);
705
706 expected_mode = DATAPATH_MODE(softmac);
707 if (expected_mode == SOFTMAC_SLOWPATH)
708 sup->su_mode = SOFTMAC_SLOWPATH;
709 list_insert_head(&softmac->smac_sup_list, sup);
710 mutex_exit(&softmac->smac_fp_mutex);
711
712 /*
713 * Setup the fast-path dedicated lower stream if fast-path
714 * is expected. Note that no lock is held here, and if
715 * smac_expected_mode is changed from SOFTMAC_FASTPATH to
716 * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
717 * data-path switching would already be queued and will
718 * be processed by softmac_wput_single_nondata() later.
719 */
720 if (expected_mode == SOFTMAC_FASTPATH)
721 (void) softmac_fastpath_setup(sup);
722 return;
723 }
724 case M_PROTO:
725 case M_PCPROTO:
726 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
727 freemsg(mp);
728 return;
729 }
730 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
731 switch (prim) {
732 case DL_NOTIFY_IND:
733 if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
734 ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
735 DL_NOTE_REPLUMB) {
736 freemsg(mp);
737 return;
738 }
739 /*
740 * This DL_NOTE_REPLUMB message is initiated
741 * and queued by the softmac itself, when the
742 * sup is trying to switching its datapath mode
743 * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
744 * Send this message upstream.
745 */
746 qreply(sup->su_wq, mp);
747 return;
748 case DL_NOTIFY_CONF:
749 if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
750 ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
751 DL_NOTE_REPLUMB_DONE) {
752 freemsg(mp);
753 return;
754 }
755 /*
756 * This is an indication from IP/ARP that the
757 * fastpath->slowpath switch is done.
758 */
759 freemsg(mp);
760 softmac_datapath_switch_done(sup);
761 return;
762 }
763 break;
764 }
765
766 /*
767 * No need to hold lock to check su_mode, since su_mode updating only
768 * operation is is serialized by softmac_wput_nondata_task().
769 */
770 if (sup->su_mode != SOFTMAC_FASTPATH) {
771 (void) dld_wput(sup->su_wq, mp);
772 return;
773 }
774
775 /*
776 * Fastpath non-data message processing. Most of non-data messages
777 * can be directly passed down to the dedicated-lower-stream, aside
778 * from the following M_PROTO/M_PCPROTO messages.
779 */
780 switch (dbtype) {
781 case M_PROTO:
782 case M_PCPROTO:
783 switch (prim) {
784 case DL_BIND_REQ:
785 softmac_bind_req(sup, mp);
786 break;
787 case DL_UNBIND_REQ:
788 softmac_unbind_req(sup, mp);
789 break;
790 case DL_CAPABILITY_REQ:
791 softmac_capability_req(sup, mp);
792 break;
793 default:
794 putnext(slp->sl_wq, mp);
795 break;
796 }
797 break;
798 default:
799 putnext(slp->sl_wq, mp);
800 break;
801 }
802 }
803
804 /*
805 * The worker thread which processes non-data messages. Note we only process
806 * one message at one time in order to be able to "flush" the queued message
807 * and serialize the processing.
808 */
809 static void
810 softmac_wput_nondata_task(void *arg)
811 {
812 softmac_upper_t *sup = arg;
813 mblk_t *mp;
814
815 mutex_enter(&sup->su_disp_mutex);
816
817 while (sup->su_pending_head != NULL) {
818 if (sup->su_closing)
819 break;
820
821 SOFTMAC_DQ_PENDING(sup, &mp);
822 mutex_exit(&sup->su_disp_mutex);
823 softmac_wput_single_nondata(sup, mp);
824 mutex_enter(&sup->su_disp_mutex);
825 }
826
827 /*
828 * If the stream is closing, flush all queued messages and inform
829 * the stream to be closed.
830 */
831 freemsgchain(sup->su_pending_head);
832 sup->su_pending_head = sup->su_pending_tail = NULL;
833 sup->su_dlpi_pending = B_FALSE;
834 cv_signal(&sup->su_disp_cv);
835 mutex_exit(&sup->su_disp_mutex);
836 }
837
838 /*
839 * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
840 * This thread is started when the softmac module is first loaded.
841 */
842 static void
843 softmac_taskq_dispatch(void)
844 {
845 callb_cpr_t cprinfo;
846 softmac_upper_t *sup;
847
848 CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
849 "softmac_taskq_dispatch");
850 mutex_enter(&softmac_taskq_lock);
851
852 while (!softmac_taskq_quit) {
853 sup = list_head(&softmac_taskq_list);
854 while (sup != NULL) {
855 list_remove(&softmac_taskq_list, sup);
856 sup->su_taskq_scheduled = B_FALSE;
857 mutex_exit(&softmac_taskq_lock);
858 VERIFY(taskq_dispatch(system_taskq,
859 softmac_wput_nondata_task, sup, TQ_SLEEP) !=
860 TASKQID_INVALID);
861 mutex_enter(&softmac_taskq_lock);
862 sup = list_head(&softmac_taskq_list);
863 }
864
865 CALLB_CPR_SAFE_BEGIN(&cprinfo);
866 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
867 CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
868 }
869
870 softmac_taskq_done = B_TRUE;
871 cv_signal(&softmac_taskq_cv);
872 CALLB_CPR_EXIT(&cprinfo);
873 thread_exit();
874 }
875
876 void
877 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
878 {
879 /*
880 * The processing of the message might block. Enqueue the
881 * message for later processing.
882 */
883 mutex_enter(&sup->su_disp_mutex);
884
885 if (sup->su_closing) {
886 mutex_exit(&sup->su_disp_mutex);
887 freemsg(mp);
888 return;
889 }
890
891 SOFTMAC_EQ_PENDING(sup, mp);
892
893 if (sup->su_dlpi_pending) {
894 mutex_exit(&sup->su_disp_mutex);
895 return;
896 }
897 sup->su_dlpi_pending = B_TRUE;
898 mutex_exit(&sup->su_disp_mutex);
899
900 if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
901 sup, TQ_NOSLEEP) != TASKQID_INVALID) {
902 return;
903 }
904
905 mutex_enter(&softmac_taskq_lock);
906 if (!sup->su_taskq_scheduled) {
907 list_insert_tail(&softmac_taskq_list, sup);
908 cv_signal(&softmac_taskq_cv);
909 }
910 sup->su_taskq_scheduled = B_TRUE;
911 mutex_exit(&softmac_taskq_lock);
912 }
913
914 /*
915 * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
916 */
917 static int
918 softmac_fastpath_setup(softmac_upper_t *sup)
919 {
920 softmac_t *softmac = sup->su_softmac;
921 softmac_lower_t *slp;
922 int err;
923
924 err = softmac_lower_setup(softmac, sup, &slp);
925
926 mutex_enter(&sup->su_mutex);
927 /*
928 * Wait for all data messages to be processed so that we can change
929 * the su_mode.
930 */
931 while (sup->su_tx_inprocess != 0)
932 cv_wait(&sup->su_cv, &sup->su_mutex);
933
934 ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
935 ASSERT(sup->su_slp == NULL);
936 if (err != 0) {
937 sup->su_mode = SOFTMAC_SLOWPATH;
938 } else {
939 sup->su_slp = slp;
940 sup->su_mode = SOFTMAC_FASTPATH;
941 }
942 mutex_exit(&sup->su_mutex);
943 return (err);
944 }
945
946 /*
947 * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
948 */
949 static void
950 softmac_fastpath_tear(softmac_upper_t *sup)
951 {
952 mutex_enter(&sup->su_mutex);
953 /*
954 * Wait for all data messages in the dedicated-lower-stream
955 * to be processed.
956 */
957 while (sup->su_tx_inprocess != 0)
958 cv_wait(&sup->su_cv, &sup->su_mutex);
959
960 /*
961 * Note that this function is called either when the stream is closed,
962 * or the stream is unbound (fastpath-slowpath-switch). Therefore,
963 * No need to call the tx_notify callback.
964 */
965 sup->su_tx_notify_func = NULL;
966 sup->su_tx_notify_arg = NULL;
967 if (sup->su_tx_busy) {
968 ASSERT(sup->su_tx_flow_mp == NULL);
969 VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
970 sup->su_tx_busy = B_FALSE;
971 }
972
973 sup->su_mode = SOFTMAC_SLOWPATH;
974
975 /*
976 * Destroy the dedicated-lower-stream. Note that slp is destroyed
977 * when lh is closed.
978 */
979 (void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
980 sup->su_slp = NULL;
981 mutex_exit(&sup->su_mutex);
982 }
983
984 void
985 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
986 {
987 /*
988 * No lock is required to access the su_mode field since the data
989 * traffic is quiesce by IP when the data-path mode is in the
990 * process of switching.
991 */
992 if (sup->su_mode != SOFTMAC_FASTPATH)
993 (void) dld_wput(sup->su_wq, mp);
994 else
995 (void) softmac_fastpath_wput_data(sup, mp, NULL, 0);
996 }
997
998 /*ARGSUSED*/
999 static mac_tx_cookie_t
1000 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
1001 uint16_t flag)
1002 {
1003 queue_t *wq = sup->su_slp->sl_wq;
1004
1005 /*
1006 * This function is called from IP, only the MAC_DROP_ON_NO_DESC
1007 * flag can be specified.
1008 */
1009 ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
1010 ASSERT(mp->b_next == NULL);
1011
1012 /*
1013 * Check wether the dedicated-lower-stream is able to handle more
1014 * messages, and enable the flow-control if it is not.
1015 *
1016 * Note that in order not to introduce any packet reordering, we
1017 * always send the message down to the dedicated-lower-stream:
1018 *
1019 * If the flow-control is already enabled, but we still get
1020 * the messages from the upper-stream, it means that the upper
1021 * stream does not respect STREAMS flow-control (e.g., TCP). Simply
1022 * pass the message down to the lower-stream in that case.
1023 */
1024 if (SOFTMAC_CANPUTNEXT(wq)) {
1025 putnext(wq, mp);
1026 return (NULL);
1027 }
1028
1029 if (sup->su_tx_busy) {
1030 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1031 freemsg(mp);
1032 else
1033 putnext(wq, mp);
1034 return ((mac_tx_cookie_t)sup);
1035 }
1036
1037 mutex_enter(&sup->su_mutex);
1038 if (!sup->su_tx_busy) {
1039 /*
1040 * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1041 * called when the flow control can be disabled. Otherwise,
1042 * put the tx_flow_mp into the wq to make use of the old
1043 * streams flow control.
1044 */
1045 ASSERT(sup->su_tx_flow_mp != NULL);
1046 (void) putq(sup->su_wq, sup->su_tx_flow_mp);
1047 sup->su_tx_flow_mp = NULL;
1048 sup->su_tx_busy = B_TRUE;
1049 qenable(wq);
1050 }
1051 mutex_exit(&sup->su_mutex);
1052
1053 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1054 freemsg(mp);
1055 else
1056 putnext(wq, mp);
1057 return ((mac_tx_cookie_t)sup);
1058 }
1059
1060 boolean_t
1061 softmac_active_set(void *arg)
1062 {
1063 softmac_t *softmac = arg;
1064
1065 mutex_enter(&softmac->smac_active_mutex);
1066 if (softmac->smac_nactive != 0) {
1067 mutex_exit(&softmac->smac_active_mutex);
1068 return (B_FALSE);
1069 }
1070 softmac->smac_active = B_TRUE;
1071 mutex_exit(&softmac->smac_active_mutex);
1072 return (B_TRUE);
1073 }
1074
1075 void
1076 softmac_active_clear(void *arg)
1077 {
1078 softmac_t *softmac = arg;
1079
1080 mutex_enter(&softmac->smac_active_mutex);
1081 ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1082 softmac->smac_active = B_FALSE;
1083 mutex_exit(&softmac->smac_active_mutex);
1084 }
1085
1086 /*
1087 * Disable/reenable fastpath on given softmac. This request could come from a
1088 * MAC client or directly from administrators.
1089 */
1090 int
1091 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1092 {
1093 softmac_upper_t *sup;
1094 mblk_t *head = NULL, *tail = NULL, *mp;
1095 list_t reqlist;
1096 softmac_switch_req_t *req;
1097 uint32_t current_mode, expected_mode;
1098 int err = 0;
1099
1100 mutex_enter(&softmac->smac_fp_mutex);
1101
1102 current_mode = DATAPATH_MODE(softmac);
1103 if (admin) {
1104 if (softmac->smac_fastpath_admin_disabled == disable) {
1105 mutex_exit(&softmac->smac_fp_mutex);
1106 return (0);
1107 }
1108 softmac->smac_fastpath_admin_disabled = disable;
1109 } else if (disable) {
1110 softmac->smac_fp_disable_clients++;
1111 } else {
1112 ASSERT(softmac->smac_fp_disable_clients != 0);
1113 softmac->smac_fp_disable_clients--;
1114 }
1115
1116 expected_mode = DATAPATH_MODE(softmac);
1117 if (current_mode == expected_mode) {
1118 mutex_exit(&softmac->smac_fp_mutex);
1119 return (0);
1120 }
1121
1122 /*
1123 * The expected mode is different from whatever datapath mode
1124 * this softmac is expected from last request, enqueue the data-path
1125 * switch request.
1126 */
1127 list_create(&reqlist, sizeof (softmac_switch_req_t),
1128 offsetof(softmac_switch_req_t, ssq_req_list_node));
1129
1130 /*
1131 * Allocate all DL_NOTIFY_IND messages and request structures that
1132 * are required to switch each IP/ARP stream to the expected mode.
1133 */
1134 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1135 sup = list_next(&softmac->smac_sup_list, sup)) {
1136 dl_notify_ind_t *dlip;
1137
1138 req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1139 if (req == NULL)
1140 break;
1141
1142 req->ssq_expected_mode = expected_mode;
1143 if (sup->su_is_arp) {
1144 list_insert_tail(&reqlist, req);
1145 continue;
1146 }
1147 /*
1148 * Allocate the DL_NOTE_REPLUMB message.
1149 */
1150 if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1151 kmem_free(req, sizeof (softmac_switch_req_t));
1152 break;
1153 }
1154
1155 list_insert_tail(&reqlist, req);
1156
1157 mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1158 mp->b_datap->db_type = M_PROTO;
1159 bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1160 dlip = (dl_notify_ind_t *)mp->b_rptr;
1161 dlip->dl_primitive = DL_NOTIFY_IND;
1162 dlip->dl_notification = DL_NOTE_REPLUMB;
1163 if (head == NULL) {
1164 head = tail = mp;
1165 } else {
1166 tail->b_next = mp;
1167 tail = mp;
1168 }
1169 }
1170
1171 /*
1172 * Note that it is fine if the expected data-path mode is fast-path
1173 * and some of streams fails to switch. Only return failure if we
1174 * are expected to switch to the slow-path.
1175 */
1176 if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1177 err = ENOMEM;
1178 goto fail;
1179 }
1180
1181 /*
1182 * Start switching for each IP/ARP stream. The switching operation
1183 * will eventually succeed and there is no need to wait for it
1184 * to finish.
1185 */
1186 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1187 sup = list_next(&softmac->smac_sup_list, sup)) {
1188 if (!sup->su_is_arp) {
1189 mp = head->b_next;
1190 head->b_next = NULL;
1191 softmac_wput_nondata(sup, head);
1192 head = mp;
1193 }
1194 /*
1195 * Add the switch request to the requests list of the stream.
1196 */
1197 req = list_head(&reqlist);
1198 ASSERT(req != NULL);
1199 list_remove(&reqlist, req);
1200 list_insert_tail(&sup->su_req_list, req);
1201 }
1202
1203 mutex_exit(&softmac->smac_fp_mutex);
1204 ASSERT(list_is_empty(&reqlist));
1205 list_destroy(&reqlist);
1206 return (0);
1207 fail:
1208 if (admin) {
1209 softmac->smac_fastpath_admin_disabled = !disable;
1210 } else if (disable) {
1211 softmac->smac_fp_disable_clients--;
1212 } else {
1213 softmac->smac_fp_disable_clients++;
1214 }
1215
1216 mutex_exit(&softmac->smac_fp_mutex);
1217 while ((req = list_head(&reqlist)) != NULL) {
1218 list_remove(&reqlist, req);
1219 kmem_free(req, sizeof (softmac_switch_req_t));
1220 }
1221 freemsgchain(head);
1222 list_destroy(&reqlist);
1223 return (err);
1224 }
1225
1226 int
1227 softmac_fastpath_disable(void *arg)
1228 {
1229 return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1230 }
1231
1232 void
1233 softmac_fastpath_enable(void *arg)
1234 {
1235 VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1236 B_FALSE) == 0);
1237 }
1238
1239 void
1240 softmac_upperstream_close(softmac_upper_t *sup)
1241 {
1242 softmac_t *softmac = sup->su_softmac;
1243 softmac_switch_req_t *req;
1244
1245 mutex_enter(&softmac->smac_fp_mutex);
1246
1247 if (sup->su_mode == SOFTMAC_FASTPATH)
1248 softmac_fastpath_tear(sup);
1249
1250 if (sup->su_mode != SOFTMAC_UNKNOWN) {
1251 list_remove(&softmac->smac_sup_list, sup);
1252 sup->su_mode = SOFTMAC_UNKNOWN;
1253 }
1254
1255 /*
1256 * Cleanup all the switch requests queueed on this stream.
1257 */
1258 while ((req = list_head(&sup->su_req_list)) != NULL) {
1259 list_remove(&sup->su_req_list, req);
1260 kmem_free(req, sizeof (softmac_switch_req_t));
1261 }
1262 mutex_exit(&softmac->smac_fp_mutex);
1263 }
1264
1265 /*
1266 * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1267 * stream from the fastpath mode to the slowpath mode.
1268 */
1269 static void
1270 softmac_datapath_switch_done(softmac_upper_t *sup)
1271 {
1272 softmac_t *softmac = sup->su_softmac;
1273 softmac_switch_req_t *req;
1274 uint32_t expected_mode;
1275
1276 mutex_enter(&softmac->smac_fp_mutex);
1277 req = list_head(&sup->su_req_list);
1278 list_remove(&sup->su_req_list, req);
1279 expected_mode = req->ssq_expected_mode;
1280 kmem_free(req, sizeof (softmac_switch_req_t));
1281
1282 if (expected_mode == sup->su_mode) {
1283 mutex_exit(&softmac->smac_fp_mutex);
1284 return;
1285 }
1286
1287 ASSERT(!sup->su_bound);
1288 mutex_exit(&softmac->smac_fp_mutex);
1289
1290 /*
1291 * It is fine if the expected mode is fast-path and we fail
1292 * to enable fastpath on this stream.
1293 */
1294 if (expected_mode == SOFTMAC_SLOWPATH)
1295 softmac_fastpath_tear(sup);
1296 else
1297 (void) softmac_fastpath_setup(sup);
1298 }