1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Softmac data-path switching:
28 *
29 * - Fast-path model
30 *
31 * When the softmac fast-path is used, a dedicated lower-stream
32 * will be opened over the legacy device for each IP/ARP (upper-)stream
33 * over the softMAC, and all DLPI messages (including control messages
34 * and data messages) will be exchanged between the upper-stream and
35 * the corresponding lower-stream directly. Therefore, the data
36 * demultiplexing, filtering and classification processing will be done
37 * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
38 * no longer needed.
39 *
40 * - Slow-path model
41 *
42 * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
43 * not be bypassed to assure its function correctness. For example,
44 * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
45 * In this case, a shared lower-stream will be opened over the legacy
46 * device, which is responsible for implementing the GLDv3 callbacks
47 * and passing RAW data messages between the legacy devices and the GLDv3
48 * framework.
49 *
50 * By default, the softmac fast-path mode will be used to assure the
51 * performance; MAC clients will be able to request to disable the softmac
52 * fast-path mode to support certain features, and if that succeeds,
53 * the system will fallback to the slow-path softmac data-path model.
54 *
55 *
56 * The details of the softmac data fast-path model is stated as below
57 *
58 * 1. When a stream is opened on a softMAC, the softmac module will takes
59 * over the DLPI processing on this stream;
60 *
61 * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
62 * used by default, unless fast-path is disabled by any MAC client
63 * explicitly. The softmac module first identifies an IP/ARP stream
64 * by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
65 * if there is one, this stream is either an IP or an ARP stream
66 * and will use fast-path potentially;
67 *
68 * 3. When the softmac fast-path is used, an dedicated lower-stream will
69 * be setup for each IP/ARP stream (1-1 mapping). From that point on,
70 * all control and data messages will be exchanged between the IP/ARP
71 * upper-stream and the legacy device through this dedicated
72 * lower-stream. As a result, the DLS/MAC layer processing in GLDv3
73 * will be skipped, and this greatly improves the performance;
74 *
75 * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
76 * by a VNIC), all the IP/ARP upper streams will try to switch from
77 * the fast-path to the slow-path. The dedicated lower-stream will be
78 * destroyed, and all the control and data-messages will go through the
79 * existing GLDv3 code path and (in the end) the shared lower-stream;
80 *
81 * 5. On the other hand, when the last MAC client cancels its fast-path
82 * disable request, all the IP/ARP streams will try to switch back to
83 * the fast-path mode;
84 *
85 * Step 5 and 6 both rely on the data-path mode switching process
86 * described below:
87 *
88 * 1) To switch the softmac data-path mode (between fast-path and slow-path),
89 * softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
90 * upstream over each IP/ARP streams that needs data-path mode switching;
91 *
92 * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
93 * all the IP interfaces on the corresponding ill (IP Lower level
94 * structure), and bring up those interfaces over again; this will in
95 * turn cause the ARP to "replumb" the interface.
96 *
97 * During the replumb process, both IP and ARP will send downstream the
98 * necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
99 * the old state of the underlying softMAC, following with the necessary
100 * DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
101 * Between the cleanup and re-setup process, IP/ARP will also send down
102 * a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
103 * indicate the *switching point*;
104 *
105 * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
106 * creates or destroys the dedicated lower-stream (depending on which
107 * data-path mode the softMAC switches to), and change the softmac
108 * data-path mode. From then on, softmac will process all the succeeding
109 * control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
110 * messages) and data messages based on new data-path mode.
111 */
112
113 #include <sys/types.h>
114 #include <sys/disp.h>
115 #include <sys/callb.h>
116 #include <sys/sysmacros.h>
117 #include <sys/file.h>
118 #include <sys/vlan.h>
119 #include <sys/dld.h>
120 #include <sys/sockio.h>
121 #include <sys/softmac_impl.h>
122 #include <net/if.h>
123
124 static kmutex_t softmac_taskq_lock;
125 static kcondvar_t softmac_taskq_cv;
126 static list_t softmac_taskq_list; /* List of softmac_upper_t */
127 boolean_t softmac_taskq_quit;
128 boolean_t softmac_taskq_done;
129
130 static void softmac_taskq_dispatch();
131 static int softmac_fastpath_setup(softmac_upper_t *);
132 static mac_tx_cookie_t softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
133 uintptr_t, uint16_t);
134 static void softmac_datapath_switch_done(softmac_upper_t *);
135
136 void
137 softmac_fp_init()
138 {
139 mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
140 cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
141
142 softmac_taskq_quit = B_FALSE;
143 softmac_taskq_done = B_FALSE;
144 list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
145 offsetof(softmac_upper_t, su_taskq_list_node));
146 (void) thread_create(NULL, 0, softmac_taskq_dispatch, NULL, 0,
147 &p0, TS_RUN, minclsyspri);
148 }
149
150 void
151 softmac_fp_fini()
152 {
153 /*
154 * Request the softmac_taskq thread to quit and wait for it to be done.
155 */
156 mutex_enter(&softmac_taskq_lock);
157 softmac_taskq_quit = B_TRUE;
158 cv_signal(&softmac_taskq_cv);
159 while (!softmac_taskq_done)
160 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
161 mutex_exit(&softmac_taskq_lock);
162 list_destroy(&softmac_taskq_list);
163
164 mutex_destroy(&softmac_taskq_lock);
165 cv_destroy(&softmac_taskq_cv);
166 }
167
168 static boolean_t
169 check_ip_above(queue_t *q)
170 {
171 queue_t *next_q;
172 boolean_t ret = B_TRUE;
173
174 claimstr(q);
175 next_q = q->q_next;
176 if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
177 ret = B_FALSE;
178 releasestr(q);
179 return (ret);
180 }
181
182 /* ARGSUSED */
183 static int
184 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
185 {
186 switch (flags) {
187 case DLD_ENABLE:
188 mutex_enter(&sup->su_mutex);
189 break;
190 case DLD_DISABLE:
191 mutex_exit(&sup->su_mutex);
192 break;
193 case DLD_QUERY:
194 return (MUTEX_HELD(&sup->su_mutex));
195 }
196 return (0);
197 }
198
199 static mac_tx_notify_handle_t
200 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
201 {
202 ASSERT(MUTEX_HELD(&sup->su_mutex));
203
204 if (func != NULL) {
205 sup->su_tx_notify_func = func;
206 sup->su_tx_notify_arg = arg;
207 } else {
208 /*
209 * Wait for all tx_notify_func call to be done.
210 */
211 while (sup->su_tx_inprocess != 0)
212 cv_wait(&sup->su_cv, &sup->su_mutex);
213
214 sup->su_tx_notify_func = NULL;
215 sup->su_tx_notify_arg = NULL;
216 }
217 return ((mac_tx_notify_handle_t)sup);
218 }
219
220 static boolean_t
221 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
222 {
223 ASSERT(cookie == (mac_tx_cookie_t)sup);
224 return (sup->su_tx_busy);
225 }
226
227 static int
228 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
229 {
230 dld_capab_direct_t *direct = data;
231 softmac_lower_t *slp = sup->su_slp;
232
233 ASSERT(MUTEX_HELD(&sup->su_mutex));
234
235 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
236
237 switch (flags) {
238 case DLD_ENABLE:
239 if (sup->su_direct)
240 return (0);
241
242 sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
243 sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
244 slp->sl_rxinfo = &sup->su_direct_rxinfo;
245 direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
246 direct->di_tx_dh = sup;
247 direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
248 direct->di_tx_fctl_dh = sup;
249 direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
250 direct->di_tx_cb_dh = sup;
251 sup->su_direct = B_TRUE;
252 return (0);
253
254 case DLD_DISABLE:
255 if (!sup->su_direct)
256 return (0);
257
258 slp->sl_rxinfo = &sup->su_rxinfo;
259 sup->su_direct = B_FALSE;
260 return (0);
261 }
262 return (ENOTSUP);
263 }
264
265 static int
266 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
267 {
268 int err;
269
270 /*
271 * Don't enable direct callback capabilities unless the caller is
272 * the IP client. When a module is inserted in a stream (_I_INSERT)
273 * the stack initiates capability disable, but due to races, the
274 * module insertion may complete before the capability disable
275 * completes. So we limit the check to DLD_ENABLE case.
276 */
277 if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
278 !check_ip_above(sup->su_rq)) {
279 return (ENOTSUP);
280 }
281
282 switch (type) {
283 case DLD_CAPAB_DIRECT:
284 err = softmac_capab_direct(sup, data, flags);
285 break;
286
287 case DLD_CAPAB_PERIM:
288 err = softmac_capab_perim(sup, data, flags);
289 break;
290
291 default:
292 err = ENOTSUP;
293 break;
294 }
295 return (err);
296 }
297
298 static void
299 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
300 {
301 dl_capability_ack_t *dlap;
302 dl_capability_sub_t *dlsp;
303 t_uscalar_t subsize;
304 uint8_t *ptr;
305 queue_t *q = sup->su_wq;
306 mblk_t *mp1;
307 softmac_t *softmac = sup->su_softmac;
308 boolean_t dld_capable = B_FALSE;
309 boolean_t hcksum_capable = B_FALSE;
310 boolean_t zcopy_capable = B_FALSE;
311 boolean_t mdt_capable = B_FALSE;
312
313 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
314
315 /*
316 * Initially assume no capabilities.
317 */
318 subsize = 0;
319
320 /*
321 * Direct capability negotiation interface between IP and softmac
322 */
323 if (check_ip_above(sup->su_rq)) {
324 dld_capable = B_TRUE;
325 subsize += sizeof (dl_capability_sub_t) +
326 sizeof (dl_capab_dld_t);
327 }
328
329 /*
330 * Check if checksum offload is supported on this MAC.
331 */
332 if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
333 hcksum_capable = B_TRUE;
334 subsize += sizeof (dl_capability_sub_t) +
335 sizeof (dl_capab_hcksum_t);
336 }
337
338 /*
339 * Check if zerocopy is supported on this interface.
340 */
341 if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
342 zcopy_capable = B_TRUE;
343 subsize += sizeof (dl_capability_sub_t) +
344 sizeof (dl_capab_zerocopy_t);
345 }
346
347 if (softmac->smac_mdt) {
348 mdt_capable = B_TRUE;
349 subsize += sizeof (dl_capability_sub_t) +
350 sizeof (dl_capab_mdt_t);
351 }
352
353 /*
354 * If there are no capabilities to advertise or if we
355 * can't allocate a response, send a DL_ERROR_ACK.
356 */
357 if ((subsize == 0) || (mp1 = reallocb(mp,
358 sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
359 dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
360 return;
361 }
362
363 mp = mp1;
364 DB_TYPE(mp) = M_PROTO;
365 mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
366 bzero(mp->b_rptr, MBLKL(mp));
367 dlap = (dl_capability_ack_t *)mp->b_rptr;
368 dlap->dl_primitive = DL_CAPABILITY_ACK;
369 dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
370 dlap->dl_sub_length = subsize;
371 ptr = (uint8_t *)&dlap[1];
372
373 /*
374 * IP polling interface.
375 */
376 if (dld_capable) {
377 dl_capab_dld_t dld;
378
379 dlsp = (dl_capability_sub_t *)ptr;
380 dlsp->dl_cap = DL_CAPAB_DLD;
381 dlsp->dl_length = sizeof (dl_capab_dld_t);
382 ptr += sizeof (dl_capability_sub_t);
383
384 bzero(&dld, sizeof (dl_capab_dld_t));
385 dld.dld_version = DLD_CURRENT_VERSION;
386 dld.dld_capab = (uintptr_t)softmac_dld_capab;
387 dld.dld_capab_handle = (uintptr_t)sup;
388
389 dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
390 bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
391 ptr += sizeof (dl_capab_dld_t);
392 }
393
394 /*
395 * TCP/IP checksum offload.
396 */
397 if (hcksum_capable) {
398 dl_capab_hcksum_t hcksum;
399
400 dlsp = (dl_capability_sub_t *)ptr;
401
402 dlsp->dl_cap = DL_CAPAB_HCKSUM;
403 dlsp->dl_length = sizeof (dl_capab_hcksum_t);
404 ptr += sizeof (dl_capability_sub_t);
405
406 bzero(&hcksum, sizeof (dl_capab_hcksum_t));
407 hcksum.hcksum_version = HCKSUM_VERSION_1;
408 hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
409 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
410 bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
411 ptr += sizeof (dl_capab_hcksum_t);
412 }
413
414 /*
415 * Zero copy
416 */
417 if (zcopy_capable) {
418 dl_capab_zerocopy_t zcopy;
419
420 dlsp = (dl_capability_sub_t *)ptr;
421
422 dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
423 dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
424 ptr += sizeof (dl_capability_sub_t);
425
426 bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
427 zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
428 zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
429 dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
430 bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
431 ptr += sizeof (dl_capab_zerocopy_t);
432 }
433
434 /*
435 * MDT
436 */
437 if (mdt_capable) {
438 dl_capab_mdt_t mdt;
439
440 dlsp = (dl_capability_sub_t *)ptr;
441
442 dlsp->dl_cap = DL_CAPAB_MDT;
443 dlsp->dl_length = sizeof (dl_capab_mdt_t);
444 ptr += sizeof (dl_capability_sub_t);
445
446 bzero(&mdt, sizeof (dl_capab_mdt_t));
447 mdt.mdt_version = MDT_VERSION_2;
448 mdt.mdt_flags = DL_CAPAB_MDT_ENABLE;
449 mdt.mdt_hdr_head = softmac->smac_mdt_capab.mdt_hdr_head;
450 mdt.mdt_hdr_tail = softmac->smac_mdt_capab.mdt_hdr_tail;
451 mdt.mdt_max_pld = softmac->smac_mdt_capab.mdt_max_pld;
452 mdt.mdt_span_limit = softmac->smac_mdt_capab.mdt_span_limit;
453 dlcapabsetqid(&(mdt.mdt_mid), sup->su_rq);
454 bcopy(&mdt, ptr, sizeof (dl_capab_mdt_t));
455 ptr += sizeof (dl_capab_mdt_t);
456 }
457
458 ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
459 qreply(q, mp);
460 }
461
462 static void
463 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
464 {
465 dl_capability_req_t *dlp = (dl_capability_req_t *)mp->b_rptr;
466 dl_capability_sub_t *sp;
467 size_t size, len;
468 offset_t off, end;
469 t_uscalar_t dl_err;
470 queue_t *q = sup->su_wq;
471
472 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
473 if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
474 dl_err = DL_BADPRIM;
475 goto failed;
476 }
477
478 if (!sup->su_bound) {
479 dl_err = DL_OUTSTATE;
480 goto failed;
481 }
482
483 /*
484 * This request is overloaded. If there are no requested capabilities
485 * then we just want to acknowledge with all the capabilities we
486 * support. Otherwise we enable the set of capabilities requested.
487 */
488 if (dlp->dl_sub_length == 0) {
489 softmac_capability_advertise(sup, mp);
490 return;
491 }
492
493 if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
494 dl_err = DL_BADPRIM;
495 goto failed;
496 }
497
498 dlp->dl_primitive = DL_CAPABILITY_ACK;
499
500 off = dlp->dl_sub_offset;
501 len = dlp->dl_sub_length;
502
503 /*
504 * Walk the list of capabilities to be enabled.
505 */
506 for (end = off + len; off < end; ) {
507 sp = (dl_capability_sub_t *)(mp->b_rptr + off);
508 size = sizeof (dl_capability_sub_t) + sp->dl_length;
509
510 if (off + size > end ||
511 !IS_P2ALIGNED(off, sizeof (uint32_t))) {
512 dl_err = DL_BADPRIM;
513 goto failed;
514 }
515
516 switch (sp->dl_cap) {
517 /*
518 * TCP/IP checksum offload to hardware.
519 */
520 case DL_CAPAB_HCKSUM: {
521 dl_capab_hcksum_t *hcksump;
522 dl_capab_hcksum_t hcksum;
523
524 hcksump = (dl_capab_hcksum_t *)&sp[1];
525 /*
526 * Copy for alignment.
527 */
528 bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
529 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
530 bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
531 break;
532 }
533
534 default:
535 break;
536 }
537
538 off += size;
539 }
540 qreply(q, mp);
541 return;
542 failed:
543 dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
544 }
545
546 static void
547 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
548 {
549 softmac_lower_t *slp = sup->su_slp;
550 softmac_t *softmac = sup->su_softmac;
551 mblk_t *ackmp, *mp1;
552 int err;
553
554 if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
555 freemsg(mp);
556 return;
557 }
558
559 /*
560 * Allocate ackmp incase the underlying driver does not ack timely.
561 */
562 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
563 dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
564 return;
565 }
566
567 err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
568 if (ackmp != NULL) {
569 freemsg(mp1);
570 } else {
571 /*
572 * The driver does not ack timely.
573 */
574 ASSERT(err == ENOMSG);
575 ackmp = mp1;
576 }
577 if (err != 0)
578 goto failed;
579
580 /*
581 * Enable capabilities the underlying driver claims to support.
582 */
583 if ((err = softmac_capab_enable(slp)) != 0)
584 goto failed;
585
586 /*
587 * Check whether this softmac is already marked as exclusively used,
588 * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
589 */
590 mutex_enter(&softmac->smac_active_mutex);
591 if (softmac->smac_active) {
592 mutex_exit(&softmac->smac_active_mutex);
593 err = EBUSY;
594 goto failed;
595 }
596 softmac->smac_nactive++;
597 sup->su_active = B_TRUE;
598 mutex_exit(&softmac->smac_active_mutex);
599 sup->su_bound = B_TRUE;
600
601 qreply(sup->su_wq, ackmp);
602 return;
603 failed:
604 if (err != 0) {
605 dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
606 return;
607 }
608 }
609
610 static void
611 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
612 {
613 softmac_lower_t *slp = sup->su_slp;
614 softmac_t *softmac = sup->su_softmac;
615 mblk_t *ackmp, *mp1;
616 int err;
617
618 if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
619 freemsg(mp);
620 return;
621 }
622
623 if (!sup->su_bound) {
624 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
625 return;
626 }
627
628 /*
629 * Allocate ackmp incase the underlying driver does not ack timely.
630 */
631 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
632 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
633 return;
634 }
635
636 err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
637 if (ackmp != NULL) {
638 freemsg(mp1);
639 } else {
640 /*
641 * The driver does not ack timely.
642 */
643 ASSERT(err == ENOMSG);
644 ackmp = mp1;
645 }
646 if (err != 0) {
647 dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
648 return;
649 }
650
651 sup->su_bound = B_FALSE;
652
653 mutex_enter(&softmac->smac_active_mutex);
654 if (sup->su_active) {
655 ASSERT(!softmac->smac_active);
656 softmac->smac_nactive--;
657 sup->su_active = B_FALSE;
658 }
659 mutex_exit(&softmac->smac_active_mutex);
660
661 done:
662 qreply(sup->su_wq, ackmp);
663 }
664
665 /*
666 * Process the non-data mblk.
667 */
668 static void
669 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
670 {
671 softmac_t *softmac = sup->su_softmac;
672 softmac_lower_t *slp = sup->su_slp;
673 unsigned char dbtype;
674 t_uscalar_t prim;
675
676 dbtype = DB_TYPE(mp);
677 sup->su_is_arp = 0;
678 switch (dbtype) {
679 case M_CTL:
680 sup->su_is_arp = 1;
681 /* FALLTHROUGH */
682 case M_IOCTL: {
683 uint32_t expected_mode;
684
685 if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
686 break;
687
688 /*
689 * Nak the M_IOCTL based on the STREAMS specification.
690 */
691 if (dbtype == M_IOCTL)
692 miocnak(sup->su_wq, mp, 0, EINVAL);
693 else
694 freemsg(mp);
695
696 /*
697 * This stream is either IP or ARP. See whether
698 * we need to setup a dedicated-lower-stream for it.
699 */
700 mutex_enter(&softmac->smac_fp_mutex);
701
702 expected_mode = DATAPATH_MODE(softmac);
703 if (expected_mode == SOFTMAC_SLOWPATH)
704 sup->su_mode = SOFTMAC_SLOWPATH;
705 list_insert_head(&softmac->smac_sup_list, sup);
706 mutex_exit(&softmac->smac_fp_mutex);
707
708 /*
709 * Setup the fast-path dedicated lower stream if fast-path
710 * is expected. Note that no lock is held here, and if
711 * smac_expected_mode is changed from SOFTMAC_FASTPATH to
712 * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
713 * data-path switching would already be queued and will
714 * be processed by softmac_wput_single_nondata() later.
715 */
716 if (expected_mode == SOFTMAC_FASTPATH)
717 (void) softmac_fastpath_setup(sup);
718 return;
719 }
720 case M_PROTO:
721 case M_PCPROTO:
722 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
723 freemsg(mp);
724 return;
725 }
726 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
727 switch (prim) {
728 case DL_NOTIFY_IND:
729 if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
730 ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
731 DL_NOTE_REPLUMB) {
732 freemsg(mp);
733 return;
734 }
735 /*
736 * This DL_NOTE_REPLUMB message is initiated
737 * and queued by the softmac itself, when the
738 * sup is trying to switching its datapath mode
739 * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
740 * Send this message upstream.
741 */
742 qreply(sup->su_wq, mp);
743 return;
744 case DL_NOTIFY_CONF:
745 if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
746 ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
747 DL_NOTE_REPLUMB_DONE) {
748 freemsg(mp);
749 return;
750 }
751 /*
752 * This is an indication from IP/ARP that the
753 * fastpath->slowpath switch is done.
754 */
755 freemsg(mp);
756 softmac_datapath_switch_done(sup);
757 return;
758 }
759 break;
760 }
761
762 /*
763 * No need to hold lock to check su_mode, since su_mode updating only
764 * operation is is serialized by softmac_wput_nondata_task().
765 */
766 if (sup->su_mode != SOFTMAC_FASTPATH) {
767 dld_wput(sup->su_wq, mp);
768 return;
769 }
770
771 /*
772 * Fastpath non-data message processing. Most of non-data messages
773 * can be directly passed down to the dedicated-lower-stream, aside
774 * from the following M_PROTO/M_PCPROTO messages.
775 */
776 switch (dbtype) {
777 case M_PROTO:
778 case M_PCPROTO:
779 switch (prim) {
780 case DL_BIND_REQ:
781 softmac_bind_req(sup, mp);
782 break;
783 case DL_UNBIND_REQ:
784 softmac_unbind_req(sup, mp);
785 break;
786 case DL_CAPABILITY_REQ:
787 softmac_capability_req(sup, mp);
788 break;
789 default:
790 putnext(slp->sl_wq, mp);
791 break;
792 }
793 break;
794 default:
795 putnext(slp->sl_wq, mp);
796 break;
797 }
798 }
799
800 /*
801 * The worker thread which processes non-data messages. Note we only process
802 * one message at one time in order to be able to "flush" the queued message
803 * and serialize the processing.
804 */
805 static void
806 softmac_wput_nondata_task(void *arg)
807 {
808 softmac_upper_t *sup = arg;
809 mblk_t *mp;
810
811 mutex_enter(&sup->su_disp_mutex);
812
813 while (sup->su_pending_head != NULL) {
814 if (sup->su_closing)
815 break;
816
817 SOFTMAC_DQ_PENDING(sup, &mp);
818 mutex_exit(&sup->su_disp_mutex);
819 softmac_wput_single_nondata(sup, mp);
820 mutex_enter(&sup->su_disp_mutex);
821 }
822
823 /*
824 * If the stream is closing, flush all queued messages and inform
825 * the stream to be closed.
826 */
827 freemsgchain(sup->su_pending_head);
828 sup->su_pending_head = sup->su_pending_tail = NULL;
829 sup->su_dlpi_pending = B_FALSE;
830 cv_signal(&sup->su_disp_cv);
831 mutex_exit(&sup->su_disp_mutex);
832 }
833
834 /*
835 * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
836 * This thread is started when the softmac module is first loaded.
837 */
838 static void
839 softmac_taskq_dispatch(void)
840 {
841 callb_cpr_t cprinfo;
842 softmac_upper_t *sup;
843
844 CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
845 "softmac_taskq_dispatch");
846 mutex_enter(&softmac_taskq_lock);
847
848 while (!softmac_taskq_quit) {
849 sup = list_head(&softmac_taskq_list);
850 while (sup != NULL) {
851 list_remove(&softmac_taskq_list, sup);
852 sup->su_taskq_scheduled = B_FALSE;
853 mutex_exit(&softmac_taskq_lock);
854 VERIFY(taskq_dispatch(system_taskq,
855 softmac_wput_nondata_task, sup, TQ_SLEEP) !=
856 TASKQID_INVALID);
857 mutex_enter(&softmac_taskq_lock);
858 sup = list_head(&softmac_taskq_list);
859 }
860
861 CALLB_CPR_SAFE_BEGIN(&cprinfo);
862 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
863 CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
864 }
865
866 softmac_taskq_done = B_TRUE;
867 cv_signal(&softmac_taskq_cv);
868 CALLB_CPR_EXIT(&cprinfo);
869 thread_exit();
870 }
871
872 void
873 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
874 {
875 /*
876 * The processing of the message might block. Enqueue the
877 * message for later processing.
878 */
879 mutex_enter(&sup->su_disp_mutex);
880
881 if (sup->su_closing) {
882 mutex_exit(&sup->su_disp_mutex);
883 freemsg(mp);
884 return;
885 }
886
887 SOFTMAC_EQ_PENDING(sup, mp);
888
889 if (sup->su_dlpi_pending) {
890 mutex_exit(&sup->su_disp_mutex);
891 return;
892 }
893 sup->su_dlpi_pending = B_TRUE;
894 mutex_exit(&sup->su_disp_mutex);
895
896 if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
897 sup, TQ_NOSLEEP) != TASKQID_INVALID) {
898 return;
899 }
900
901 mutex_enter(&softmac_taskq_lock);
902 if (!sup->su_taskq_scheduled) {
903 list_insert_tail(&softmac_taskq_list, sup);
904 cv_signal(&softmac_taskq_cv);
905 }
906 sup->su_taskq_scheduled = B_TRUE;
907 mutex_exit(&softmac_taskq_lock);
908 }
909
910 /*
911 * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
912 */
913 static int
914 softmac_fastpath_setup(softmac_upper_t *sup)
915 {
916 softmac_t *softmac = sup->su_softmac;
917 softmac_lower_t *slp;
918 int err;
919
920 err = softmac_lower_setup(softmac, sup, &slp);
921
922 mutex_enter(&sup->su_mutex);
923 /*
924 * Wait for all data messages to be processed so that we can change
925 * the su_mode.
926 */
927 while (sup->su_tx_inprocess != 0)
928 cv_wait(&sup->su_cv, &sup->su_mutex);
929
930 ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
931 ASSERT(sup->su_slp == NULL);
932 if (err != 0) {
933 sup->su_mode = SOFTMAC_SLOWPATH;
934 } else {
935 sup->su_slp = slp;
936 sup->su_mode = SOFTMAC_FASTPATH;
937 }
938 mutex_exit(&sup->su_mutex);
939 return (err);
940 }
941
942 /*
943 * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
944 */
945 static void
946 softmac_fastpath_tear(softmac_upper_t *sup)
947 {
948 mutex_enter(&sup->su_mutex);
949 /*
950 * Wait for all data messages in the dedicated-lower-stream
951 * to be processed.
952 */
953 while (sup->su_tx_inprocess != 0)
954 cv_wait(&sup->su_cv, &sup->su_mutex);
955
956 /*
957 * Note that this function is called either when the stream is closed,
958 * or the stream is unbound (fastpath-slowpath-switch). Therefore,
959 * No need to call the tx_notify callback.
960 */
961 sup->su_tx_notify_func = NULL;
962 sup->su_tx_notify_arg = NULL;
963 if (sup->su_tx_busy) {
964 ASSERT(sup->su_tx_flow_mp == NULL);
965 VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
966 sup->su_tx_busy = B_FALSE;
967 }
968
969 sup->su_mode = SOFTMAC_SLOWPATH;
970
971 /*
972 * Destroy the dedicated-lower-stream. Note that slp is destroyed
973 * when lh is closed.
974 */
975 (void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
976 sup->su_slp = NULL;
977 mutex_exit(&sup->su_mutex);
978 }
979
980 void
981 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
982 {
983 /*
984 * No lock is required to access the su_mode field since the data
985 * traffic is quiesce by IP when the data-path mode is in the
986 * process of switching.
987 */
988 if (sup->su_mode != SOFTMAC_FASTPATH)
989 dld_wput(sup->su_wq, mp);
990 else
991 (void) softmac_fastpath_wput_data(sup, mp, NULL, 0);
992 }
993
994 /*ARGSUSED*/
995 static mac_tx_cookie_t
996 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
997 uint16_t flag)
998 {
999 queue_t *wq = sup->su_slp->sl_wq;
1000
1001 /*
1002 * This function is called from IP, only the MAC_DROP_ON_NO_DESC
1003 * flag can be specified.
1004 */
1005 ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
1006 ASSERT(mp->b_next == NULL);
1007
1008 /*
1009 * Check wether the dedicated-lower-stream is able to handle more
1010 * messages, and enable the flow-control if it is not.
1011 *
1012 * Note that in order not to introduce any packet reordering, we
1013 * always send the message down to the dedicated-lower-stream:
1014 *
1015 * If the flow-control is already enabled, but we still get
1016 * the messages from the upper-stream, it means that the upper
1017 * stream does not respect STREAMS flow-control (e.g., TCP). Simply
1018 * pass the message down to the lower-stream in that case.
1019 */
1020 if (SOFTMAC_CANPUTNEXT(wq)) {
1021 putnext(wq, mp);
1022 return (NULL);
1023 }
1024
1025 if (sup->su_tx_busy) {
1026 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1027 freemsg(mp);
1028 else
1029 putnext(wq, mp);
1030 return ((mac_tx_cookie_t)sup);
1031 }
1032
1033 mutex_enter(&sup->su_mutex);
1034 if (!sup->su_tx_busy) {
1035 /*
1036 * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1037 * called when the flow control can be disabled. Otherwise,
1038 * put the tx_flow_mp into the wq to make use of the old
1039 * streams flow control.
1040 */
1041 ASSERT(sup->su_tx_flow_mp != NULL);
1042 (void) putq(sup->su_wq, sup->su_tx_flow_mp);
1043 sup->su_tx_flow_mp = NULL;
1044 sup->su_tx_busy = B_TRUE;
1045 qenable(wq);
1046 }
1047 mutex_exit(&sup->su_mutex);
1048
1049 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1050 freemsg(mp);
1051 else
1052 putnext(wq, mp);
1053 return ((mac_tx_cookie_t)sup);
1054 }
1055
1056 boolean_t
1057 softmac_active_set(void *arg)
1058 {
1059 softmac_t *softmac = arg;
1060
1061 mutex_enter(&softmac->smac_active_mutex);
1062 if (softmac->smac_nactive != 0) {
1063 mutex_exit(&softmac->smac_active_mutex);
1064 return (B_FALSE);
1065 }
1066 softmac->smac_active = B_TRUE;
1067 mutex_exit(&softmac->smac_active_mutex);
1068 return (B_TRUE);
1069 }
1070
1071 void
1072 softmac_active_clear(void *arg)
1073 {
1074 softmac_t *softmac = arg;
1075
1076 mutex_enter(&softmac->smac_active_mutex);
1077 ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1078 softmac->smac_active = B_FALSE;
1079 mutex_exit(&softmac->smac_active_mutex);
1080 }
1081
1082 /*
1083 * Disable/reenable fastpath on given softmac. This request could come from a
1084 * MAC client or directly from administrators.
1085 */
1086 int
1087 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1088 {
1089 softmac_upper_t *sup;
1090 mblk_t *head = NULL, *tail = NULL, *mp;
1091 list_t reqlist;
1092 softmac_switch_req_t *req;
1093 uint32_t current_mode, expected_mode;
1094 int err = 0;
1095
1096 mutex_enter(&softmac->smac_fp_mutex);
1097
1098 current_mode = DATAPATH_MODE(softmac);
1099 if (admin) {
1100 if (softmac->smac_fastpath_admin_disabled == disable) {
1101 mutex_exit(&softmac->smac_fp_mutex);
1102 return (0);
1103 }
1104 softmac->smac_fastpath_admin_disabled = disable;
1105 } else if (disable) {
1106 softmac->smac_fp_disable_clients++;
1107 } else {
1108 ASSERT(softmac->smac_fp_disable_clients != 0);
1109 softmac->smac_fp_disable_clients--;
1110 }
1111
1112 expected_mode = DATAPATH_MODE(softmac);
1113 if (current_mode == expected_mode) {
1114 mutex_exit(&softmac->smac_fp_mutex);
1115 return (0);
1116 }
1117
1118 /*
1119 * The expected mode is different from whatever datapath mode
1120 * this softmac is expected from last request, enqueue the data-path
1121 * switch request.
1122 */
1123 list_create(&reqlist, sizeof (softmac_switch_req_t),
1124 offsetof(softmac_switch_req_t, ssq_req_list_node));
1125
1126 /*
1127 * Allocate all DL_NOTIFY_IND messages and request structures that
1128 * are required to switch each IP/ARP stream to the expected mode.
1129 */
1130 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1131 sup = list_next(&softmac->smac_sup_list, sup)) {
1132 dl_notify_ind_t *dlip;
1133
1134 req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1135 if (req == NULL)
1136 break;
1137
1138 req->ssq_expected_mode = expected_mode;
1139 if (sup->su_is_arp) {
1140 list_insert_tail(&reqlist, req);
1141 continue;
1142 }
1143 /*
1144 * Allocate the DL_NOTE_REPLUMB message.
1145 */
1146 if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1147 kmem_free(req, sizeof (softmac_switch_req_t));
1148 break;
1149 }
1150
1151 list_insert_tail(&reqlist, req);
1152
1153 mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1154 mp->b_datap->db_type = M_PROTO;
1155 bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1156 dlip = (dl_notify_ind_t *)mp->b_rptr;
1157 dlip->dl_primitive = DL_NOTIFY_IND;
1158 dlip->dl_notification = DL_NOTE_REPLUMB;
1159 if (head == NULL) {
1160 head = tail = mp;
1161 } else {
1162 tail->b_next = mp;
1163 tail = mp;
1164 }
1165 }
1166
1167 /*
1168 * Note that it is fine if the expected data-path mode is fast-path
1169 * and some of streams fails to switch. Only return failure if we
1170 * are expected to switch to the slow-path.
1171 */
1172 if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1173 err = ENOMEM;
1174 goto fail;
1175 }
1176
1177 /*
1178 * Start switching for each IP/ARP stream. The switching operation
1179 * will eventually succeed and there is no need to wait for it
1180 * to finish.
1181 */
1182 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1183 sup = list_next(&softmac->smac_sup_list, sup)) {
1184 if (!sup->su_is_arp) {
1185 mp = head->b_next;
1186 head->b_next = NULL;
1187 softmac_wput_nondata(sup, head);
1188 head = mp;
1189 }
1190 /*
1191 * Add the switch request to the requests list of the stream.
1192 */
1193 req = list_head(&reqlist);
1194 ASSERT(req != NULL);
1195 list_remove(&reqlist, req);
1196 list_insert_tail(&sup->su_req_list, req);
1197 }
1198
1199 mutex_exit(&softmac->smac_fp_mutex);
1200 ASSERT(list_is_empty(&reqlist));
1201 list_destroy(&reqlist);
1202 return (0);
1203 fail:
1204 if (admin) {
1205 softmac->smac_fastpath_admin_disabled = !disable;
1206 } else if (disable) {
1207 softmac->smac_fp_disable_clients--;
1208 } else {
1209 softmac->smac_fp_disable_clients++;
1210 }
1211
1212 mutex_exit(&softmac->smac_fp_mutex);
1213 while ((req = list_head(&reqlist)) != NULL) {
1214 list_remove(&reqlist, req);
1215 kmem_free(req, sizeof (softmac_switch_req_t));
1216 }
1217 freemsgchain(head);
1218 list_destroy(&reqlist);
1219 return (err);
1220 }
1221
1222 int
1223 softmac_fastpath_disable(void *arg)
1224 {
1225 return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1226 }
1227
1228 void
1229 softmac_fastpath_enable(void *arg)
1230 {
1231 VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1232 B_FALSE) == 0);
1233 }
1234
1235 void
1236 softmac_upperstream_close(softmac_upper_t *sup)
1237 {
1238 softmac_t *softmac = sup->su_softmac;
1239 softmac_switch_req_t *req;
1240
1241 mutex_enter(&softmac->smac_fp_mutex);
1242
1243 if (sup->su_mode == SOFTMAC_FASTPATH)
1244 softmac_fastpath_tear(sup);
1245
1246 if (sup->su_mode != SOFTMAC_UNKNOWN) {
1247 list_remove(&softmac->smac_sup_list, sup);
1248 sup->su_mode = SOFTMAC_UNKNOWN;
1249 }
1250
1251 /*
1252 * Cleanup all the switch requests queueed on this stream.
1253 */
1254 while ((req = list_head(&sup->su_req_list)) != NULL) {
1255 list_remove(&sup->su_req_list, req);
1256 kmem_free(req, sizeof (softmac_switch_req_t));
1257 }
1258 mutex_exit(&softmac->smac_fp_mutex);
1259 }
1260
1261 /*
1262 * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1263 * stream from the fastpath mode to the slowpath mode.
1264 */
1265 static void
1266 softmac_datapath_switch_done(softmac_upper_t *sup)
1267 {
1268 softmac_t *softmac = sup->su_softmac;
1269 softmac_switch_req_t *req;
1270 uint32_t expected_mode;
1271
1272 mutex_enter(&softmac->smac_fp_mutex);
1273 req = list_head(&sup->su_req_list);
1274 list_remove(&sup->su_req_list, req);
1275 expected_mode = req->ssq_expected_mode;
1276 kmem_free(req, sizeof (softmac_switch_req_t));
1277
1278 if (expected_mode == sup->su_mode) {
1279 mutex_exit(&softmac->smac_fp_mutex);
1280 return;
1281 }
1282
1283 ASSERT(!sup->su_bound);
1284 mutex_exit(&softmac->smac_fp_mutex);
1285
1286 /*
1287 * It is fine if the expected mode is fast-path and we fail
1288 * to enable fastpath on this stream.
1289 */
1290 if (expected_mode == SOFTMAC_SLOWPATH)
1291 softmac_fastpath_tear(sup);
1292 else
1293 (void) softmac_fastpath_setup(sup);
1294 }