1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 /*
  27  * Copyright 2019, Joyent, Inc.
  28  */
  29 
  30 /*
  31  * Softmac data-path switching:
  32  *
  33  * - Fast-path model
  34  *
  35  * When the softmac fast-path is used, a dedicated lower-stream
  36  * will be opened over the legacy device for each IP/ARP (upper-)stream
  37  * over the softMAC, and all DLPI messages (including control messages
  38  * and data messages) will be exchanged between the upper-stream and
  39  * the corresponding lower-stream directly. Therefore, the data
  40  * demultiplexing, filtering and classification processing will be done
  41  * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
  42  * no longer needed.
  43  *
  44  * - Slow-path model
  45  *
  46  * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
  47  * not be bypassed to assure its function correctness. For example,
  48  * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
  49  * In this case, a shared lower-stream will be opened over the legacy
  50  * device, which is responsible for implementing the GLDv3 callbacks
  51  * and passing RAW data messages between the legacy devices and the GLDv3
  52  * framework.
  53  *
  54  * By default, the softmac fast-path mode will be used to assure the
  55  * performance; MAC clients will be able to request to disable the softmac
  56  * fast-path mode to support certain features, and if that succeeds,
  57  * the system will fallback to the slow-path softmac data-path model.
  58  *
  59  *
  60  * The details of the softmac data fast-path model is stated as below
  61  *
  62  * 1. When a stream is opened on a softMAC, the softmac module will takes
  63  *    over the DLPI processing on this stream;
  64  *
  65  * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
  66  *    used by default, unless fast-path is disabled by any MAC client
  67  *    explicitly. The softmac module first identifies an IP/ARP stream
  68  *    by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
  69  *    if there is one, this stream is either an IP or an ARP stream
  70  *    and will use fast-path potentially;
  71  *
  72  * 3. When the softmac fast-path is used, an dedicated lower-stream will
  73  *    be setup for each IP/ARP stream (1-1 mapping). From that point on,
  74  *    all control and data messages will be exchanged between the IP/ARP
  75  *    upper-stream and the legacy device through this dedicated
  76  *    lower-stream. As a result, the DLS/MAC layer processing in GLDv3
  77  *    will be skipped, and this greatly improves the performance;
  78  *
  79  * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
  80  *    by a VNIC), all the IP/ARP upper streams will try to switch from
  81  *    the fast-path to the slow-path. The dedicated lower-stream will be
  82  *    destroyed, and all the control and data-messages will go through the
  83  *    existing GLDv3 code path and (in the end) the shared lower-stream;
  84  *
  85  * 5. On the other hand, when the last MAC client cancels its fast-path
  86  *    disable request, all the IP/ARP streams will try to switch back to
  87  *    the fast-path mode;
  88  *
  89  * Step 5 and 6 both rely on the data-path mode switching process
  90  * described below:
  91  *
  92  * 1) To switch the softmac data-path mode (between fast-path and slow-path),
  93  *    softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
  94  *    upstream over each IP/ARP streams that needs data-path mode switching;
  95  *
  96  * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
  97  *    all the IP interfaces on the corresponding ill (IP Lower level
  98  *    structure), and bring up those interfaces over again; this will in
  99  *    turn cause the ARP to "replumb" the interface.
 100  *
 101  *    During the replumb process, both IP and ARP will send downstream the
 102  *    necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
 103  *    the old state of the underlying softMAC, following with the necessary
 104  *    DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
 105  *    Between the cleanup and re-setup process, IP/ARP will also send down
 106  *    a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
 107  *    indicate the *switching point*;
 108  *
 109  * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
 110  *    creates or destroys the dedicated lower-stream (depending on which
 111  *    data-path mode the softMAC switches to), and change the softmac
 112  *    data-path mode. From then on, softmac will process all the succeeding
 113  *    control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
 114  *    messages) and data messages based on new data-path mode.
 115  */
 116 
 117 #include <sys/types.h>
 118 #include <sys/disp.h>
 119 #include <sys/callb.h>
 120 #include <sys/sysmacros.h>
 121 #include <sys/file.h>
 122 #include <sys/vlan.h>
 123 #include <sys/dld.h>
 124 #include <sys/sockio.h>
 125 #include <sys/softmac_impl.h>
 126 #include <net/if.h>
 127 
 128 static kmutex_t         softmac_taskq_lock;
 129 static kcondvar_t       softmac_taskq_cv;
 130 static list_t           softmac_taskq_list;     /* List of softmac_upper_t */
 131 boolean_t               softmac_taskq_quit;
 132 boolean_t               softmac_taskq_done;
 133 
 134 static void             softmac_taskq_dispatch();
 135 static int              softmac_fastpath_setup(softmac_upper_t *);
 136 static mac_tx_cookie_t  softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
 137                             uintptr_t, uint16_t);
 138 static void             softmac_datapath_switch_done(softmac_upper_t *);
 139 
 140 void
 141 softmac_fp_init()
 142 {
 143         mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
 144         cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
 145 
 146         softmac_taskq_quit = B_FALSE;
 147         softmac_taskq_done = B_FALSE;
 148         list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
 149             offsetof(softmac_upper_t, su_taskq_list_node));
 150         (void) thread_create(NULL, 0, softmac_taskq_dispatch, NULL, 0,
 151             &p0, TS_RUN, minclsyspri);
 152 }
 153 
 154 void
 155 softmac_fp_fini()
 156 {
 157         /*
 158          * Request the softmac_taskq thread to quit and wait for it to be done.
 159          */
 160         mutex_enter(&softmac_taskq_lock);
 161         softmac_taskq_quit = B_TRUE;
 162         cv_signal(&softmac_taskq_cv);
 163         while (!softmac_taskq_done)
 164                 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
 165         mutex_exit(&softmac_taskq_lock);
 166         list_destroy(&softmac_taskq_list);
 167 
 168         mutex_destroy(&softmac_taskq_lock);
 169         cv_destroy(&softmac_taskq_cv);
 170 }
 171 
 172 static boolean_t
 173 check_ip_above(queue_t *q)
 174 {
 175         queue_t         *next_q;
 176         boolean_t       ret = B_TRUE;
 177 
 178         claimstr(q);
 179         next_q = q->q_next;
 180         if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
 181                 ret = B_FALSE;
 182         releasestr(q);
 183         return (ret);
 184 }
 185 
 186 /* ARGSUSED */
 187 static int
 188 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
 189 {
 190         switch (flags) {
 191         case DLD_ENABLE:
 192                 mutex_enter(&sup->su_mutex);
 193                 break;
 194         case DLD_DISABLE:
 195                 mutex_exit(&sup->su_mutex);
 196                 break;
 197         case DLD_QUERY:
 198                 return (MUTEX_HELD(&sup->su_mutex));
 199         }
 200         return (0);
 201 }
 202 
 203 static mac_tx_notify_handle_t
 204 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
 205 {
 206         ASSERT(MUTEX_HELD(&sup->su_mutex));
 207 
 208         if (func != NULL) {
 209                 sup->su_tx_notify_func = func;
 210                 sup->su_tx_notify_arg = arg;
 211         } else {
 212                 /*
 213                  * Wait for all tx_notify_func call to be done.
 214                  */
 215                 while (sup->su_tx_inprocess != 0)
 216                         cv_wait(&sup->su_cv, &sup->su_mutex);
 217 
 218                 sup->su_tx_notify_func = NULL;
 219                 sup->su_tx_notify_arg = NULL;
 220         }
 221         return ((mac_tx_notify_handle_t)sup);
 222 }
 223 
 224 static boolean_t
 225 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
 226 {
 227         ASSERT(cookie == (mac_tx_cookie_t)sup);
 228         return (sup->su_tx_busy);
 229 }
 230 
 231 static int
 232 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
 233 {
 234         dld_capab_direct_t      *direct = data;
 235         softmac_lower_t         *slp = sup->su_slp;
 236 
 237         ASSERT(MUTEX_HELD(&sup->su_mutex));
 238 
 239         ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
 240 
 241         switch (flags) {
 242         case DLD_ENABLE:
 243                 if (sup->su_direct)
 244                         return (0);
 245 
 246                 sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
 247                 sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
 248                 slp->sl_rxinfo = &sup->su_direct_rxinfo;
 249                 direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
 250                 direct->di_tx_dh = sup;
 251                 direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
 252                 direct->di_tx_fctl_dh = sup;
 253                 direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
 254                 direct->di_tx_cb_dh = sup;
 255                 sup->su_direct = B_TRUE;
 256                 return (0);
 257 
 258         case DLD_DISABLE:
 259                 if (!sup->su_direct)
 260                         return (0);
 261 
 262                 slp->sl_rxinfo = &sup->su_rxinfo;
 263                 sup->su_direct = B_FALSE;
 264                 return (0);
 265         }
 266         return (ENOTSUP);
 267 }
 268 
 269 static int
 270 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
 271 {
 272         int     err;
 273 
 274         /*
 275          * Don't enable direct callback capabilities unless the caller is
 276          * the IP client. When a module is inserted in a stream (_I_INSERT)
 277          * the stack initiates capability disable, but due to races, the
 278          * module insertion may complete before the capability disable
 279          * completes. So we limit the check to DLD_ENABLE case.
 280          */
 281         if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
 282             !check_ip_above(sup->su_rq)) {
 283                 return (ENOTSUP);
 284         }
 285 
 286         switch (type) {
 287         case DLD_CAPAB_DIRECT:
 288                 err = softmac_capab_direct(sup, data, flags);
 289                 break;
 290 
 291         case DLD_CAPAB_PERIM:
 292                 err = softmac_capab_perim(sup, data, flags);
 293                 break;
 294 
 295         default:
 296                 err = ENOTSUP;
 297                 break;
 298         }
 299         return (err);
 300 }
 301 
 302 static void
 303 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
 304 {
 305         dl_capability_ack_t     *dlap;
 306         dl_capability_sub_t     *dlsp;
 307         t_uscalar_t             subsize;
 308         uint8_t                 *ptr;
 309         queue_t                 *q = sup->su_wq;
 310         mblk_t                  *mp1;
 311         softmac_t               *softmac = sup->su_softmac;
 312         boolean_t               dld_capable = B_FALSE;
 313         boolean_t               hcksum_capable = B_FALSE;
 314         boolean_t               zcopy_capable = B_FALSE;
 315         boolean_t               mdt_capable = B_FALSE;
 316 
 317         ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
 318 
 319         /*
 320          * Initially assume no capabilities.
 321          */
 322         subsize = 0;
 323 
 324         /*
 325          * Direct capability negotiation interface between IP and softmac
 326          */
 327         if (check_ip_above(sup->su_rq)) {
 328                 dld_capable = B_TRUE;
 329                 subsize += sizeof (dl_capability_sub_t) +
 330                     sizeof (dl_capab_dld_t);
 331         }
 332 
 333         /*
 334          * Check if checksum offload is supported on this MAC.
 335          */
 336         if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
 337                 hcksum_capable = B_TRUE;
 338                 subsize += sizeof (dl_capability_sub_t) +
 339                     sizeof (dl_capab_hcksum_t);
 340         }
 341 
 342         /*
 343          * Check if zerocopy is supported on this interface.
 344          */
 345         if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
 346                 zcopy_capable = B_TRUE;
 347                 subsize += sizeof (dl_capability_sub_t) +
 348                     sizeof (dl_capab_zerocopy_t);
 349         }
 350 
 351         if (softmac->smac_mdt) {
 352                 mdt_capable = B_TRUE;
 353                 subsize += sizeof (dl_capability_sub_t) +
 354                     sizeof (dl_capab_mdt_t);
 355         }
 356 
 357         /*
 358          * If there are no capabilities to advertise or if we
 359          * can't allocate a response, send a DL_ERROR_ACK.
 360          */
 361         if ((subsize == 0) || (mp1 = reallocb(mp,
 362             sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
 363                 dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
 364                 return;
 365         }
 366 
 367         mp = mp1;
 368         DB_TYPE(mp) = M_PROTO;
 369         mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
 370         bzero(mp->b_rptr, MBLKL(mp));
 371         dlap = (dl_capability_ack_t *)mp->b_rptr;
 372         dlap->dl_primitive = DL_CAPABILITY_ACK;
 373         dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
 374         dlap->dl_sub_length = subsize;
 375         ptr = (uint8_t *)&dlap[1];
 376 
 377         /*
 378          * IP polling interface.
 379          */
 380         if (dld_capable) {
 381                 dl_capab_dld_t          dld;
 382 
 383                 dlsp = (dl_capability_sub_t *)ptr;
 384                 dlsp->dl_cap = DL_CAPAB_DLD;
 385                 dlsp->dl_length = sizeof (dl_capab_dld_t);
 386                 ptr += sizeof (dl_capability_sub_t);
 387 
 388                 bzero(&dld, sizeof (dl_capab_dld_t));
 389                 dld.dld_version = DLD_CURRENT_VERSION;
 390                 dld.dld_capab = (uintptr_t)softmac_dld_capab;
 391                 dld.dld_capab_handle = (uintptr_t)sup;
 392 
 393                 dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
 394                 bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
 395                 ptr += sizeof (dl_capab_dld_t);
 396         }
 397 
 398         /*
 399          * TCP/IP checksum offload.
 400          */
 401         if (hcksum_capable) {
 402                 dl_capab_hcksum_t       hcksum;
 403 
 404                 dlsp = (dl_capability_sub_t *)ptr;
 405 
 406                 dlsp->dl_cap = DL_CAPAB_HCKSUM;
 407                 dlsp->dl_length = sizeof (dl_capab_hcksum_t);
 408                 ptr += sizeof (dl_capability_sub_t);
 409 
 410                 bzero(&hcksum, sizeof (dl_capab_hcksum_t));
 411                 hcksum.hcksum_version = HCKSUM_VERSION_1;
 412                 hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
 413                 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
 414                 bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
 415                 ptr += sizeof (dl_capab_hcksum_t);
 416         }
 417 
 418         /*
 419          * Zero copy
 420          */
 421         if (zcopy_capable) {
 422                 dl_capab_zerocopy_t     zcopy;
 423 
 424                 dlsp = (dl_capability_sub_t *)ptr;
 425 
 426                 dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
 427                 dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
 428                 ptr += sizeof (dl_capability_sub_t);
 429 
 430                 bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
 431                 zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
 432                 zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
 433                 dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
 434                 bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
 435                 ptr += sizeof (dl_capab_zerocopy_t);
 436         }
 437 
 438         /*
 439          * MDT
 440          */
 441         if (mdt_capable) {
 442                 dl_capab_mdt_t mdt;
 443 
 444                 dlsp = (dl_capability_sub_t *)ptr;
 445 
 446                 dlsp->dl_cap = DL_CAPAB_MDT;
 447                 dlsp->dl_length = sizeof (dl_capab_mdt_t);
 448                 ptr += sizeof (dl_capability_sub_t);
 449 
 450                 bzero(&mdt, sizeof (dl_capab_mdt_t));
 451                 mdt.mdt_version = MDT_VERSION_2;
 452                 mdt.mdt_flags = DL_CAPAB_MDT_ENABLE;
 453                 mdt.mdt_hdr_head = softmac->smac_mdt_capab.mdt_hdr_head;
 454                 mdt.mdt_hdr_tail = softmac->smac_mdt_capab.mdt_hdr_tail;
 455                 mdt.mdt_max_pld = softmac->smac_mdt_capab.mdt_max_pld;
 456                 mdt.mdt_span_limit = softmac->smac_mdt_capab.mdt_span_limit;
 457                 dlcapabsetqid(&(mdt.mdt_mid), sup->su_rq);
 458                 bcopy(&mdt, ptr, sizeof (dl_capab_mdt_t));
 459                 ptr += sizeof (dl_capab_mdt_t);
 460         }
 461 
 462         ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
 463         qreply(q, mp);
 464 }
 465 
 466 static void
 467 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
 468 {
 469         dl_capability_req_t     *dlp = (dl_capability_req_t *)mp->b_rptr;
 470         dl_capability_sub_t     *sp;
 471         size_t                  size, len;
 472         offset_t                off, end;
 473         t_uscalar_t             dl_err;
 474         queue_t                 *q = sup->su_wq;
 475 
 476         ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
 477         if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
 478                 dl_err = DL_BADPRIM;
 479                 goto failed;
 480         }
 481 
 482         if (!sup->su_bound) {
 483                 dl_err = DL_OUTSTATE;
 484                 goto failed;
 485         }
 486 
 487         /*
 488          * This request is overloaded. If there are no requested capabilities
 489          * then we just want to acknowledge with all the capabilities we
 490          * support. Otherwise we enable the set of capabilities requested.
 491          */
 492         if (dlp->dl_sub_length == 0) {
 493                 softmac_capability_advertise(sup, mp);
 494                 return;
 495         }
 496 
 497         if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
 498                 dl_err = DL_BADPRIM;
 499                 goto failed;
 500         }
 501 
 502         dlp->dl_primitive = DL_CAPABILITY_ACK;
 503 
 504         off = dlp->dl_sub_offset;
 505         len = dlp->dl_sub_length;
 506 
 507         /*
 508          * Walk the list of capabilities to be enabled.
 509          */
 510         for (end = off + len; off < end; ) {
 511                 sp = (dl_capability_sub_t *)(mp->b_rptr + off);
 512                 size = sizeof (dl_capability_sub_t) + sp->dl_length;
 513 
 514                 if (off + size > end ||
 515                     !IS_P2ALIGNED(off, sizeof (uint32_t))) {
 516                         dl_err = DL_BADPRIM;
 517                         goto failed;
 518                 }
 519 
 520                 switch (sp->dl_cap) {
 521                 /*
 522                  * TCP/IP checksum offload to hardware.
 523                  */
 524                 case DL_CAPAB_HCKSUM: {
 525                         dl_capab_hcksum_t *hcksump;
 526                         dl_capab_hcksum_t hcksum;
 527 
 528                         hcksump = (dl_capab_hcksum_t *)&sp[1];
 529                         /*
 530                          * Copy for alignment.
 531                          */
 532                         bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
 533                         dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
 534                         bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
 535                         break;
 536                 }
 537 
 538                 default:
 539                         break;
 540                 }
 541 
 542                 off += size;
 543         }
 544         qreply(q, mp);
 545         return;
 546 failed:
 547         dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
 548 }
 549 
 550 static void
 551 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
 552 {
 553         softmac_lower_t *slp = sup->su_slp;
 554         softmac_t       *softmac = sup->su_softmac;
 555         mblk_t          *ackmp, *mp1;
 556         int             err;
 557 
 558         if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
 559                 freemsg(mp);
 560                 return;
 561         }
 562 
 563         /*
 564          * Allocate ackmp incase the underlying driver does not ack timely.
 565          */
 566         if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
 567                 dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
 568                 return;
 569         }
 570 
 571         err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
 572         if (ackmp != NULL) {
 573                 freemsg(mp1);
 574         } else {
 575                 /*
 576                  * The driver does not ack timely.
 577                  */
 578                 ASSERT(err == ENOMSG);
 579                 ackmp = mp1;
 580         }
 581         if (err != 0)
 582                 goto failed;
 583 
 584         /*
 585          * Enable capabilities the underlying driver claims to support.
 586          */
 587         if ((err = softmac_capab_enable(slp)) != 0)
 588                 goto failed;
 589 
 590         /*
 591          * Check whether this softmac is already marked as exclusively used,
 592          * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
 593          */
 594         mutex_enter(&softmac->smac_active_mutex);
 595         if (softmac->smac_active) {
 596                 mutex_exit(&softmac->smac_active_mutex);
 597                 err = EBUSY;
 598                 goto failed;
 599         }
 600         softmac->smac_nactive++;
 601         sup->su_active = B_TRUE;
 602         mutex_exit(&softmac->smac_active_mutex);
 603         sup->su_bound = B_TRUE;
 604 
 605         qreply(sup->su_wq, ackmp);
 606         return;
 607 failed:
 608         if (err != 0) {
 609                 dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
 610                 return;
 611         }
 612 }
 613 
 614 static void
 615 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
 616 {
 617         softmac_lower_t *slp = sup->su_slp;
 618         softmac_t       *softmac = sup->su_softmac;
 619         mblk_t          *ackmp, *mp1;
 620         int             err;
 621 
 622         if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
 623                 freemsg(mp);
 624                 return;
 625         }
 626 
 627         if (!sup->su_bound) {
 628                 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
 629                 return;
 630         }
 631 
 632         /*
 633          * Allocate ackmp incase the underlying driver does not ack timely.
 634          */
 635         if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
 636                 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
 637                 return;
 638         }
 639 
 640         err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
 641         if (ackmp != NULL) {
 642                 freemsg(mp1);
 643         } else {
 644                 /*
 645                  * The driver does not ack timely.
 646                  */
 647                 ASSERT(err == ENOMSG);
 648                 ackmp = mp1;
 649         }
 650         if (err != 0) {
 651                 dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
 652                 return;
 653         }
 654 
 655         sup->su_bound = B_FALSE;
 656 
 657         mutex_enter(&softmac->smac_active_mutex);
 658         if (sup->su_active) {
 659                 ASSERT(!softmac->smac_active);
 660                 softmac->smac_nactive--;
 661                 sup->su_active = B_FALSE;
 662         }
 663         mutex_exit(&softmac->smac_active_mutex);
 664 
 665 done:
 666         qreply(sup->su_wq, ackmp);
 667 }
 668 
 669 /*
 670  * Process the non-data mblk.
 671  */
 672 static void
 673 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
 674 {
 675         softmac_t *softmac = sup->su_softmac;
 676         softmac_lower_t *slp = sup->su_slp;
 677         unsigned char   dbtype;
 678         t_uscalar_t     prim;
 679 
 680         dbtype = DB_TYPE(mp);
 681         sup->su_is_arp = 0;
 682         switch (dbtype) {
 683         case M_CTL:
 684                 sup->su_is_arp = 1;
 685                 /* FALLTHROUGH */
 686         case M_IOCTL: {
 687                 uint32_t        expected_mode;
 688 
 689                 if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
 690                         break;
 691 
 692                 /*
 693                  * Nak the M_IOCTL based on the STREAMS specification.
 694                  */
 695                 if (dbtype == M_IOCTL)
 696                         miocnak(sup->su_wq, mp, 0, EINVAL);
 697                 else
 698                         freemsg(mp);
 699 
 700                 /*
 701                  * This stream is either IP or ARP. See whether
 702                  * we need to setup a dedicated-lower-stream for it.
 703                  */
 704                 mutex_enter(&softmac->smac_fp_mutex);
 705 
 706                 expected_mode = DATAPATH_MODE(softmac);
 707                 if (expected_mode == SOFTMAC_SLOWPATH)
 708                         sup->su_mode = SOFTMAC_SLOWPATH;
 709                 list_insert_head(&softmac->smac_sup_list, sup);
 710                 mutex_exit(&softmac->smac_fp_mutex);
 711 
 712                 /*
 713                  * Setup the fast-path dedicated lower stream if fast-path
 714                  * is expected. Note that no lock is held here, and if
 715                  * smac_expected_mode is changed from SOFTMAC_FASTPATH to
 716                  * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
 717                  * data-path switching would already be queued and will
 718                  * be processed by softmac_wput_single_nondata() later.
 719                  */
 720                 if (expected_mode == SOFTMAC_FASTPATH)
 721                         (void) softmac_fastpath_setup(sup);
 722                 return;
 723         }
 724         case M_PROTO:
 725         case M_PCPROTO:
 726                 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
 727                         freemsg(mp);
 728                         return;
 729                 }
 730                 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
 731                 switch (prim) {
 732                 case DL_NOTIFY_IND:
 733                         if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
 734                             ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
 735                             DL_NOTE_REPLUMB) {
 736                                 freemsg(mp);
 737                                 return;
 738                         }
 739                         /*
 740                          * This DL_NOTE_REPLUMB message is initiated
 741                          * and queued by the softmac itself, when the
 742                          * sup is trying to switching its datapath mode
 743                          * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
 744                          * Send this message upstream.
 745                          */
 746                         qreply(sup->su_wq, mp);
 747                         return;
 748                 case DL_NOTIFY_CONF:
 749                         if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
 750                             ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
 751                             DL_NOTE_REPLUMB_DONE) {
 752                                 freemsg(mp);
 753                                 return;
 754                         }
 755                         /*
 756                          * This is an indication from IP/ARP that the
 757                          * fastpath->slowpath switch is done.
 758                          */
 759                         freemsg(mp);
 760                         softmac_datapath_switch_done(sup);
 761                         return;
 762                 }
 763                 break;
 764         }
 765 
 766         /*
 767          * No need to hold lock to check su_mode, since su_mode updating only
 768          * operation is is serialized by softmac_wput_nondata_task().
 769          */
 770         if (sup->su_mode != SOFTMAC_FASTPATH) {
 771                 (void) dld_wput(sup->su_wq, mp);
 772                 return;
 773         }
 774 
 775         /*
 776          * Fastpath non-data message processing. Most of non-data messages
 777          * can be directly passed down to the dedicated-lower-stream, aside
 778          * from the following M_PROTO/M_PCPROTO messages.
 779          */
 780         switch (dbtype) {
 781         case M_PROTO:
 782         case M_PCPROTO:
 783                 switch (prim) {
 784                 case DL_BIND_REQ:
 785                         softmac_bind_req(sup, mp);
 786                         break;
 787                 case DL_UNBIND_REQ:
 788                         softmac_unbind_req(sup, mp);
 789                         break;
 790                 case DL_CAPABILITY_REQ:
 791                         softmac_capability_req(sup, mp);
 792                         break;
 793                 default:
 794                         putnext(slp->sl_wq, mp);
 795                         break;
 796                 }
 797                 break;
 798         default:
 799                 putnext(slp->sl_wq, mp);
 800                 break;
 801         }
 802 }
 803 
 804 /*
 805  * The worker thread which processes non-data messages. Note we only process
 806  * one message at one time in order to be able to "flush" the queued message
 807  * and serialize the processing.
 808  */
 809 static void
 810 softmac_wput_nondata_task(void *arg)
 811 {
 812         softmac_upper_t *sup = arg;
 813         mblk_t          *mp;
 814 
 815         mutex_enter(&sup->su_disp_mutex);
 816 
 817         while (sup->su_pending_head != NULL) {
 818                 if (sup->su_closing)
 819                         break;
 820 
 821                 SOFTMAC_DQ_PENDING(sup, &mp);
 822                 mutex_exit(&sup->su_disp_mutex);
 823                 softmac_wput_single_nondata(sup, mp);
 824                 mutex_enter(&sup->su_disp_mutex);
 825         }
 826 
 827         /*
 828          * If the stream is closing, flush all queued messages and inform
 829          * the stream to be closed.
 830          */
 831         freemsgchain(sup->su_pending_head);
 832         sup->su_pending_head = sup->su_pending_tail = NULL;
 833         sup->su_dlpi_pending = B_FALSE;
 834         cv_signal(&sup->su_disp_cv);
 835         mutex_exit(&sup->su_disp_mutex);
 836 }
 837 
 838 /*
 839  * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
 840  * This thread is started when the softmac module is first loaded.
 841  */
 842 static void
 843 softmac_taskq_dispatch(void)
 844 {
 845         callb_cpr_t     cprinfo;
 846         softmac_upper_t *sup;
 847 
 848         CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
 849             "softmac_taskq_dispatch");
 850         mutex_enter(&softmac_taskq_lock);
 851 
 852         while (!softmac_taskq_quit) {
 853                 sup = list_head(&softmac_taskq_list);
 854                 while (sup != NULL) {
 855                         list_remove(&softmac_taskq_list, sup);
 856                         sup->su_taskq_scheduled = B_FALSE;
 857                         mutex_exit(&softmac_taskq_lock);
 858                         VERIFY(taskq_dispatch(system_taskq,
 859                             softmac_wput_nondata_task, sup, TQ_SLEEP) !=
 860                             TASKQID_INVALID);
 861                         mutex_enter(&softmac_taskq_lock);
 862                         sup = list_head(&softmac_taskq_list);
 863                 }
 864 
 865                 CALLB_CPR_SAFE_BEGIN(&cprinfo);
 866                 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
 867                 CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
 868         }
 869 
 870         softmac_taskq_done = B_TRUE;
 871         cv_signal(&softmac_taskq_cv);
 872         CALLB_CPR_EXIT(&cprinfo);
 873         thread_exit();
 874 }
 875 
 876 void
 877 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
 878 {
 879         /*
 880          * The processing of the message might block. Enqueue the
 881          * message for later processing.
 882          */
 883         mutex_enter(&sup->su_disp_mutex);
 884 
 885         if (sup->su_closing) {
 886                 mutex_exit(&sup->su_disp_mutex);
 887                 freemsg(mp);
 888                 return;
 889         }
 890 
 891         SOFTMAC_EQ_PENDING(sup, mp);
 892 
 893         if (sup->su_dlpi_pending) {
 894                 mutex_exit(&sup->su_disp_mutex);
 895                 return;
 896         }
 897         sup->su_dlpi_pending = B_TRUE;
 898         mutex_exit(&sup->su_disp_mutex);
 899 
 900         if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
 901             sup, TQ_NOSLEEP) != TASKQID_INVALID) {
 902                 return;
 903         }
 904 
 905         mutex_enter(&softmac_taskq_lock);
 906         if (!sup->su_taskq_scheduled) {
 907                 list_insert_tail(&softmac_taskq_list, sup);
 908                 cv_signal(&softmac_taskq_cv);
 909         }
 910         sup->su_taskq_scheduled = B_TRUE;
 911         mutex_exit(&softmac_taskq_lock);
 912 }
 913 
 914 /*
 915  * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
 916  */
 917 static int
 918 softmac_fastpath_setup(softmac_upper_t *sup)
 919 {
 920         softmac_t       *softmac = sup->su_softmac;
 921         softmac_lower_t *slp;
 922         int             err;
 923 
 924         err = softmac_lower_setup(softmac, sup, &slp);
 925 
 926         mutex_enter(&sup->su_mutex);
 927         /*
 928          * Wait for all data messages to be processed so that we can change
 929          * the su_mode.
 930          */
 931         while (sup->su_tx_inprocess != 0)
 932                 cv_wait(&sup->su_cv, &sup->su_mutex);
 933 
 934         ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
 935         ASSERT(sup->su_slp == NULL);
 936         if (err != 0) {
 937                 sup->su_mode = SOFTMAC_SLOWPATH;
 938         } else {
 939                 sup->su_slp = slp;
 940                 sup->su_mode = SOFTMAC_FASTPATH;
 941         }
 942         mutex_exit(&sup->su_mutex);
 943         return (err);
 944 }
 945 
 946 /*
 947  * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
 948  */
 949 static void
 950 softmac_fastpath_tear(softmac_upper_t *sup)
 951 {
 952         mutex_enter(&sup->su_mutex);
 953         /*
 954          * Wait for all data messages in the dedicated-lower-stream
 955          * to be processed.
 956          */
 957         while (sup->su_tx_inprocess != 0)
 958                 cv_wait(&sup->su_cv, &sup->su_mutex);
 959 
 960         /*
 961          * Note that this function is called either when the stream is closed,
 962          * or the stream is unbound (fastpath-slowpath-switch). Therefore,
 963          * No need to call the tx_notify callback.
 964          */
 965         sup->su_tx_notify_func = NULL;
 966         sup->su_tx_notify_arg = NULL;
 967         if (sup->su_tx_busy) {
 968                 ASSERT(sup->su_tx_flow_mp == NULL);
 969                 VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
 970                 sup->su_tx_busy = B_FALSE;
 971         }
 972 
 973         sup->su_mode = SOFTMAC_SLOWPATH;
 974 
 975         /*
 976          * Destroy the dedicated-lower-stream. Note that slp is destroyed
 977          * when lh is closed.
 978          */
 979         (void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
 980         sup->su_slp = NULL;
 981         mutex_exit(&sup->su_mutex);
 982 }
 983 
 984 void
 985 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
 986 {
 987         /*
 988          * No lock is required to access the su_mode field since the data
 989          * traffic is quiesce by IP when the data-path mode is in the
 990          * process of switching.
 991          */
 992         if (sup->su_mode != SOFTMAC_FASTPATH)
 993                 (void) dld_wput(sup->su_wq, mp);
 994         else
 995                 (void) softmac_fastpath_wput_data(sup, mp, NULL, 0);
 996 }
 997 
 998 /*ARGSUSED*/
 999 static mac_tx_cookie_t
1000 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
1001     uint16_t flag)
1002 {
1003         queue_t         *wq = sup->su_slp->sl_wq;
1004 
1005         /*
1006          * This function is called from IP, only the MAC_DROP_ON_NO_DESC
1007          * flag can be specified.
1008          */
1009         ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
1010         ASSERT(mp->b_next == NULL);
1011 
1012         /*
1013          * Check wether the dedicated-lower-stream is able to handle more
1014          * messages, and enable the flow-control if it is not.
1015          *
1016          * Note that in order not to introduce any packet reordering, we
1017          * always send the message down to the dedicated-lower-stream:
1018          *
1019          * If the flow-control is already enabled, but we still get
1020          * the messages from the upper-stream, it means that the upper
1021          * stream does not respect STREAMS flow-control (e.g., TCP). Simply
1022          * pass the message down to the lower-stream in that case.
1023          */
1024         if (SOFTMAC_CANPUTNEXT(wq)) {
1025                 putnext(wq, mp);
1026                 return (NULL);
1027         }
1028 
1029         if (sup->su_tx_busy) {
1030                 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1031                         freemsg(mp);
1032                 else
1033                         putnext(wq, mp);
1034                 return ((mac_tx_cookie_t)sup);
1035         }
1036 
1037         mutex_enter(&sup->su_mutex);
1038         if (!sup->su_tx_busy) {
1039                 /*
1040                  * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1041                  * called when the flow control can be disabled. Otherwise,
1042                  * put the tx_flow_mp into the wq to make use of the old
1043                  * streams flow control.
1044                  */
1045                 ASSERT(sup->su_tx_flow_mp != NULL);
1046                 (void) putq(sup->su_wq, sup->su_tx_flow_mp);
1047                 sup->su_tx_flow_mp = NULL;
1048                 sup->su_tx_busy = B_TRUE;
1049                 qenable(wq);
1050         }
1051         mutex_exit(&sup->su_mutex);
1052 
1053         if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1054                 freemsg(mp);
1055         else
1056                 putnext(wq, mp);
1057         return ((mac_tx_cookie_t)sup);
1058 }
1059 
1060 boolean_t
1061 softmac_active_set(void *arg)
1062 {
1063         softmac_t       *softmac = arg;
1064 
1065         mutex_enter(&softmac->smac_active_mutex);
1066         if (softmac->smac_nactive != 0) {
1067                 mutex_exit(&softmac->smac_active_mutex);
1068                 return (B_FALSE);
1069         }
1070         softmac->smac_active = B_TRUE;
1071         mutex_exit(&softmac->smac_active_mutex);
1072         return (B_TRUE);
1073 }
1074 
1075 void
1076 softmac_active_clear(void *arg)
1077 {
1078         softmac_t       *softmac = arg;
1079 
1080         mutex_enter(&softmac->smac_active_mutex);
1081         ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1082         softmac->smac_active = B_FALSE;
1083         mutex_exit(&softmac->smac_active_mutex);
1084 }
1085 
1086 /*
1087  * Disable/reenable fastpath on given softmac. This request could come from a
1088  * MAC client or directly from administrators.
1089  */
1090 int
1091 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1092 {
1093         softmac_upper_t         *sup;
1094         mblk_t                  *head = NULL, *tail = NULL, *mp;
1095         list_t                  reqlist;
1096         softmac_switch_req_t    *req;
1097         uint32_t                current_mode, expected_mode;
1098         int                     err = 0;
1099 
1100         mutex_enter(&softmac->smac_fp_mutex);
1101 
1102         current_mode = DATAPATH_MODE(softmac);
1103         if (admin) {
1104                 if (softmac->smac_fastpath_admin_disabled == disable) {
1105                         mutex_exit(&softmac->smac_fp_mutex);
1106                         return (0);
1107                 }
1108                 softmac->smac_fastpath_admin_disabled = disable;
1109         } else if (disable) {
1110                 softmac->smac_fp_disable_clients++;
1111         } else {
1112                 ASSERT(softmac->smac_fp_disable_clients != 0);
1113                 softmac->smac_fp_disable_clients--;
1114         }
1115 
1116         expected_mode = DATAPATH_MODE(softmac);
1117         if (current_mode == expected_mode) {
1118                 mutex_exit(&softmac->smac_fp_mutex);
1119                 return (0);
1120         }
1121 
1122         /*
1123          * The expected mode is different from whatever datapath mode
1124          * this softmac is expected from last request, enqueue the data-path
1125          * switch request.
1126          */
1127         list_create(&reqlist, sizeof (softmac_switch_req_t),
1128             offsetof(softmac_switch_req_t, ssq_req_list_node));
1129 
1130         /*
1131          * Allocate all DL_NOTIFY_IND messages and request structures that
1132          * are required to switch each IP/ARP stream to the expected mode.
1133          */
1134         for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1135             sup = list_next(&softmac->smac_sup_list, sup)) {
1136                 dl_notify_ind_t *dlip;
1137 
1138                 req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1139                 if (req == NULL)
1140                         break;
1141 
1142                 req->ssq_expected_mode = expected_mode;
1143                 if (sup->su_is_arp) {
1144                         list_insert_tail(&reqlist, req);
1145                         continue;
1146                 }
1147                 /*
1148                  * Allocate the DL_NOTE_REPLUMB message.
1149                  */
1150                 if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1151                         kmem_free(req, sizeof (softmac_switch_req_t));
1152                         break;
1153                 }
1154 
1155                 list_insert_tail(&reqlist, req);
1156 
1157                 mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1158                 mp->b_datap->db_type = M_PROTO;
1159                 bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1160                 dlip = (dl_notify_ind_t *)mp->b_rptr;
1161                 dlip->dl_primitive = DL_NOTIFY_IND;
1162                 dlip->dl_notification = DL_NOTE_REPLUMB;
1163                 if (head == NULL) {
1164                         head = tail = mp;
1165                 } else {
1166                         tail->b_next = mp;
1167                         tail = mp;
1168                 }
1169         }
1170 
1171         /*
1172          * Note that it is fine if the expected data-path mode is fast-path
1173          * and some of streams fails to switch. Only return failure if we
1174          * are expected to switch to the slow-path.
1175          */
1176         if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1177                 err = ENOMEM;
1178                 goto fail;
1179         }
1180 
1181         /*
1182          * Start switching for each IP/ARP stream. The switching operation
1183          * will eventually succeed and there is no need to wait for it
1184          * to finish.
1185          */
1186         for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1187             sup = list_next(&softmac->smac_sup_list, sup)) {
1188                 if (!sup->su_is_arp) {
1189                         mp = head->b_next;
1190                         head->b_next = NULL;
1191                         softmac_wput_nondata(sup, head);
1192                         head = mp;
1193                 }
1194                 /*
1195                  * Add the switch request to the requests list of the stream.
1196                  */
1197                 req = list_head(&reqlist);
1198                 ASSERT(req != NULL);
1199                 list_remove(&reqlist, req);
1200                 list_insert_tail(&sup->su_req_list, req);
1201         }
1202 
1203         mutex_exit(&softmac->smac_fp_mutex);
1204         ASSERT(list_is_empty(&reqlist));
1205         list_destroy(&reqlist);
1206         return (0);
1207 fail:
1208         if (admin) {
1209                 softmac->smac_fastpath_admin_disabled = !disable;
1210         } else if (disable) {
1211                 softmac->smac_fp_disable_clients--;
1212         } else {
1213                 softmac->smac_fp_disable_clients++;
1214         }
1215 
1216         mutex_exit(&softmac->smac_fp_mutex);
1217         while ((req = list_head(&reqlist)) != NULL) {
1218                 list_remove(&reqlist, req);
1219                 kmem_free(req, sizeof (softmac_switch_req_t));
1220         }
1221         freemsgchain(head);
1222         list_destroy(&reqlist);
1223         return (err);
1224 }
1225 
1226 int
1227 softmac_fastpath_disable(void *arg)
1228 {
1229         return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1230 }
1231 
1232 void
1233 softmac_fastpath_enable(void *arg)
1234 {
1235         VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1236             B_FALSE) == 0);
1237 }
1238 
1239 void
1240 softmac_upperstream_close(softmac_upper_t *sup)
1241 {
1242         softmac_t               *softmac = sup->su_softmac;
1243         softmac_switch_req_t    *req;
1244 
1245         mutex_enter(&softmac->smac_fp_mutex);
1246 
1247         if (sup->su_mode == SOFTMAC_FASTPATH)
1248                 softmac_fastpath_tear(sup);
1249 
1250         if (sup->su_mode != SOFTMAC_UNKNOWN) {
1251                 list_remove(&softmac->smac_sup_list, sup);
1252                 sup->su_mode = SOFTMAC_UNKNOWN;
1253         }
1254 
1255         /*
1256          * Cleanup all the switch requests queueed on this stream.
1257          */
1258         while ((req = list_head(&sup->su_req_list)) != NULL) {
1259                 list_remove(&sup->su_req_list, req);
1260                 kmem_free(req, sizeof (softmac_switch_req_t));
1261         }
1262         mutex_exit(&softmac->smac_fp_mutex);
1263 }
1264 
1265 /*
1266  * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1267  * stream from the fastpath mode to the slowpath mode.
1268  */
1269 static void
1270 softmac_datapath_switch_done(softmac_upper_t *sup)
1271 {
1272         softmac_t               *softmac = sup->su_softmac;
1273         softmac_switch_req_t    *req;
1274         uint32_t                expected_mode;
1275 
1276         mutex_enter(&softmac->smac_fp_mutex);
1277         req = list_head(&sup->su_req_list);
1278         list_remove(&sup->su_req_list, req);
1279         expected_mode = req->ssq_expected_mode;
1280         kmem_free(req, sizeof (softmac_switch_req_t));
1281 
1282         if (expected_mode == sup->su_mode) {
1283                 mutex_exit(&softmac->smac_fp_mutex);
1284                 return;
1285         }
1286 
1287         ASSERT(!sup->su_bound);
1288         mutex_exit(&softmac->smac_fp_mutex);
1289 
1290         /*
1291          * It is fine if the expected mode is fast-path and we fail
1292          * to enable fastpath on this stream.
1293          */
1294         if (expected_mode == SOFTMAC_SLOWPATH)
1295                 softmac_fastpath_tear(sup);
1296         else
1297                 (void) softmac_fastpath_setup(sup);
1298 }