1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 
  27 #include <sys/types.h>
  28 #include <inet/common.h>
  29 #include <sys/stropts.h>
  30 #include <sys/modctl.h>
  31 #include <sys/dld.h>
  32 #include <sys/softmac_impl.h>
  33 
  34 dev_info_t              *softmac_dip = NULL;
  35 static kmem_cache_t     *softmac_upper_cachep;
  36 
  37 /*
  38  * This function is a generic open(9E) entry point into the softmac for
  39  * both the softmac module and the softmac driver.
  40  */
  41 static int softmac_cmn_open(queue_t *, dev_t *, int, int, cred_t *);
  42 
  43 /*
  44  * The following softmac_mod_xxx() functions are (9E) entry point functions for
  45  * the softmac module.
  46  */
  47 static int softmac_mod_close(queue_t *);
  48 static void softmac_mod_rput(queue_t *, mblk_t *);
  49 static void softmac_mod_wput(queue_t *, mblk_t *);
  50 static void softmac_mod_wsrv(queue_t *);
  51 
  52 /*
  53  * The following softmac_drv_xxx() functions are (9E) entry point functions for
  54  * the softmac driver.
  55  */
  56 static int softmac_drv_open(queue_t *, dev_t *, int, int, cred_t *);
  57 static int softmac_drv_close(queue_t *);
  58 static void softmac_drv_wput(queue_t *, mblk_t *);
  59 static void softmac_drv_wsrv(queue_t *);
  60 
  61 static int softmac_attach(dev_info_t *, ddi_attach_cmd_t);
  62 static int softmac_detach(dev_info_t *, ddi_detach_cmd_t);
  63 static int softmac_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
  64 
  65 static struct module_info softmac_modinfo = {
  66         0,
  67         SOFTMAC_DEV_NAME,
  68         0,
  69         INFPSZ,
  70         65536,
  71         1024
  72 };
  73 
  74 /*
  75  * hi-water mark is 1 because of the flow control mechanism implemented in
  76  * dld.  Refer to the comments in dld_str.c for details.
  77  */
  78 static struct module_info softmac_dld_modinfo = {
  79         0,
  80         SOFTMAC_DEV_NAME,
  81         0,
  82         INFPSZ,
  83         1,
  84         0
  85 };
  86 
  87 static struct qinit softmac_urinit = {
  88         (pfi_t)softmac_mod_rput,        /* qi_putp */
  89         (pfi_t)NULL,                    /* qi_srvp */
  90         softmac_cmn_open,               /* qi_qopen */
  91         softmac_mod_close,              /* qi_qclose */
  92         NULL,                           /* qi_qadmin */
  93         &softmac_modinfo            /* qi_minfo */
  94 };
  95 
  96 static struct qinit softmac_uwinit = {
  97         (pfi_t)softmac_mod_wput,        /* qi_putp */
  98         (pfi_t)softmac_mod_wsrv,        /* qi_srvp */
  99         NULL,                           /* qi_qopen */
 100         NULL,                           /* qi_qclose */
 101         NULL,                           /* qi_qadmin */
 102         &softmac_modinfo            /* qi_minfo */
 103 };
 104 
 105 static struct streamtab softmac_tab = {
 106         &softmac_urinit,    /* st_rdinit */
 107         &softmac_uwinit             /* st_wrinit */
 108 };
 109 
 110 DDI_DEFINE_STREAM_OPS(softmac_ops, nulldev, nulldev, softmac_attach,
 111     softmac_detach, nodev, softmac_info, D_MP, &softmac_tab,
 112     ddi_quiesce_not_supported);
 113 
 114 static struct qinit softmac_dld_r_qinit = {
 115         NULL, NULL, softmac_drv_open, softmac_drv_close, NULL,
 116         &softmac_dld_modinfo
 117 };
 118 
 119 static struct qinit softmac_dld_w_qinit = {
 120         (pfi_t)softmac_drv_wput, (pfi_t)softmac_drv_wsrv, NULL, NULL, NULL,
 121         &softmac_dld_modinfo
 122 };
 123 
 124 static struct fmodsw softmac_fmodsw = {
 125         SOFTMAC_DEV_NAME,
 126         &softmac_tab,
 127         D_MP
 128 };
 129 
 130 static struct modldrv softmac_modldrv = {
 131         &mod_driverops,
 132         "softmac driver",
 133         &softmac_ops
 134 };
 135 
 136 static struct modlstrmod softmac_modlstrmod = {
 137         &mod_strmodops,
 138         "softmac module",
 139         &softmac_fmodsw
 140 };
 141 
 142 static struct modlinkage softmac_modlinkage = {
 143         MODREV_1,
 144         &softmac_modlstrmod,
 145         &softmac_modldrv,
 146         NULL
 147 };
 148 
 149 static void softmac_dedicated_rx(void *, mac_resource_handle_t, mblk_t *,
 150     mac_header_info_t *);
 151 
 152 /*ARGSUSED*/
 153 static int
 154 softmac_upper_constructor(void *buf, void *arg, int kmflag)
 155 {
 156         softmac_upper_t *sup = buf;
 157 
 158         bzero(buf, sizeof (softmac_upper_t));
 159 
 160         mutex_init(&sup->su_mutex, NULL, MUTEX_DEFAULT, NULL);
 161         cv_init(&sup->su_cv, NULL, CV_DEFAULT, NULL);
 162         mutex_init(&sup->su_disp_mutex, NULL, MUTEX_DEFAULT, NULL);
 163         cv_init(&sup->su_disp_cv, NULL, CV_DEFAULT, NULL);
 164         list_create(&sup->su_req_list, sizeof (softmac_switch_req_t),
 165             offsetof(softmac_switch_req_t, ssq_req_list_node));
 166         return (0);
 167 }
 168 
 169 /*ARGSUSED*/
 170 static void
 171 softmac_upper_destructor(void *buf, void *arg)
 172 {
 173         softmac_upper_t *sup = buf;
 174 
 175         ASSERT(sup->su_slp == NULL);
 176         ASSERT(sup->su_pending_head == NULL && sup->su_pending_tail == NULL);
 177         ASSERT(!sup->su_dlpi_pending);
 178         ASSERT(!sup->su_active);
 179         ASSERT(!sup->su_closing);
 180         ASSERT(sup->su_tx_flow_mp == NULL);
 181         ASSERT(sup->su_tx_inprocess == 0);
 182         ASSERT(sup->su_mode == SOFTMAC_UNKNOWN);
 183         ASSERT(!sup->su_tx_busy);
 184         ASSERT(!sup->su_bound);
 185         ASSERT(!sup->su_taskq_scheduled);
 186         ASSERT(sup->su_tx_notify_func == NULL);
 187         ASSERT(sup->su_tx_notify_arg == NULL);
 188         ASSERT(list_is_empty(&sup->su_req_list));
 189 
 190         list_destroy(&sup->su_req_list);
 191         mutex_destroy(&sup->su_mutex);
 192         cv_destroy(&sup->su_cv);
 193         mutex_destroy(&sup->su_disp_mutex);
 194         cv_destroy(&sup->su_disp_cv);
 195 }
 196 
 197 int
 198 _init(void)
 199 {
 200         int     err;
 201 
 202         mac_init_ops(NULL, SOFTMAC_DEV_NAME);
 203         softmac_init();
 204 
 205         softmac_upper_cachep = kmem_cache_create("softmac_upper_cache",
 206             sizeof (softmac_upper_t), 0, softmac_upper_constructor,
 207             softmac_upper_destructor, NULL, NULL, NULL, 0);
 208         ASSERT(softmac_upper_cachep != NULL);
 209 
 210         if ((err = mod_install(&softmac_modlinkage)) != 0) {
 211                 softmac_fini();
 212                 return (err);
 213         }
 214 
 215         return (0);
 216 }
 217 
 218 int
 219 _fini(void)
 220 {
 221         int err;
 222 
 223         if (softmac_busy())
 224                 return (EBUSY);
 225 
 226         if ((err = mod_remove(&softmac_modlinkage)) != 0)
 227                 return (err);
 228 
 229         kmem_cache_destroy(softmac_upper_cachep);
 230         softmac_fini();
 231 
 232         return (0);
 233 }
 234 
 235 int
 236 _info(struct modinfo *modinfop)
 237 {
 238         return (mod_info(&softmac_modlinkage, modinfop));
 239 }
 240 
 241 static int
 242 softmac_cmn_open(queue_t *rq, dev_t *devp, int flag, int sflag, cred_t *credp)
 243 {
 244         softmac_lower_t *slp;
 245         /*
 246          * This is a self-cloning driver so that each queue should only
 247          * get opened once.
 248          */
 249         if (rq->q_ptr != NULL)
 250                 return (EBUSY);
 251 
 252         if (sflag == MODOPEN) {
 253                 /*
 254                  * This is the softmac module pushed over an underlying
 255                  * legacy device.  Initialize the lower structure.
 256                  */
 257                 if ((slp = kmem_zalloc(sizeof (*slp), KM_NOSLEEP)) == NULL)
 258                         return (ENOMEM);
 259 
 260                 slp->sl_wq = WR(rq);
 261                 cv_init(&slp->sl_cv, NULL, CV_DRIVER, NULL);
 262                 mutex_init(&slp->sl_mutex, NULL, MUTEX_DRIVER, NULL);
 263                 slp->sl_pending_prim = DL_PRIM_INVAL;
 264                 rq->q_ptr = WR(rq)->q_ptr = slp;
 265                 qprocson(rq);
 266                 return (0);
 267         }
 268 
 269         /*
 270          * Regular device open of a softmac DLPI node.  We modify
 271          * the queues' q_qinfo pointer such that all future STREAMS
 272          * operations will go through another set of entry points
 273          */
 274         rq->q_qinfo = &softmac_dld_r_qinit;
 275         WR(rq)->q_qinfo = &softmac_dld_w_qinit;
 276         return (softmac_drv_open(rq, devp, flag, sflag, credp));
 277 }
 278 
 279 static int
 280 softmac_mod_close(queue_t *rq)
 281 {
 282         softmac_lower_t *slp = rq->q_ptr;
 283 
 284         /*
 285          * Call the appropriate delete routine depending on whether this is
 286          * a module or device.
 287          */
 288         ASSERT(WR(rq)->q_next != NULL);
 289 
 290         qprocsoff(rq);
 291 
 292         slp->sl_softmac = NULL;
 293         slp->sl_lh = NULL;
 294 
 295         ASSERT(slp->sl_ack_mp == NULL);
 296         ASSERT(slp->sl_pending_prim == DL_PRIM_INVAL);
 297         ASSERT(slp->sl_pending_ioctl == B_FALSE);
 298 
 299         cv_destroy(&slp->sl_cv);
 300         mutex_destroy(&slp->sl_mutex);
 301 
 302         kmem_free(slp, sizeof (*slp));
 303         return (0);
 304 }
 305 
 306 static void
 307 softmac_mod_rput(queue_t *rq, mblk_t *mp)
 308 {
 309         softmac_lower_t         *slp = rq->q_ptr;
 310         softmac_lower_rxinfo_t  *rxinfo;
 311         union DL_primitives     *dlp;
 312 
 313         /*
 314          * This is the softmac module.
 315          */
 316         ASSERT(WR(rq)->q_next != NULL);
 317         ASSERT((mp->b_next == NULL) && (mp->b_prev == NULL));
 318 
 319         switch (DB_TYPE(mp)) {
 320         case M_DATA: {
 321 
 322                 /*
 323                  * If sl_rxinfo is non-NULL. This is dedicated-lower-stream
 324                  * created for fastpath. Directly call the rx callback.
 325                  */
 326                 if ((rxinfo = slp->sl_rxinfo) != NULL) {
 327                         rxinfo->slr_rx(rxinfo->slr_arg, NULL, mp, NULL);
 328                         break;
 329                 }
 330 
 331                 /*
 332                  * A shared-lower-stream. Some driver starts to send up
 333                  * packets even it not in the DL_IDLE state, where
 334                  * sl_softmac is not set yet. Drop the packet in this case.
 335                  */
 336                 if (slp->sl_softmac == NULL) {
 337                         freemsg(mp);
 338                         return;
 339                 }
 340 
 341                 /*
 342                  * If this message is looped back from the legacy devices,
 343                  * drop it as the Nemo framework will be responsible for
 344                  * looping it back by the mac_txloop() function.
 345                  */
 346                 if (mp->b_flag & MSGNOLOOP) {
 347                         freemsg(mp);
 348                         return;
 349                 }
 350 
 351                 /*
 352                  * This is the most common case.
 353                  */
 354                 if (DB_REF(mp) == 1) {
 355                         ASSERT(slp->sl_softmac != NULL);
 356                         mac_rx(slp->sl_softmac->smac_mh, NULL, mp);
 357                         return;
 358                 } else {
 359                         softmac_rput_process_data(slp, mp);
 360                 }
 361                 break;
 362         }
 363         case M_PROTO:
 364         case M_PCPROTO:
 365                 if (MBLKL(mp) < sizeof (dlp->dl_primitive)) {
 366                         freemsg(mp);
 367                         break;
 368                 }
 369                 dlp = (union DL_primitives *)mp->b_rptr;
 370                 if (dlp->dl_primitive == DL_UNITDATA_IND) {
 371 
 372                         if ((rxinfo = slp->sl_rxinfo) != NULL) {
 373                                 softmac_dedicated_rx(slp->sl_sup, NULL, mp,
 374                                     NULL);
 375                                 break;
 376                         }
 377 
 378                         cmn_err(CE_WARN, "got unexpected %s message",
 379                             dl_primstr(DL_UNITDATA_IND));
 380                         freemsg(mp);
 381                         break;
 382                 }
 383                 /*FALLTHROUGH*/
 384         default:
 385                 softmac_rput_process_notdata(rq, slp->sl_sup, mp);
 386                 break;
 387         }
 388 }
 389 
 390 static void
 391 softmac_mod_wput(queue_t *wq, mblk_t *mp)
 392 {
 393         /*
 394          * This is the softmac module
 395          */
 396         ASSERT(wq->q_next != NULL);
 397 
 398         switch (DB_TYPE(mp)) {
 399         case M_IOCTL: {
 400                 struct iocblk           *ioc = (struct iocblk *)mp->b_rptr;
 401 
 402                 switch (ioc->ioc_cmd) {
 403                 case SMAC_IOC_START: {
 404                         softmac_lower_t         *slp = wq->q_ptr;
 405                         smac_ioc_start_t        *arg;
 406 
 407                         if (ioc->ioc_count != sizeof (*arg)) {
 408                                 miocnak(wq, mp, 0, EINVAL);
 409                                 break;
 410                         }
 411 
 412                         /*
 413                          * Assign the devname and perstream handle of the
 414                          * specific lower stream and return it as a part
 415                          * of the ioctl.
 416                          */
 417                         arg = (smac_ioc_start_t *)mp->b_cont->b_rptr;
 418                         arg->si_slp = slp;
 419                         miocack(wq, mp, sizeof (*arg), 0);
 420                         break;
 421                 }
 422                 default:
 423                         miocnak(wq, mp, 0, EINVAL);
 424                         break;
 425                 }
 426                 break;
 427         }
 428         default:
 429                 freemsg(mp);
 430                 break;
 431         }
 432 }
 433 
 434 static void
 435 softmac_mod_wsrv(queue_t *wq)
 436 {
 437         softmac_lower_t *slp = wq->q_ptr;
 438 
 439         /*
 440          * This is the softmac module
 441          */
 442         ASSERT(wq->q_next != NULL);
 443 
 444         /*
 445          * Inform that the tx resource is available; mac_tx_update() will
 446          * inform all the upper streams sharing this lower stream.
 447          */
 448         if (slp->sl_sup != NULL)
 449                 qenable(slp->sl_sup->su_wq);
 450         else if (slp->sl_softmac != NULL)
 451                 mac_tx_update(slp->sl_softmac->smac_mh);
 452 }
 453 
 454 static int
 455 softmac_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 456 {
 457         ASSERT(ddi_get_instance(dip) == 0);
 458 
 459         if (cmd != DDI_ATTACH)
 460                 return (DDI_FAILURE);
 461 
 462         softmac_dip = dip;
 463 
 464         return (DDI_SUCCESS);
 465 }
 466 
 467 /* ARGSUSED */
 468 static int
 469 softmac_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 470 {
 471         if (cmd != DDI_DETACH)
 472                 return (DDI_FAILURE);
 473 
 474         softmac_dip = NULL;
 475         return (DDI_SUCCESS);
 476 }
 477 
 478 /* ARGSUSED */
 479 static int
 480 softmac_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
 481 {
 482         switch (infocmd) {
 483         case DDI_INFO_DEVT2DEVINFO:
 484                 if (softmac_dip != NULL) {
 485                         *result = softmac_dip;
 486                         return (DDI_SUCCESS);
 487                 }
 488                 break;
 489 
 490         case DDI_INFO_DEVT2INSTANCE:
 491                 *result = NULL;
 492                 return (DDI_SUCCESS);
 493 
 494         }
 495 
 496         return (DDI_FAILURE);
 497 }
 498 
 499 /*ARGSUSED*/
 500 static void
 501 softmac_dedicated_rx(void *arg, mac_resource_handle_t mrh, mblk_t *mp,
 502     mac_header_info_t *mhip)
 503 {
 504         queue_t *rq = ((softmac_upper_t *)arg)->su_rq;
 505 
 506         if (canputnext(rq))
 507                 putnext(rq, mp);
 508         else
 509                 freemsg(mp);
 510 }
 511 
 512 /*ARGSUSED*/
 513 static int
 514 softmac_drv_open(queue_t *rq, dev_t *devp, int flag, int sflag, cred_t *credp)
 515 {
 516         softmac_upper_t *sup = NULL;
 517         softmac_t       *softmac;
 518         int             err = 0;
 519 
 520         /*
 521          * This is a softmac device created for a legacy device, find the
 522          * associated softmac and initialize the softmac_upper_t structure.
 523          */
 524         if ((err = softmac_hold(*devp, &softmac)) != 0)
 525                 return (err);
 526 
 527         sup = kmem_cache_alloc(softmac_upper_cachep, KM_NOSLEEP);
 528         if (sup == NULL) {
 529                 err = ENOMEM;
 530                 goto fail;
 531         }
 532 
 533         ASSERT(list_is_empty(&sup->su_req_list));
 534 
 535         if ((sup->su_tx_flow_mp = allocb(1, BPRI_HI)) == NULL) {
 536                 err = ENOMEM;
 537                 goto fail;
 538         }
 539 
 540         sup->su_rq = rq;
 541         sup->su_wq = WR(rq);
 542         sup->su_softmac = softmac;
 543         sup->su_mode = SOFTMAC_UNKNOWN;
 544 
 545         sup->su_rxinfo.slr_arg = sup;
 546         sup->su_rxinfo.slr_rx = softmac_dedicated_rx;
 547         sup->su_direct_rxinfo.slr_arg = sup;
 548         sup->su_direct_rxinfo.slr_rx = softmac_dedicated_rx;
 549 
 550         if ((err = dld_str_open(rq, devp, sup)) != 0) {
 551                 freeb(sup->su_tx_flow_mp);
 552                 sup->su_tx_flow_mp = NULL;
 553                 goto fail;
 554         }
 555 
 556         return (0);
 557 
 558 fail:
 559         if (sup != NULL)
 560                 kmem_cache_free(softmac_upper_cachep, sup);
 561         softmac_rele(softmac);
 562         return (err);
 563 }
 564 
 565 static int
 566 softmac_drv_close(queue_t *rq)
 567 {
 568         softmac_upper_t *sup = dld_str_private(rq);
 569         softmac_t       *softmac = sup->su_softmac;
 570 
 571         ASSERT(WR(rq)->q_next == NULL);
 572 
 573         qprocsoff(rq);
 574 
 575         ASSERT(sup->su_tx_inprocess == 0);
 576 
 577         /*
 578          * Wait until the pending request are processed by the worker thread.
 579          */
 580         mutex_enter(&sup->su_disp_mutex);
 581         sup->su_closing = B_TRUE;
 582         while (sup->su_dlpi_pending)
 583                 cv_wait(&sup->su_disp_cv, &sup->su_disp_mutex);
 584         mutex_exit(&sup->su_disp_mutex);
 585 
 586         softmac_upperstream_close(sup);
 587 
 588         if (sup->su_tx_flow_mp != NULL) {
 589                 freeb(sup->su_tx_flow_mp);
 590                 sup->su_tx_flow_mp = NULL;
 591         }
 592 
 593         if (sup->su_active) {
 594                 mutex_enter(&softmac->smac_active_mutex);
 595                 softmac->smac_nactive--;
 596                 mutex_exit(&softmac->smac_active_mutex);
 597                 sup->su_active = B_FALSE;
 598         }
 599 
 600         sup->su_bound = B_FALSE;
 601         sup->su_softmac = NULL;
 602         sup->su_closing = B_FALSE;
 603 
 604         kmem_cache_free(softmac_upper_cachep, sup);
 605 
 606         softmac_rele(softmac);
 607         return (dld_str_close(rq));
 608 }
 609 
 610 static void
 611 softmac_drv_wput(queue_t *wq, mblk_t *mp)
 612 {
 613         softmac_upper_t *sup = dld_str_private(wq);
 614         t_uscalar_t     prim;
 615 
 616         ASSERT(wq->q_next == NULL);
 617 
 618         switch (DB_TYPE(mp)) {
 619         case M_DATA:
 620         case M_MULTIDATA:
 621                 softmac_wput_data(sup, mp);
 622                 break;
 623         case M_PROTO:
 624         case M_PCPROTO:
 625 
 626                 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
 627                         freemsg(mp);
 628                         return;
 629                 }
 630 
 631                 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
 632                 if (prim == DL_UNITDATA_REQ) {
 633                         softmac_wput_data(sup, mp);
 634                         return;
 635                 }
 636 
 637                 softmac_wput_nondata(sup, mp);
 638                 break;
 639         default:
 640                 softmac_wput_nondata(sup, mp);
 641                 break;
 642         }
 643 }
 644 
 645 static void
 646 softmac_drv_wsrv(queue_t *wq)
 647 {
 648         softmac_upper_t *sup = dld_str_private(wq);
 649 
 650         ASSERT(wq->q_next == NULL);
 651 
 652         mutex_enter(&sup->su_mutex);
 653         if (sup->su_mode != SOFTMAC_FASTPATH) {
 654                 /*
 655                  * Bump su_tx_inprocess so that su_mode won't change.
 656                  */
 657                 sup->su_tx_inprocess++;
 658                 mutex_exit(&sup->su_mutex);
 659                 dld_wsrv(wq);
 660                 mutex_enter(&sup->su_mutex);
 661                 if (--sup->su_tx_inprocess == 0)
 662                         cv_signal(&sup->su_cv);
 663         } else if (sup->su_tx_busy && SOFTMAC_CANPUTNEXT(sup->su_slp->sl_wq)) {
 664                 /*
 665                  * The flow-conctol of the dedicated-lower-stream is
 666                  * relieved. If DLD_CAPAB_DIRECT is enabled, call tx_notify
 667                  * callback to relieve the flow-control of the specific client,
 668                  * otherwise relieve the flow-control of all the upper-stream
 669                  * using the traditional STREAM mechanism.
 670                  */
 671                 if (sup->su_tx_notify_func != NULL) {
 672                         sup->su_tx_inprocess++;
 673                         mutex_exit(&sup->su_mutex);
 674                         sup->su_tx_notify_func(sup->su_tx_notify_arg,
 675                             (mac_tx_cookie_t)sup);
 676                         mutex_enter(&sup->su_mutex);
 677                         if (--sup->su_tx_inprocess == 0)
 678                                 cv_signal(&sup->su_cv);
 679                 }
 680                 ASSERT(sup->su_tx_flow_mp == NULL);
 681                 VERIFY((sup->su_tx_flow_mp = getq(wq)) != NULL);
 682                 sup->su_tx_busy = B_FALSE;
 683         }
 684         mutex_exit(&sup->su_mutex);
 685 }