1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  22 /*        All Rights Reserved   */
  23 
  24 
  25 /*
  26  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  27  * Use is subject to license terms.
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/sysmacros.h>
  32 #include <sys/param.h>
  33 #include <sys/errno.h>
  34 #include <sys/signal.h>
  35 #include <sys/proc.h>
  36 #include <sys/conf.h>
  37 #include <sys/cred.h>
  38 #include <sys/user.h>
  39 #include <sys/vnode.h>
  40 #include <sys/file.h>
  41 #include <sys/session.h>
  42 #include <sys/stream.h>
  43 #include <sys/strsubr.h>
  44 #include <sys/stropts.h>
  45 #include <sys/poll.h>
  46 #include <sys/systm.h>
  47 #include <sys/cpuvar.h>
  48 #include <sys/uio.h>
  49 #include <sys/cmn_err.h>
  50 #include <sys/priocntl.h>
  51 #include <sys/procset.h>
  52 #include <sys/vmem.h>
  53 #include <sys/bitmap.h>
  54 #include <sys/kmem.h>
  55 #include <sys/siginfo.h>
  56 #include <sys/vtrace.h>
  57 #include <sys/callb.h>
  58 #include <sys/debug.h>
  59 #include <sys/modctl.h>
  60 #include <sys/vmsystm.h>
  61 #include <vm/page.h>
  62 #include <sys/atomic.h>
  63 #include <sys/suntpi.h>
  64 #include <sys/strlog.h>
  65 #include <sys/promif.h>
  66 #include <sys/project.h>
  67 #include <sys/vm.h>
  68 #include <sys/taskq.h>
  69 #include <sys/sunddi.h>
  70 #include <sys/sunldi_impl.h>
  71 #include <sys/strsun.h>
  72 #include <sys/isa_defs.h>
  73 #include <sys/multidata.h>
  74 #include <sys/pattr.h>
  75 #include <sys/strft.h>
  76 #include <sys/fs/snode.h>
  77 #include <sys/zone.h>
  78 #include <sys/open.h>
  79 #include <sys/sunldi.h>
  80 #include <sys/sad.h>
  81 #include <sys/netstack.h>
  82 
  83 #define O_SAMESTR(q)    (((q)->q_next) && \
  84         (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
  85 
  86 /*
  87  * WARNING:
  88  * The variables and routines in this file are private, belonging
  89  * to the STREAMS subsystem. These should not be used by modules
  90  * or drivers. Compatibility will not be guaranteed.
  91  */
  92 
  93 /*
  94  * Id value used to distinguish between different multiplexor links.
  95  */
  96 static int32_t lnk_id = 0;
  97 
  98 #define STREAMS_LOPRI MINCLSYSPRI
  99 static pri_t streams_lopri = STREAMS_LOPRI;
 100 
 101 #define STRSTAT(x)      (str_statistics.x.value.ui64++)
 102 typedef struct str_stat {
 103         kstat_named_t   sqenables;
 104         kstat_named_t   stenables;
 105         kstat_named_t   syncqservice;
 106         kstat_named_t   freebs;
 107         kstat_named_t   qwr_outer;
 108         kstat_named_t   rservice;
 109         kstat_named_t   strwaits;
 110         kstat_named_t   taskqfails;
 111         kstat_named_t   bufcalls;
 112         kstat_named_t   qhelps;
 113         kstat_named_t   qremoved;
 114         kstat_named_t   sqremoved;
 115         kstat_named_t   bcwaits;
 116         kstat_named_t   sqtoomany;
 117 } str_stat_t;
 118 
 119 static str_stat_t str_statistics = {
 120         { "sqenables",          KSTAT_DATA_UINT64 },
 121         { "stenables",          KSTAT_DATA_UINT64 },
 122         { "syncqservice",       KSTAT_DATA_UINT64 },
 123         { "freebs",             KSTAT_DATA_UINT64 },
 124         { "qwr_outer",          KSTAT_DATA_UINT64 },
 125         { "rservice",           KSTAT_DATA_UINT64 },
 126         { "strwaits",           KSTAT_DATA_UINT64 },
 127         { "taskqfails",         KSTAT_DATA_UINT64 },
 128         { "bufcalls",           KSTAT_DATA_UINT64 },
 129         { "qhelps",             KSTAT_DATA_UINT64 },
 130         { "qremoved",           KSTAT_DATA_UINT64 },
 131         { "sqremoved",          KSTAT_DATA_UINT64 },
 132         { "bcwaits",            KSTAT_DATA_UINT64 },
 133         { "sqtoomany",          KSTAT_DATA_UINT64 },
 134 };
 135 
 136 static kstat_t *str_kstat;
 137 
 138 /*
 139  * qrunflag was used previously to control background scheduling of queues. It
 140  * is not used anymore, but kept here in case some module still wants to access
 141  * it via qready() and setqsched macros.
 142  */
 143 char qrunflag;                  /*  Unused */
 144 
 145 /*
 146  * Most of the streams scheduling is done via task queues. Task queues may fail
 147  * for non-sleep dispatches, so there are two backup threads servicing failed
 148  * requests for queues and syncqs. Both of these threads also service failed
 149  * dispatches freebs requests. Queues are put in the list specified by `qhead'
 150  * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
 151  * requests are put into `freebs_list' which has no tail pointer. All three
 152  * lists are protected by a single `service_queue' lock and use
 153  * `services_to_run' condition variable for signaling background threads. Use of
 154  * a single lock should not be a problem because it is only used under heavy
 155  * loads when task queues start to fail and at that time it may be a good idea
 156  * to throttle scheduling requests.
 157  *
 158  * NOTE: queues and syncqs should be scheduled by two separate threads because
 159  * queue servicing may be blocked waiting for a syncq which may be also
 160  * scheduled for background execution. This may create a deadlock when only one
 161  * thread is used for both.
 162  */
 163 
 164 static taskq_t *streams_taskq;          /* Used for most STREAMS scheduling */
 165 
 166 static kmutex_t service_queue;          /* protects all of servicing vars */
 167 static kcondvar_t services_to_run;      /* wake up background service thread */
 168 static kcondvar_t syncqs_to_run;        /* wake up background service thread */
 169 
 170 /*
 171  * List of queues scheduled for background processing due to lack of resources
 172  * in the task queues. Protected by service_queue lock;
 173  */
 174 static struct queue *qhead;
 175 static struct queue *qtail;
 176 
 177 /*
 178  * Same list for syncqs
 179  */
 180 static syncq_t *sqhead;
 181 static syncq_t *sqtail;
 182 
 183 static mblk_t *freebs_list;     /* list of buffers to free */
 184 
 185 /*
 186  * Backup threads for servicing queues and syncqs
 187  */
 188 kthread_t *streams_qbkgrnd_thread;
 189 kthread_t *streams_sqbkgrnd_thread;
 190 
 191 /*
 192  * Bufcalls related variables.
 193  */
 194 struct bclist   strbcalls;      /* list of waiting bufcalls */
 195 kmutex_t        strbcall_lock;  /* protects bufcall list (strbcalls) */
 196 kcondvar_t      strbcall_cv;    /* Signaling when a bufcall is added */
 197 kmutex_t        bcall_monitor;  /* sleep/wakeup style monitor */
 198 kcondvar_t      bcall_cv;       /* wait 'till executing bufcall completes */
 199 kthread_t       *bc_bkgrnd_thread; /* Thread to service bufcall requests */
 200 
 201 kmutex_t        strresources;   /* protects global resources */
 202 kmutex_t        muxifier;       /* single-threads multiplexor creation */
 203 
 204 static void     *str_stack_init(netstackid_t stackid, netstack_t *ns);
 205 static void     str_stack_shutdown(netstackid_t stackid, void *arg);
 206 static void     str_stack_fini(netstackid_t stackid, void *arg);
 207 
 208 /*
 209  * run_queues is no longer used, but is kept in case some 3rd party
 210  * module/driver decides to use it.
 211  */
 212 int run_queues = 0;
 213 
 214 /*
 215  * sq_max_size is the depth of the syncq (in number of messages) before
 216  * qfill_syncq() starts QFULL'ing destination queues. As its primary
 217  * consumer - IP is no longer D_MTPERMOD, but there may be other
 218  * modules/drivers depend on this syncq flow control, we prefer to
 219  * choose a large number as the default value. For potential
 220  * performance gain, this value is tunable in /etc/system.
 221  */
 222 int sq_max_size = 10000;
 223 
 224 /*
 225  * The number of ciputctrl structures per syncq and stream we create when
 226  * needed.
 227  */
 228 int n_ciputctrl;
 229 int max_n_ciputctrl = 16;
 230 /*
 231  * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
 232  */
 233 int min_n_ciputctrl = 2;
 234 
 235 /*
 236  * Per-driver/module syncqs
 237  * ========================
 238  *
 239  * For drivers/modules that use PERMOD or outer syncqs we keep a list of
 240  * perdm structures, new entries being added (and new syncqs allocated) when
 241  * setq() encounters a module/driver with a streamtab that it hasn't seen
 242  * before.
 243  * The reason for this mechanism is that some modules and drivers share a
 244  * common streamtab and it is necessary for those modules and drivers to also
 245  * share a common PERMOD syncq.
 246  *
 247  * perdm_list --> dm_str == streamtab_1
 248  *                dm_sq == syncq_1
 249  *                dm_ref
 250  *                dm_next --> dm_str == streamtab_2
 251  *                            dm_sq == syncq_2
 252  *                            dm_ref
 253  *                            dm_next --> ... NULL
 254  *
 255  * The dm_ref field is incremented for each new driver/module that takes
 256  * a reference to the perdm structure and hence shares the syncq.
 257  * References are held in the fmodsw_impl_t structure for each STREAMS module
 258  * or the dev_impl array (indexed by device major number) for each driver.
 259  *
 260  * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
 261  *                   ^                 ^ ^               ^
 262  *                   |  ______________/  |               |
 263  *                   | /                 |               |
 264  * dev_impl:     ...|x|y|...          module A        module B
 265  *
 266  * When a module/driver is unloaded the reference count is decremented and,
 267  * when it falls to zero, the perdm structure is removed from the list and
 268  * the syncq is freed (see rele_dm()).
 269  */
 270 perdm_t *perdm_list = NULL;
 271 static krwlock_t perdm_rwlock;
 272 cdevsw_impl_t *devimpl;
 273 
 274 extern struct qinit strdata;
 275 extern struct qinit stwdata;
 276 
 277 static void runservice(queue_t *);
 278 static void streams_bufcall_service(void);
 279 static void streams_qbkgrnd_service(void);
 280 static void streams_sqbkgrnd_service(void);
 281 static syncq_t *new_syncq(void);
 282 static void free_syncq(syncq_t *);
 283 static void outer_insert(syncq_t *, syncq_t *);
 284 static void outer_remove(syncq_t *, syncq_t *);
 285 static void write_now(syncq_t *);
 286 static void clr_qfull(queue_t *);
 287 static void runbufcalls(void);
 288 static void sqenable(syncq_t *);
 289 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
 290 static void wait_q_syncq(queue_t *);
 291 static void backenable_insertedq(queue_t *);
 292 
 293 static void queue_service(queue_t *);
 294 static void stream_service(stdata_t *);
 295 static void syncq_service(syncq_t *);
 296 static void qwriter_outer_service(syncq_t *);
 297 static void mblk_free(mblk_t *);
 298 #ifdef DEBUG
 299 static int qprocsareon(queue_t *);
 300 #endif
 301 
 302 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
 303 static void reset_nfsrv_ptr(queue_t *, queue_t *);
 304 void set_qfull(queue_t *);
 305 
 306 static void sq_run_events(syncq_t *);
 307 static int propagate_syncq(queue_t *);
 308 
 309 static void     blocksq(syncq_t *, ushort_t, int);
 310 static void     unblocksq(syncq_t *, ushort_t, int);
 311 static int      dropsq(syncq_t *, uint16_t);
 312 static void     emptysq(syncq_t *);
 313 static sqlist_t *sqlist_alloc(struct stdata *, int);
 314 static void     sqlist_free(sqlist_t *);
 315 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t);
 316 static void     sqlist_insert(sqlist_t *, syncq_t *);
 317 static void     sqlist_insertall(sqlist_t *, queue_t *);
 318 
 319 static void     strsetuio(stdata_t *);
 320 
 321 struct kmem_cache *stream_head_cache;
 322 struct kmem_cache *queue_cache;
 323 struct kmem_cache *syncq_cache;
 324 struct kmem_cache *qband_cache;
 325 struct kmem_cache *linkinfo_cache;
 326 struct kmem_cache *ciputctrl_cache = NULL;
 327 
 328 static linkinfo_t *linkinfo_list;
 329 
 330 /* Global esballoc throttling queue */
 331 static esb_queue_t system_esbq;
 332 
 333 /* Array of esballoc throttling queues, of length esbq_nelem */
 334 static esb_queue_t *volatile system_esbq_array;
 335 static int esbq_nelem;
 336 static kmutex_t esbq_lock;
 337 static int esbq_log2_cpus_per_q = 0;
 338 
 339 /* Scale the system_esbq length by setting number of CPUs per queue. */
 340 uint_t esbq_cpus_per_q = 1;
 341 
 342 /*
 343  * esballoc tunable parameters.
 344  */
 345 int             esbq_max_qlen = 0x16;   /* throttled queue length */
 346 clock_t         esbq_timeout = 0x8;     /* timeout to process esb queue */
 347 
 348 /*
 349  * Routines to handle esballoc queueing.
 350  */
 351 static void esballoc_process_queue(esb_queue_t *);
 352 static void esballoc_enqueue_mblk(mblk_t *);
 353 static void esballoc_timer(void *);
 354 static void esballoc_set_timer(esb_queue_t *, clock_t);
 355 static void esballoc_mblk_free(mblk_t *);
 356 
 357 /*
 358  *  Qinit structure and Module_info structures
 359  *      for passthru read and write queues
 360  */
 361 
 362 static void pass_wput(queue_t *, mblk_t *);
 363 static queue_t *link_addpassthru(stdata_t *);
 364 static void link_rempassthru(queue_t *);
 365 
 366 struct  module_info passthru_info = {
 367         0,
 368         "passthru",
 369         0,
 370         INFPSZ,
 371         STRHIGH,
 372         STRLOW
 373 };
 374 
 375 struct  qinit passthru_rinit = {
 376         (int (*)())putnext,
 377         NULL,
 378         NULL,
 379         NULL,
 380         NULL,
 381         &passthru_info,
 382         NULL
 383 };
 384 
 385 struct  qinit passthru_winit = {
 386         (int (*)()) pass_wput,
 387         NULL,
 388         NULL,
 389         NULL,
 390         NULL,
 391         &passthru_info,
 392         NULL
 393 };
 394 
 395 /*
 396  * Verify correctness of list head/tail pointers.
 397  */
 398 #define LISTCHECK(head, tail, link) {                           \
 399         EQUIV(head, tail);                                      \
 400         IMPLY(tail != NULL, tail->link == NULL);             \
 401 }
 402 
 403 /*
 404  * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
 405  * using a `link' field.
 406  */
 407 #define ENQUEUE(el, head, tail, link) {                         \
 408         ASSERT(el->link == NULL);                            \
 409         LISTCHECK(head, tail, link);                            \
 410         if (head == NULL)                                       \
 411                 head = el;                                      \
 412         else                                                    \
 413                 tail->link = el;                             \
 414         tail = el;                                              \
 415 }
 416 
 417 /*
 418  * Dequeue the first element of the list denoted by `head' and `tail' pointers
 419  * using a `link' field and put result into `el'.
 420  */
 421 #define DQ(el, head, tail, link) {                              \
 422         LISTCHECK(head, tail, link);                            \
 423         el = head;                                              \
 424         if (head != NULL) {                                     \
 425                 head = head->link;                           \
 426                 if (head == NULL)                               \
 427                         tail = NULL;                            \
 428                 el->link = NULL;                             \
 429         }                                                       \
 430 }
 431 
 432 /*
 433  * Remove `el' from the list using `chase' and `curr' pointers and return result
 434  * in `succeed'.
 435  */
 436 #define RMQ(el, head, tail, link, chase, curr, succeed) {       \
 437         LISTCHECK(head, tail, link);                            \
 438         chase = NULL;                                           \
 439         succeed = 0;                                            \
 440         for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
 441                 chase = curr;                                   \
 442         if (curr != NULL) {                                     \
 443                 succeed = 1;                                    \
 444                 ASSERT(curr == el);                             \
 445                 if (chase != NULL)                              \
 446                         chase->link = curr->link;         \
 447                 else                                            \
 448                         head = curr->link;                   \
 449                 curr->link = NULL;                           \
 450                 if (curr == tail)                               \
 451                         tail = chase;                           \
 452         }                                                       \
 453         LISTCHECK(head, tail, link);                            \
 454 }
 455 
 456 /* Handling of delayed messages on the inner syncq. */
 457 
 458 /*
 459  * DEBUG versions should use function versions (to simplify tracing) and
 460  * non-DEBUG kernels should use macro versions.
 461  */
 462 
 463 /*
 464  * Put a queue on the syncq list of queues.
 465  * Assumes SQLOCK held.
 466  */
 467 #define SQPUT_Q(sq, qp)                                                 \
 468 {                                                                       \
 469         ASSERT(MUTEX_HELD(SQLOCK(sq)));                                 \
 470         if (!(qp->q_sqflags & Q_SQQUEUED)) {                             \
 471                 /* The queue should not be linked anywhere */           \
 472                 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
 473                 /* Head and tail may only be NULL simultaneously */     \
 474                 EQUIV(sq->sq_head, sq->sq_tail);                  \
 475                 /* Queue may be only enqueued on its syncq */           \
 476                 ASSERT(sq == qp->q_syncq);                           \
 477                 /* Check the correctness of SQ_MESSAGES flag */         \
 478                 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES));     \
 479                 /* Sanity check first/last elements of the list */      \
 480                 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
 481                 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
 482                 /*                                                      \
 483                  * Sanity check of priority field: empty queue should   \
 484                  * have zero priority                                   \
 485                  * and nqueues equal to zero.                           \
 486                  */                                                     \
 487                 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0);              \
 488                 /* Sanity check of sq_nqueues field */                  \
 489                 EQUIV(sq->sq_head, sq->sq_nqueues);                       \
 490                 if (sq->sq_head == NULL) {                           \
 491                         sq->sq_head = sq->sq_tail = qp;                   \
 492                         sq->sq_flags |= SQ_MESSAGES;                 \
 493                 } else if (qp->q_spri == 0) {                                \
 494                         qp->q_sqprev = sq->sq_tail;                       \
 495                         sq->sq_tail->q_sqnext = qp;                       \
 496                         sq->sq_tail = qp;                            \
 497                 } else {                                                \
 498                         /*                                              \
 499                          * Put this queue in priority order: higher     \
 500                          * priority gets closer to the head.            \
 501                          */                                             \
 502                         queue_t **qpp = &sq->sq_tail;                    \
 503                         queue_t *qnext = NULL;                          \
 504                                                                         \
 505                         while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
 506                                 qnext = *qpp;                           \
 507                                 qpp = &(*qpp)->q_sqprev;         \
 508                         }                                               \
 509                         qp->q_sqnext = qnext;                                \
 510                         qp->q_sqprev = *qpp;                         \
 511                         if (*qpp != NULL) {                             \
 512                                 (*qpp)->q_sqnext = qp;                       \
 513                         } else {                                        \
 514                                 sq->sq_head = qp;                    \
 515                                 sq->sq_pri = sq->sq_head->q_spri;      \
 516                         }                                               \
 517                         *qpp = qp;                                      \
 518                 }                                                       \
 519                 qp->q_sqflags |= Q_SQQUEUED;                         \
 520                 qp->q_sqtstamp = ddi_get_lbolt();                    \
 521                 sq->sq_nqueues++;                                    \
 522         }                                                               \
 523 }
 524 
 525 /*
 526  * Remove a queue from the syncq list
 527  * Assumes SQLOCK held.
 528  */
 529 #define SQRM_Q(sq, qp)                                                  \
 530         {                                                               \
 531                 ASSERT(MUTEX_HELD(SQLOCK(sq)));                         \
 532                 ASSERT(qp->q_sqflags & Q_SQQUEUED);                      \
 533                 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL);       \
 534                 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0);               \
 535                 /* Check that the queue is actually in the list */      \
 536                 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp);        \
 537                 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp);        \
 538                 ASSERT(sq->sq_nqueues != 0);                         \
 539                 if (qp->q_sqprev == NULL) {                          \
 540                         /* First queue on list, make head q_sqnext */   \
 541                         sq->sq_head = qp->q_sqnext;                       \
 542                 } else {                                                \
 543                         /* Make prev->next == next */                        \
 544                         qp->q_sqprev->q_sqnext = qp->q_sqnext;         \
 545                 }                                                       \
 546                 if (qp->q_sqnext == NULL) {                          \
 547                         /* Last queue on list, make tail sqprev */      \
 548                         sq->sq_tail = qp->q_sqprev;                       \
 549                 } else {                                                \
 550                         /* Make next->prev == prev */                        \
 551                         qp->q_sqnext->q_sqprev = qp->q_sqprev;         \
 552                 }                                                       \
 553                 /* clear out references on this queue */                \
 554                 qp->q_sqprev = qp->q_sqnext = NULL;                       \
 555                 qp->q_sqflags &= ~Q_SQQUEUED;                            \
 556                 /* If there is nothing queued, clear SQ_MESSAGES */     \
 557                 if (sq->sq_head != NULL) {                           \
 558                         sq->sq_pri = sq->sq_head->q_spri;              \
 559                 } else  {                                               \
 560                         sq->sq_flags &= ~SQ_MESSAGES;                    \
 561                         sq->sq_pri = 0;                                      \
 562                 }                                                       \
 563                 sq->sq_nqueues--;                                    \
 564                 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL ||    \
 565                     (sq->sq_flags & SQ_QUEUED) == 0);                    \
 566         }
 567 
 568 /* Hide the definition from the header file. */
 569 #ifdef SQPUT_MP
 570 #undef SQPUT_MP
 571 #endif
 572 
 573 /*
 574  * Put a message on the queue syncq.
 575  * Assumes QLOCK held.
 576  */
 577 #define SQPUT_MP(qp, mp)                                                \
 578         {                                                               \
 579                 ASSERT(MUTEX_HELD(QLOCK(qp)));                          \
 580                 ASSERT(qp->q_sqhead == NULL ||                               \
 581                     (qp->q_sqtail != NULL &&                         \
 582                     qp->q_sqtail->b_next == NULL));                       \
 583                 qp->q_syncqmsgs++;                                   \
 584                 ASSERT(qp->q_syncqmsgs != 0);        /* Wraparound */        \
 585                 if (qp->q_sqhead == NULL) {                          \
 586                         qp->q_sqhead = qp->q_sqtail = mp;         \
 587                 } else {                                                \
 588                         qp->q_sqtail->b_next = mp;                        \
 589                         qp->q_sqtail = mp;                           \
 590                 }                                                       \
 591                 ASSERT(qp->q_syncqmsgs > 0);                              \
 592                 set_qfull(qp);                                          \
 593         }
 594 
 595 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) {                                \
 596                 ASSERT(MUTEX_HELD(SQLOCK(sq)));                         \
 597                 if ((sq)->sq_ciputctrl != NULL) {                    \
 598                         int i;                                          \
 599                         int nlocks = (sq)->sq_nciputctrl;            \
 600                         ciputctrl_t *cip = (sq)->sq_ciputctrl;               \
 601                         ASSERT((sq)->sq_type & SQ_CIPUT);                \
 602                         for (i = 0; i <= nlocks; i++) {                      \
 603                                 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
 604                                 cip[i].ciputctrl_count |= SQ_FASTPUT;   \
 605                         }                                               \
 606                 }                                                       \
 607         }
 608 
 609 
 610 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) {                                \
 611                 ASSERT(MUTEX_HELD(SQLOCK(sq)));                         \
 612                 if ((sq)->sq_ciputctrl != NULL) {                    \
 613                         int i;                                          \
 614                         int nlocks = (sq)->sq_nciputctrl;            \
 615                         ciputctrl_t *cip = (sq)->sq_ciputctrl;               \
 616                         ASSERT((sq)->sq_type & SQ_CIPUT);                \
 617                         for (i = 0; i <= nlocks; i++) {                      \
 618                                 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
 619                                 cip[i].ciputctrl_count &= ~SQ_FASTPUT;      \
 620                         }                                               \
 621                 }                                                       \
 622         }
 623 
 624 /*
 625  * Run service procedures for all queues in the stream head.
 626  */
 627 #define STR_SERVICE(stp, q) {                                           \
 628         ASSERT(MUTEX_HELD(&stp->sd_qlock));                              \
 629         while (stp->sd_qhead != NULL) {                                      \
 630                 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link);              \
 631                 ASSERT(stp->sd_nqueues > 0);                              \
 632                 stp->sd_nqueues--;                                   \
 633                 ASSERT(!(q->q_flag & QINSERVICE));                       \
 634                 mutex_exit(&stp->sd_qlock);                              \
 635                 queue_service(q);                                       \
 636                 mutex_enter(&stp->sd_qlock);                             \
 637         }                                                               \
 638         ASSERT(stp->sd_nqueues == 0);                                        \
 639         ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL));       \
 640 }
 641 
 642 /*
 643  * Constructor/destructor routines for the stream head cache
 644  */
 645 /* ARGSUSED */
 646 static int
 647 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
 648 {
 649         stdata_t *stp = buf;
 650 
 651         mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
 652         mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
 653         mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
 654         cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
 655         cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
 656         cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
 657         cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
 658         cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
 659         stp->sd_wrq = NULL;
 660 
 661         return (0);
 662 }
 663 
 664 /* ARGSUSED */
 665 static void
 666 stream_head_destructor(void *buf, void *cdrarg)
 667 {
 668         stdata_t *stp = buf;
 669 
 670         mutex_destroy(&stp->sd_lock);
 671         mutex_destroy(&stp->sd_reflock);
 672         mutex_destroy(&stp->sd_qlock);
 673         cv_destroy(&stp->sd_monitor);
 674         cv_destroy(&stp->sd_iocmonitor);
 675         cv_destroy(&stp->sd_refmonitor);
 676         cv_destroy(&stp->sd_qcv);
 677         cv_destroy(&stp->sd_zcopy_wait);
 678 }
 679 
 680 /*
 681  * Constructor/destructor routines for the queue cache
 682  */
 683 /* ARGSUSED */
 684 static int
 685 queue_constructor(void *buf, void *cdrarg, int kmflags)
 686 {
 687         queinfo_t *qip = buf;
 688         queue_t *qp = &qip->qu_rqueue;
 689         queue_t *wqp = &qip->qu_wqueue;
 690         syncq_t *sq = &qip->qu_syncq;
 691 
 692         qp->q_first = NULL;
 693         qp->q_link = NULL;
 694         qp->q_count = 0;
 695         qp->q_mblkcnt = 0;
 696         qp->q_sqhead = NULL;
 697         qp->q_sqtail = NULL;
 698         qp->q_sqnext = NULL;
 699         qp->q_sqprev = NULL;
 700         qp->q_sqflags = 0;
 701         qp->q_rwcnt = 0;
 702         qp->q_spri = 0;
 703 
 704         mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
 705         cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
 706 
 707         wqp->q_first = NULL;
 708         wqp->q_link = NULL;
 709         wqp->q_count = 0;
 710         wqp->q_mblkcnt = 0;
 711         wqp->q_sqhead = NULL;
 712         wqp->q_sqtail = NULL;
 713         wqp->q_sqnext = NULL;
 714         wqp->q_sqprev = NULL;
 715         wqp->q_sqflags = 0;
 716         wqp->q_rwcnt = 0;
 717         wqp->q_spri = 0;
 718 
 719         mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
 720         cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
 721 
 722         sq->sq_head = NULL;
 723         sq->sq_tail = NULL;
 724         sq->sq_evhead = NULL;
 725         sq->sq_evtail = NULL;
 726         sq->sq_callbpend = NULL;
 727         sq->sq_outer = NULL;
 728         sq->sq_onext = NULL;
 729         sq->sq_oprev = NULL;
 730         sq->sq_next = NULL;
 731         sq->sq_svcflags = 0;
 732         sq->sq_servcount = 0;
 733         sq->sq_needexcl = 0;
 734         sq->sq_nqueues = 0;
 735         sq->sq_pri = 0;
 736 
 737         mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
 738         cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
 739         cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
 740 
 741         return (0);
 742 }
 743 
 744 /* ARGSUSED */
 745 static void
 746 queue_destructor(void *buf, void *cdrarg)
 747 {
 748         queinfo_t *qip = buf;
 749         queue_t *qp = &qip->qu_rqueue;
 750         queue_t *wqp = &qip->qu_wqueue;
 751         syncq_t *sq = &qip->qu_syncq;
 752 
 753         ASSERT(qp->q_sqhead == NULL);
 754         ASSERT(wqp->q_sqhead == NULL);
 755         ASSERT(qp->q_sqnext == NULL);
 756         ASSERT(wqp->q_sqnext == NULL);
 757         ASSERT(qp->q_rwcnt == 0);
 758         ASSERT(wqp->q_rwcnt == 0);
 759 
 760         mutex_destroy(&qp->q_lock);
 761         cv_destroy(&qp->q_wait);
 762 
 763         mutex_destroy(&wqp->q_lock);
 764         cv_destroy(&wqp->q_wait);
 765 
 766         mutex_destroy(&sq->sq_lock);
 767         cv_destroy(&sq->sq_wait);
 768         cv_destroy(&sq->sq_exitwait);
 769 }
 770 
 771 /*
 772  * Constructor/destructor routines for the syncq cache
 773  */
 774 /* ARGSUSED */
 775 static int
 776 syncq_constructor(void *buf, void *cdrarg, int kmflags)
 777 {
 778         syncq_t *sq = buf;
 779 
 780         bzero(buf, sizeof (syncq_t));
 781 
 782         mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
 783         cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
 784         cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
 785 
 786         return (0);
 787 }
 788 
 789 /* ARGSUSED */
 790 static void
 791 syncq_destructor(void *buf, void *cdrarg)
 792 {
 793         syncq_t *sq = buf;
 794 
 795         ASSERT(sq->sq_head == NULL);
 796         ASSERT(sq->sq_tail == NULL);
 797         ASSERT(sq->sq_evhead == NULL);
 798         ASSERT(sq->sq_evtail == NULL);
 799         ASSERT(sq->sq_callbpend == NULL);
 800         ASSERT(sq->sq_callbflags == 0);
 801         ASSERT(sq->sq_outer == NULL);
 802         ASSERT(sq->sq_onext == NULL);
 803         ASSERT(sq->sq_oprev == NULL);
 804         ASSERT(sq->sq_next == NULL);
 805         ASSERT(sq->sq_needexcl == 0);
 806         ASSERT(sq->sq_svcflags == 0);
 807         ASSERT(sq->sq_servcount == 0);
 808         ASSERT(sq->sq_nqueues == 0);
 809         ASSERT(sq->sq_pri == 0);
 810         ASSERT(sq->sq_count == 0);
 811         ASSERT(sq->sq_rmqcount == 0);
 812         ASSERT(sq->sq_cancelid == 0);
 813         ASSERT(sq->sq_ciputctrl == NULL);
 814         ASSERT(sq->sq_nciputctrl == 0);
 815         ASSERT(sq->sq_type == 0);
 816         ASSERT(sq->sq_flags == 0);
 817 
 818         mutex_destroy(&sq->sq_lock);
 819         cv_destroy(&sq->sq_wait);
 820         cv_destroy(&sq->sq_exitwait);
 821 }
 822 
 823 /* ARGSUSED */
 824 static int
 825 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
 826 {
 827         ciputctrl_t *cip = buf;
 828         int i;
 829 
 830         for (i = 0; i < n_ciputctrl; i++) {
 831                 cip[i].ciputctrl_count = SQ_FASTPUT;
 832                 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
 833         }
 834 
 835         return (0);
 836 }
 837 
 838 /* ARGSUSED */
 839 static void
 840 ciputctrl_destructor(void *buf, void *cdrarg)
 841 {
 842         ciputctrl_t *cip = buf;
 843         int i;
 844 
 845         for (i = 0; i < n_ciputctrl; i++) {
 846                 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
 847                 mutex_destroy(&cip[i].ciputctrl_lock);
 848         }
 849 }
 850 
 851 /*
 852  * Init routine run from main at boot time.
 853  */
 854 void
 855 strinit(void)
 856 {
 857         int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
 858 
 859         stream_head_cache = kmem_cache_create("stream_head_cache",
 860             sizeof (stdata_t), 0,
 861             stream_head_constructor, stream_head_destructor, NULL,
 862             NULL, NULL, 0);
 863 
 864         queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
 865             queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
 866 
 867         syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
 868             syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
 869 
 870         qband_cache = kmem_cache_create("qband_cache",
 871             sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 872 
 873         linkinfo_cache = kmem_cache_create("linkinfo_cache",
 874             sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 875 
 876         n_ciputctrl = ncpus;
 877         n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
 878         ASSERT(n_ciputctrl >= 1);
 879         n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
 880         if (n_ciputctrl >= min_n_ciputctrl) {
 881                 ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
 882                     sizeof (ciputctrl_t) * n_ciputctrl,
 883                     sizeof (ciputctrl_t), ciputctrl_constructor,
 884                     ciputctrl_destructor, NULL, NULL, NULL, 0);
 885         }
 886 
 887         streams_taskq = system_taskq;
 888 
 889         if (streams_taskq == NULL)
 890                 panic("strinit: no memory for streams taskq!");
 891 
 892         bc_bkgrnd_thread = thread_create(NULL, 0,
 893             streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
 894 
 895         streams_qbkgrnd_thread = thread_create(NULL, 0,
 896             streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
 897 
 898         streams_sqbkgrnd_thread = thread_create(NULL, 0,
 899             streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
 900 
 901         /*
 902          * Create STREAMS kstats.
 903          */
 904         str_kstat = kstat_create("streams", 0, "strstat",
 905             "net", KSTAT_TYPE_NAMED,
 906             sizeof (str_statistics) / sizeof (kstat_named_t),
 907             KSTAT_FLAG_VIRTUAL);
 908 
 909         if (str_kstat != NULL) {
 910                 str_kstat->ks_data = &str_statistics;
 911                 kstat_install(str_kstat);
 912         }
 913 
 914         /*
 915          * TPI support routine initialisation.
 916          */
 917         tpi_init();
 918 
 919         /*
 920          * Handle to have autopush and persistent link information per
 921          * zone.
 922          * Note: uses shutdown hook instead of destroy hook so that the
 923          * persistent links can be torn down before the destroy hooks
 924          * in the TCP/IP stack are called.
 925          */
 926         netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
 927             str_stack_fini);
 928 }
 929 
 930 void
 931 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
 932 {
 933         struct stdata *stp;
 934 
 935         ASSERT(vp->v_stream);
 936         stp = vp->v_stream;
 937         /* Have to hold sd_lock to prevent siglist from changing */
 938         mutex_enter(&stp->sd_lock);
 939         if (stp->sd_sigflags & event)
 940                 strsendsig(stp->sd_siglist, event, band, error);
 941         mutex_exit(&stp->sd_lock);
 942 }
 943 
 944 /*
 945  * Send the "sevent" set of signals to a process.
 946  * This might send more than one signal if the process is registered
 947  * for multiple events. The caller should pass in an sevent that only
 948  * includes the events for which the process has registered.
 949  */
 950 static void
 951 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
 952         uchar_t band, int error)
 953 {
 954         ASSERT(MUTEX_HELD(&proc->p_lock));
 955 
 956         info->si_band = 0;
 957         info->si_errno = 0;
 958 
 959         if (sevent & S_ERROR) {
 960                 sevent &= ~S_ERROR;
 961                 info->si_code = POLL_ERR;
 962                 info->si_errno = error;
 963                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 964                     "strsendsig:proc %p info %p", proc, info);
 965                 sigaddq(proc, NULL, info, KM_NOSLEEP);
 966                 info->si_errno = 0;
 967         }
 968         if (sevent & S_HANGUP) {
 969                 sevent &= ~S_HANGUP;
 970                 info->si_code = POLL_HUP;
 971                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 972                     "strsendsig:proc %p info %p", proc, info);
 973                 sigaddq(proc, NULL, info, KM_NOSLEEP);
 974         }
 975         if (sevent & S_HIPRI) {
 976                 sevent &= ~S_HIPRI;
 977                 info->si_code = POLL_PRI;
 978                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 979                     "strsendsig:proc %p info %p", proc, info);
 980                 sigaddq(proc, NULL, info, KM_NOSLEEP);
 981         }
 982         if (sevent & S_RDBAND) {
 983                 sevent &= ~S_RDBAND;
 984                 if (events & S_BANDURG)
 985                         sigtoproc(proc, NULL, SIGURG);
 986                 else
 987                         sigtoproc(proc, NULL, SIGPOLL);
 988         }
 989         if (sevent & S_WRBAND) {
 990                 sevent &= ~S_WRBAND;
 991                 sigtoproc(proc, NULL, SIGPOLL);
 992         }
 993         if (sevent & S_INPUT) {
 994                 sevent &= ~S_INPUT;
 995                 info->si_code = POLL_IN;
 996                 info->si_band = band;
 997                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 998                     "strsendsig:proc %p info %p", proc, info);
 999                 sigaddq(proc, NULL, info, KM_NOSLEEP);
1000                 info->si_band = 0;
1001         }
1002         if (sevent & S_OUTPUT) {
1003                 sevent &= ~S_OUTPUT;
1004                 info->si_code = POLL_OUT;
1005                 info->si_band = band;
1006                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1007                     "strsendsig:proc %p info %p", proc, info);
1008                 sigaddq(proc, NULL, info, KM_NOSLEEP);
1009                 info->si_band = 0;
1010         }
1011         if (sevent & S_MSG) {
1012                 sevent &= ~S_MSG;
1013                 info->si_code = POLL_MSG;
1014                 info->si_band = band;
1015                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1016                     "strsendsig:proc %p info %p", proc, info);
1017                 sigaddq(proc, NULL, info, KM_NOSLEEP);
1018                 info->si_band = 0;
1019         }
1020         if (sevent & S_RDNORM) {
1021                 sevent &= ~S_RDNORM;
1022                 sigtoproc(proc, NULL, SIGPOLL);
1023         }
1024         if (sevent != 0) {
1025                 panic("strsendsig: unknown event(s) %x", sevent);
1026         }
1027 }
1028 
1029 /*
1030  * Send SIGPOLL/SIGURG signal to all processes and process groups
1031  * registered on the given signal list that want a signal for at
1032  * least one of the specified events.
1033  *
1034  * Must be called with exclusive access to siglist (caller holding sd_lock).
1035  *
1036  * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1037  * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1038  * while it is in the siglist.
1039  *
1040  * For performance reasons (MP scalability) the code drops pidlock
1041  * when sending signals to a single process.
1042  * When sending to a process group the code holds
1043  * pidlock to prevent the membership in the process group from changing
1044  * while walking the p_pglink list.
1045  */
1046 void
1047 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1048 {
1049         strsig_t *ssp;
1050         k_siginfo_t info;
1051         struct pid *pidp;
1052         proc_t  *proc;
1053 
1054         info.si_signo = SIGPOLL;
1055         info.si_errno = 0;
1056         for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1057                 int sevent;
1058 
1059                 sevent = ssp->ss_events & event;
1060                 if (sevent == 0)
1061                         continue;
1062 
1063                 if ((pidp = ssp->ss_pidp) == NULL) {
1064                         /* pid was released but still on event list */
1065                         continue;
1066                 }
1067 
1068 
1069                 if (ssp->ss_pid > 0) {
1070                         /*
1071                          * XXX This unfortunately still generates
1072                          * a signal when a fd is closed but
1073                          * the proc is active.
1074                          */
1075                         ASSERT(ssp->ss_pid == pidp->pid_id);
1076 
1077                         mutex_enter(&pidlock);
1078                         proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1079                         if (proc == NULL) {
1080                                 mutex_exit(&pidlock);
1081                                 continue;
1082                         }
1083                         mutex_enter(&proc->p_lock);
1084                         mutex_exit(&pidlock);
1085                         dosendsig(proc, ssp->ss_events, sevent, &info,
1086                             band, error);
1087                         mutex_exit(&proc->p_lock);
1088                 } else {
1089                         /*
1090                          * Send to process group. Hold pidlock across
1091                          * calls to dosendsig().
1092                          */
1093                         pid_t pgrp = -ssp->ss_pid;
1094 
1095                         mutex_enter(&pidlock);
1096                         proc = pgfind_zone(pgrp, ALL_ZONES);
1097                         while (proc != NULL) {
1098                                 mutex_enter(&proc->p_lock);
1099                                 dosendsig(proc, ssp->ss_events, sevent,
1100                                     &info, band, error);
1101                                 mutex_exit(&proc->p_lock);
1102                                 proc = proc->p_pglink;
1103                         }
1104                         mutex_exit(&pidlock);
1105                 }
1106         }
1107 }
1108 
1109 /*
1110  * Attach a stream device or module.
1111  * qp is a read queue; the new queue goes in so its next
1112  * read ptr is the argument, and the write queue corresponding
1113  * to the argument points to this queue. Return 0 on success,
1114  * or a non-zero errno on failure.
1115  */
1116 int
1117 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1118     boolean_t is_insert)
1119 {
1120         major_t                 major;
1121         cdevsw_impl_t           *dp;
1122         struct streamtab        *str;
1123         queue_t                 *rq;
1124         queue_t                 *wrq;
1125         uint32_t                qflag;
1126         uint32_t                sqtype;
1127         perdm_t                 *dmp;
1128         int                     error;
1129         int                     sflag;
1130 
1131         rq = allocq();
1132         wrq = _WR(rq);
1133         STREAM(rq) = STREAM(wrq) = STREAM(qp);
1134 
1135         if (fp != NULL) {
1136                 str = fp->f_str;
1137                 qflag = fp->f_qflag;
1138                 sqtype = fp->f_sqtype;
1139                 dmp = fp->f_dmp;
1140                 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1141                 sflag = MODOPEN;
1142 
1143                 /*
1144                  * stash away a pointer to the module structure so we can
1145                  * unref it in qdetach.
1146                  */
1147                 rq->q_fp = fp;
1148         } else {
1149                 ASSERT(!is_insert);
1150 
1151                 major = getmajor(*devp);
1152                 dp = &devimpl[major];
1153 
1154                 str = dp->d_str;
1155                 ASSERT(str == STREAMSTAB(major));
1156 
1157                 qflag = dp->d_qflag;
1158                 ASSERT(qflag & QISDRV);
1159                 sqtype = dp->d_sqtype;
1160 
1161                 /* create perdm_t if needed */
1162                 if (NEED_DM(dp->d_dmp, qflag))
1163                         dp->d_dmp = hold_dm(str, qflag, sqtype);
1164 
1165                 dmp = dp->d_dmp;
1166                 sflag = 0;
1167         }
1168 
1169         TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1170             "qattach:qflag == %X(%X)", qflag, *devp);
1171 
1172         /* setq might sleep in allocator - avoid holding locks. */
1173         setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1174 
1175         /*
1176          * Before calling the module's open routine, set up the q_next
1177          * pointer for inserting a module in the middle of a stream.
1178          *
1179          * Note that we can always set _QINSERTING and set up q_next
1180          * pointer for both inserting and pushing a module.  Then there
1181          * is no need for the is_insert parameter.  In insertq(), called
1182          * by qprocson(), assume that q_next of the new module always points
1183          * to the correct queue and use it for insertion.  Everything should
1184          * work out fine.  But in the first release of _I_INSERT, we
1185          * distinguish between inserting and pushing to make sure that
1186          * pushing a module follows the same code path as before.
1187          */
1188         if (is_insert) {
1189                 rq->q_flag |= _QINSERTING;
1190                 rq->q_next = qp;
1191         }
1192 
1193         /*
1194          * If there is an outer perimeter get exclusive access during
1195          * the open procedure.  Bump up the reference count on the queue.
1196          */
1197         entersq(rq->q_syncq, SQ_OPENCLOSE);
1198         error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1199         if (error != 0)
1200                 goto failed;
1201         leavesq(rq->q_syncq, SQ_OPENCLOSE);
1202         ASSERT(qprocsareon(rq));
1203         return (0);
1204 
1205 failed:
1206         rq->q_flag &= ~_QINSERTING;
1207         if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1208                 qprocsoff(rq);
1209         leavesq(rq->q_syncq, SQ_OPENCLOSE);
1210         rq->q_next = wrq->q_next = NULL;
1211         qdetach(rq, 0, 0, crp, B_FALSE);
1212         return (error);
1213 }
1214 
1215 /*
1216  * Handle second open of stream. For modules, set the
1217  * last argument to MODOPEN and do not pass any open flags.
1218  * Ignore dummydev since this is not the first open.
1219  */
1220 int
1221 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1222 {
1223         int     error;
1224         dev_t dummydev;
1225         queue_t *wqp = _WR(qp);
1226 
1227         ASSERT(qp->q_flag & QREADR);
1228         entersq(qp->q_syncq, SQ_OPENCLOSE);
1229 
1230         dummydev = *devp;
1231         if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1232             (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1233                 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1234                 mutex_enter(&STREAM(qp)->sd_lock);
1235                 qp->q_stream->sd_flag |= STREOPENFAIL;
1236                 mutex_exit(&STREAM(qp)->sd_lock);
1237                 return (error);
1238         }
1239         leavesq(qp->q_syncq, SQ_OPENCLOSE);
1240 
1241         /*
1242          * successful open should have done qprocson()
1243          */
1244         ASSERT(qprocsareon(_RD(qp)));
1245         return (0);
1246 }
1247 
1248 /*
1249  * Detach a stream module or device.
1250  * If clmode == 1 then the module or driver was opened and its
1251  * close routine must be called. If clmode == 0, the module
1252  * or driver was never opened or the open failed, and so its close
1253  * should not be called.
1254  */
1255 void
1256 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1257 {
1258         queue_t *wqp = _WR(qp);
1259         ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1260 
1261         if (STREAM_NEEDSERVICE(STREAM(qp)))
1262                 stream_runservice(STREAM(qp));
1263 
1264         if (clmode) {
1265                 /*
1266                  * Make sure that all the messages on the write side syncq are
1267                  * processed and nothing is left. Since we are closing, no new
1268                  * messages may appear there.
1269                  */
1270                 wait_q_syncq(wqp);
1271 
1272                 entersq(qp->q_syncq, SQ_OPENCLOSE);
1273                 if (is_remove) {
1274                         mutex_enter(QLOCK(qp));
1275                         qp->q_flag |= _QREMOVING;
1276                         mutex_exit(QLOCK(qp));
1277                 }
1278                 (*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1279                 /*
1280                  * Check that qprocsoff() was actually called.
1281                  */
1282                 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1283 
1284                 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1285         } else {
1286                 disable_svc(qp);
1287         }
1288 
1289         /*
1290          * Allow any threads blocked in entersq to proceed and discover
1291          * the QWCLOSE is set.
1292          * Note: This assumes that all users of entersq check QWCLOSE.
1293          * Currently runservice is the only entersq that can happen
1294          * after removeq has finished.
1295          * Removeq will have discarded all messages destined to the closing
1296          * pair of queues from the syncq.
1297          * NOTE: Calling a function inside an assert is unconventional.
1298          * However, it does not cause any problem since flush_syncq() does
1299          * not change any state except when it returns non-zero i.e.
1300          * when the assert will trigger.
1301          */
1302         ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1303         ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1304         ASSERT((qp->q_flag & QPERMOD) ||
1305             ((qp->q_syncq->sq_head == NULL) &&
1306             (wqp->q_syncq->sq_head == NULL)));
1307 
1308         /* release any fmodsw_impl_t structure held on behalf of the queue */
1309         ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1310         if (qp->q_fp != NULL)
1311                 fmodsw_rele(qp->q_fp);
1312 
1313         /* freeq removes us from the outer perimeter if any */
1314         freeq(qp);
1315 }
1316 
1317 /* Prevent service procedures from being called */
1318 void
1319 disable_svc(queue_t *qp)
1320 {
1321         queue_t *wqp = _WR(qp);
1322 
1323         ASSERT(qp->q_flag & QREADR);
1324         mutex_enter(QLOCK(qp));
1325         qp->q_flag |= QWCLOSE;
1326         mutex_exit(QLOCK(qp));
1327         mutex_enter(QLOCK(wqp));
1328         wqp->q_flag |= QWCLOSE;
1329         mutex_exit(QLOCK(wqp));
1330 }
1331 
1332 /* Allow service procedures to be called again */
1333 void
1334 enable_svc(queue_t *qp)
1335 {
1336         queue_t *wqp = _WR(qp);
1337 
1338         ASSERT(qp->q_flag & QREADR);
1339         mutex_enter(QLOCK(qp));
1340         qp->q_flag &= ~QWCLOSE;
1341         mutex_exit(QLOCK(qp));
1342         mutex_enter(QLOCK(wqp));
1343         wqp->q_flag &= ~QWCLOSE;
1344         mutex_exit(QLOCK(wqp));
1345 }
1346 
1347 /*
1348  * Remove queue from qhead/qtail if it is enabled.
1349  * Only reset QENAB if the queue was removed from the runlist.
1350  * A queue goes through 3 stages:
1351  *      It is on the service list and QENAB is set.
1352  *      It is removed from the service list but QENAB is still set.
1353  *      QENAB gets changed to QINSERVICE.
1354  *      QINSERVICE is reset (when the service procedure is done)
1355  * Thus we can not reset QENAB unless we actually removed it from the service
1356  * queue.
1357  */
1358 void
1359 remove_runlist(queue_t *qp)
1360 {
1361         if (qp->q_flag & QENAB && qhead != NULL) {
1362                 queue_t *q_chase;
1363                 queue_t *q_curr;
1364                 int removed;
1365 
1366                 mutex_enter(&service_queue);
1367                 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1368                 mutex_exit(&service_queue);
1369                 if (removed) {
1370                         STRSTAT(qremoved);
1371                         qp->q_flag &= ~QENAB;
1372                 }
1373         }
1374 }
1375 
1376 
1377 /*
1378  * Wait for any pending service processing to complete.
1379  * The removal of queues from the runlist is not atomic with the
1380  * clearing of the QENABLED flag and setting the INSERVICE flag.
1381  * consequently it is possible for remove_runlist in strclose
1382  * to not find the queue on the runlist but for it to be QENABLED
1383  * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1384  * as well as INSERVICE.
1385  */
1386 void
1387 wait_svc(queue_t *qp)
1388 {
1389         queue_t *wqp = _WR(qp);
1390 
1391         ASSERT(qp->q_flag & QREADR);
1392 
1393         /*
1394          * Try to remove queues from qhead/qtail list.
1395          */
1396         if (qhead != NULL) {
1397                 remove_runlist(qp);
1398                 remove_runlist(wqp);
1399         }
1400         /*
1401          * Wait till the syncqs associated with the queue disappear from the
1402          * background processing list.
1403          * This only needs to be done for non-PERMOD perimeters since
1404          * for PERMOD perimeters the syncq may be shared and will only be freed
1405          * when the last module/driver is unloaded.
1406          * If for PERMOD perimeters queue was on the syncq list, removeq()
1407          * should call propagate_syncq() or drain_syncq() for it. Both of these
1408          * functions remove the queue from its syncq list, so sqthread will not
1409          * try to access the queue.
1410          */
1411         if (!(qp->q_flag & QPERMOD)) {
1412                 syncq_t *rsq = qp->q_syncq;
1413                 syncq_t *wsq = wqp->q_syncq;
1414 
1415                 /*
1416                  * Disable rsq and wsq and wait for any background processing of
1417                  * syncq to complete.
1418                  */
1419                 wait_sq_svc(rsq);
1420                 if (wsq != rsq)
1421                         wait_sq_svc(wsq);
1422         }
1423 
1424         mutex_enter(QLOCK(qp));
1425         while (qp->q_flag & (QINSERVICE|QENAB))
1426                 cv_wait(&qp->q_wait, QLOCK(qp));
1427         mutex_exit(QLOCK(qp));
1428         mutex_enter(QLOCK(wqp));
1429         while (wqp->q_flag & (QINSERVICE|QENAB))
1430                 cv_wait(&wqp->q_wait, QLOCK(wqp));
1431         mutex_exit(QLOCK(wqp));
1432 }
1433 
1434 /*
1435  * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1436  * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1437  * also be set, and is passed through to allocb_cred_wait().
1438  *
1439  * Returns errno on failure, zero on success.
1440  */
1441 int
1442 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1443 {
1444         mblk_t *tmp;
1445         ssize_t  count;
1446         int error = 0;
1447 
1448         ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1449             (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1450 
1451         if (bp->b_datap->db_type == M_IOCTL) {
1452                 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1453         } else {
1454                 ASSERT(bp->b_datap->db_type == M_COPYIN);
1455                 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1456         }
1457         /*
1458          * strdoioctl validates ioc_count, so if this assert fails it
1459          * cannot be due to user error.
1460          */
1461         ASSERT(count >= 0);
1462 
1463         if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1464             curproc->p_pid)) == NULL) {
1465                 return (error);
1466         }
1467         error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1468         if (error != 0) {
1469                 freeb(tmp);
1470                 return (error);
1471         }
1472         DB_CPID(tmp) = curproc->p_pid;
1473         tmp->b_wptr += count;
1474         bp->b_cont = tmp;
1475 
1476         return (0);
1477 }
1478 
1479 /*
1480  * Copy ioctl data to user-land. Return non-zero errno on failure,
1481  * 0 for success.
1482  */
1483 int
1484 getiocd(mblk_t *bp, char *arg, int copymode)
1485 {
1486         ssize_t count;
1487         size_t  n;
1488         int     error;
1489 
1490         if (bp->b_datap->db_type == M_IOCACK)
1491                 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1492         else {
1493                 ASSERT(bp->b_datap->db_type == M_COPYOUT);
1494                 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1495         }
1496         ASSERT(count >= 0);
1497 
1498         for (bp = bp->b_cont; bp && count;
1499             count -= n, bp = bp->b_cont, arg += n) {
1500                 n = MIN(count, bp->b_wptr - bp->b_rptr);
1501                 error = strcopyout(bp->b_rptr, arg, n, copymode);
1502                 if (error)
1503                         return (error);
1504         }
1505         ASSERT(count == 0);
1506         return (0);
1507 }
1508 
1509 /*
1510  * Allocate a linkinfo entry given the write queue of the
1511  * bottom module of the top stream and the write queue of the
1512  * stream head of the bottom stream.
1513  */
1514 linkinfo_t *
1515 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1516 {
1517         linkinfo_t *linkp;
1518 
1519         linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1520 
1521         linkp->li_lblk.l_qtop = qup;
1522         linkp->li_lblk.l_qbot = qdown;
1523         linkp->li_fpdown = fpdown;
1524 
1525         mutex_enter(&strresources);
1526         linkp->li_next = linkinfo_list;
1527         linkp->li_prev = NULL;
1528         if (linkp->li_next)
1529                 linkp->li_next->li_prev = linkp;
1530         linkinfo_list = linkp;
1531         linkp->li_lblk.l_index = ++lnk_id;
1532         ASSERT(lnk_id != 0);    /* this should never wrap in practice */
1533         mutex_exit(&strresources);
1534 
1535         return (linkp);
1536 }
1537 
1538 /*
1539  * Free a linkinfo entry.
1540  */
1541 void
1542 lbfree(linkinfo_t *linkp)
1543 {
1544         mutex_enter(&strresources);
1545         if (linkp->li_next)
1546                 linkp->li_next->li_prev = linkp->li_prev;
1547         if (linkp->li_prev)
1548                 linkp->li_prev->li_next = linkp->li_next;
1549         else
1550                 linkinfo_list = linkp->li_next;
1551         mutex_exit(&strresources);
1552 
1553         kmem_cache_free(linkinfo_cache, linkp);
1554 }
1555 
1556 /*
1557  * Check for a potential linking cycle.
1558  * Return 1 if a link will result in a cycle,
1559  * and 0 otherwise.
1560  */
1561 int
1562 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1563 {
1564         struct mux_node *np;
1565         struct mux_edge *ep;
1566         int i;
1567         major_t lomaj;
1568         major_t upmaj;
1569         /*
1570          * if the lower stream is a pipe/FIFO, return, since link
1571          * cycles can not happen on pipes/FIFOs
1572          */
1573         if (lostp->sd_vnode->v_type == VFIFO)
1574                 return (0);
1575 
1576         for (i = 0; i < ss->ss_devcnt; i++) {
1577                 np = &ss->ss_mux_nodes[i];
1578                 MUX_CLEAR(np);
1579         }
1580         lomaj = getmajor(lostp->sd_vnode->v_rdev);
1581         upmaj = getmajor(upstp->sd_vnode->v_rdev);
1582         np = &ss->ss_mux_nodes[lomaj];
1583         for (;;) {
1584                 if (!MUX_DIDVISIT(np)) {
1585                         if (np->mn_imaj == upmaj)
1586                                 return (1);
1587                         if (np->mn_outp == NULL) {
1588                                 MUX_VISIT(np);
1589                                 if (np->mn_originp == NULL)
1590                                         return (0);
1591                                 np = np->mn_originp;
1592                                 continue;
1593                         }
1594                         MUX_VISIT(np);
1595                         np->mn_startp = np->mn_outp;
1596                 } else {
1597                         if (np->mn_startp == NULL) {
1598                                 if (np->mn_originp == NULL)
1599                                         return (0);
1600                                 else {
1601                                         np = np->mn_originp;
1602                                         continue;
1603                                 }
1604                         }
1605                         /*
1606                          * If ep->me_nodep is a FIFO (me_nodep == NULL),
1607                          * ignore the edge and move on. ep->me_nodep gets
1608                          * set to NULL in mux_addedge() if it is a FIFO.
1609                          *
1610                          */
1611                         ep = np->mn_startp;
1612                         np->mn_startp = ep->me_nextp;
1613                         if (ep->me_nodep == NULL)
1614                                 continue;
1615                         ep->me_nodep->mn_originp = np;
1616                         np = ep->me_nodep;
1617                 }
1618         }
1619 }
1620 
1621 /*
1622  * Find linkinfo entry corresponding to the parameters.
1623  */
1624 linkinfo_t *
1625 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1626 {
1627         linkinfo_t *linkp;
1628         struct mux_edge *mep;
1629         struct mux_node *mnp;
1630         queue_t *qup;
1631 
1632         mutex_enter(&strresources);
1633         if ((type & LINKTYPEMASK) == LINKNORMAL) {
1634                 qup = getendq(stp->sd_wrq);
1635                 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1636                         if ((qup == linkp->li_lblk.l_qtop) &&
1637                             (!index || (index == linkp->li_lblk.l_index))) {
1638                                 mutex_exit(&strresources);
1639                                 return (linkp);
1640                         }
1641                 }
1642         } else {
1643                 ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1644                 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1645                 mep = mnp->mn_outp;
1646                 while (mep) {
1647                         if ((index == 0) || (index == mep->me_muxid))
1648                                 break;
1649                         mep = mep->me_nextp;
1650                 }
1651                 if (!mep) {
1652                         mutex_exit(&strresources);
1653                         return (NULL);
1654                 }
1655                 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1656                         if ((!linkp->li_lblk.l_qtop) &&
1657                             (mep->me_muxid == linkp->li_lblk.l_index)) {
1658                                 mutex_exit(&strresources);
1659                                 return (linkp);
1660                         }
1661                 }
1662         }
1663         mutex_exit(&strresources);
1664         return (NULL);
1665 }
1666 
1667 /*
1668  * Given a queue ptr, follow the chain of q_next pointers until you reach the
1669  * last queue on the chain and return it.
1670  */
1671 queue_t *
1672 getendq(queue_t *q)
1673 {
1674         ASSERT(q != NULL);
1675         while (_SAMESTR(q))
1676                 q = q->q_next;
1677         return (q);
1678 }
1679 
1680 /*
1681  * Wait for the syncq count to drop to zero.
1682  * sq could be either outer or inner.
1683  */
1684 
1685 static void
1686 wait_syncq(syncq_t *sq)
1687 {
1688         uint16_t count;
1689 
1690         mutex_enter(SQLOCK(sq));
1691         count = sq->sq_count;
1692         SQ_PUTLOCKS_ENTER(sq);
1693         SUM_SQ_PUTCOUNTS(sq, count);
1694         while (count != 0) {
1695                 sq->sq_flags |= SQ_WANTWAKEUP;
1696                 SQ_PUTLOCKS_EXIT(sq);
1697                 cv_wait(&sq->sq_wait, SQLOCK(sq));
1698                 count = sq->sq_count;
1699                 SQ_PUTLOCKS_ENTER(sq);
1700                 SUM_SQ_PUTCOUNTS(sq, count);
1701         }
1702         SQ_PUTLOCKS_EXIT(sq);
1703         mutex_exit(SQLOCK(sq));
1704 }
1705 
1706 /*
1707  * Wait while there are any messages for the queue in its syncq.
1708  */
1709 static void
1710 wait_q_syncq(queue_t *q)
1711 {
1712         if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1713                 syncq_t *sq = q->q_syncq;
1714 
1715                 mutex_enter(SQLOCK(sq));
1716                 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1717                         sq->sq_flags |= SQ_WANTWAKEUP;
1718                         cv_wait(&sq->sq_wait, SQLOCK(sq));
1719                 }
1720                 mutex_exit(SQLOCK(sq));
1721         }
1722 }
1723 
1724 
1725 int
1726 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1727     int lhlink)
1728 {
1729         struct stdata *stp;
1730         struct strioctl strioc;
1731         struct linkinfo *linkp;
1732         struct stdata *stpdown;
1733         struct streamtab *str;
1734         queue_t *passq;
1735         syncq_t *passyncq;
1736         queue_t *rq;
1737         cdevsw_impl_t *dp;
1738         uint32_t qflag;
1739         uint32_t sqtype;
1740         perdm_t *dmp;
1741         int error = 0;
1742         netstack_t *ns;
1743         str_stack_t *ss;
1744 
1745         stp = vp->v_stream;
1746         TRACE_1(TR_FAC_STREAMS_FR,
1747             TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1748         /*
1749          * Test for invalid upper stream
1750          */
1751         if (stp->sd_flag & STRHUP) {
1752                 return (ENXIO);
1753         }
1754         if (vp->v_type == VFIFO) {
1755                 return (EINVAL);
1756         }
1757         if (stp->sd_strtab == NULL) {
1758                 return (EINVAL);
1759         }
1760         if (!stp->sd_strtab->st_muxwinit) {
1761                 return (EINVAL);
1762         }
1763         if (fpdown == NULL) {
1764                 return (EBADF);
1765         }
1766         ns = netstack_find_by_cred(crp);
1767         ASSERT(ns != NULL);
1768         ss = ns->netstack_str;
1769         ASSERT(ss != NULL);
1770 
1771         if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1772                 netstack_rele(ss->ss_netstack);
1773                 return (EINVAL);
1774         }
1775         mutex_enter(&muxifier);
1776         if (stp->sd_flag & STPLEX) {
1777                 mutex_exit(&muxifier);
1778                 netstack_rele(ss->ss_netstack);
1779                 return (ENXIO);
1780         }
1781 
1782         /*
1783          * Test for invalid lower stream.
1784          * The check for the v_type != VFIFO and having a major
1785          * number not >= devcnt is done to avoid problems with
1786          * adding mux_node entry past the end of mux_nodes[].
1787          * For FIFO's we don't add an entry so this isn't a
1788          * problem.
1789          */
1790         if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1791             (stpdown == stp) || (stpdown->sd_flag &
1792             (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1793             ((stpdown->sd_vnode->v_type != VFIFO) &&
1794             (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1795             linkcycle(stp, stpdown, ss)) {
1796                 mutex_exit(&muxifier);
1797                 netstack_rele(ss->ss_netstack);
1798                 return (EINVAL);
1799         }
1800         TRACE_1(TR_FAC_STREAMS_FR,
1801             TR_STPDOWN, "stpdown:%p", stpdown);
1802         rq = getendq(stp->sd_wrq);
1803         if (cmd == I_PLINK)
1804                 rq = NULL;
1805 
1806         linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1807 
1808         strioc.ic_cmd = cmd;
1809         strioc.ic_timout = INFTIM;
1810         strioc.ic_len = sizeof (struct linkblk);
1811         strioc.ic_dp = (char *)&linkp->li_lblk;
1812 
1813         /*
1814          * STRPLUMB protects plumbing changes and should be set before
1815          * link_addpassthru()/link_rempassthru() are called, so it is set here
1816          * and cleared in the end of mlink when passthru queue is removed.
1817          * Setting of STRPLUMB prevents reopens of the stream while passthru
1818          * queue is in-place (it is not a proper module and doesn't have open
1819          * entry point).
1820          *
1821          * STPLEX prevents any threads from entering the stream from above. It
1822          * can't be set before the call to link_addpassthru() because putnext
1823          * from below may cause stream head I/O routines to be called and these
1824          * routines assert that STPLEX is not set. After link_addpassthru()
1825          * nothing may come from below since the pass queue syncq is blocked.
1826          * Note also that STPLEX should be cleared before the call to
1827          * link_rempassthru() since when messages start flowing to the stream
1828          * head (e.g. because of message propagation from the pass queue) stream
1829          * head I/O routines may be called with STPLEX flag set.
1830          *
1831          * When STPLEX is set, nothing may come into the stream from above and
1832          * it is safe to do a setq which will change stream head. So, the
1833          * correct sequence of actions is:
1834          *
1835          * 1) Set STRPLUMB
1836          * 2) Call link_addpassthru()
1837          * 3) Set STPLEX
1838          * 4) Call setq and update the stream state
1839          * 5) Clear STPLEX
1840          * 6) Call link_rempassthru()
1841          * 7) Clear STRPLUMB
1842          *
1843          * The same sequence applies to munlink() code.
1844          */
1845         mutex_enter(&stpdown->sd_lock);
1846         stpdown->sd_flag |= STRPLUMB;
1847         mutex_exit(&stpdown->sd_lock);
1848         /*
1849          * Add passthru queue below lower mux. This will block
1850          * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1851          */
1852         passq = link_addpassthru(stpdown);
1853 
1854         mutex_enter(&stpdown->sd_lock);
1855         stpdown->sd_flag |= STPLEX;
1856         mutex_exit(&stpdown->sd_lock);
1857 
1858         rq = _RD(stpdown->sd_wrq);
1859         /*
1860          * There may be messages in the streamhead's syncq due to messages
1861          * that arrived before link_addpassthru() was done. To avoid
1862          * background processing of the syncq happening simultaneous with
1863          * setq processing, we disable the streamhead syncq and wait until
1864          * existing background thread finishes working on it.
1865          */
1866         wait_sq_svc(rq->q_syncq);
1867         passyncq = passq->q_syncq;
1868         if (!(passyncq->sq_flags & SQ_BLOCKED))
1869                 blocksq(passyncq, SQ_BLOCKED, 0);
1870 
1871         ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1872         ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1873         rq->q_ptr = _WR(rq)->q_ptr = NULL;
1874 
1875         /* setq might sleep in allocator - avoid holding locks. */
1876         /* Note: we are holding muxifier here. */
1877 
1878         str = stp->sd_strtab;
1879         dp = &devimpl[getmajor(vp->v_rdev)];
1880         ASSERT(dp->d_str == str);
1881 
1882         qflag = dp->d_qflag;
1883         sqtype = dp->d_sqtype;
1884 
1885         /* create perdm_t if needed */
1886         if (NEED_DM(dp->d_dmp, qflag))
1887                 dp->d_dmp = hold_dm(str, qflag, sqtype);
1888 
1889         dmp = dp->d_dmp;
1890 
1891         setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1892             B_TRUE);
1893 
1894         /*
1895          * XXX Remove any "odd" messages from the queue.
1896          * Keep only M_DATA, M_PROTO, M_PCPROTO.
1897          */
1898         error = strdoioctl(stp, &strioc, FNATIVE,
1899             K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1900         if (error != 0) {
1901                 lbfree(linkp);
1902 
1903                 if (!(passyncq->sq_flags & SQ_BLOCKED))
1904                         blocksq(passyncq, SQ_BLOCKED, 0);
1905                 /*
1906                  * Restore the stream head queue and then remove
1907                  * the passq. Turn off STPLEX before we turn on
1908                  * the stream by removing the passq.
1909                  */
1910                 rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1911                 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1912                     B_TRUE);
1913 
1914                 mutex_enter(&stpdown->sd_lock);
1915                 stpdown->sd_flag &= ~STPLEX;
1916                 mutex_exit(&stpdown->sd_lock);
1917 
1918                 link_rempassthru(passq);
1919 
1920                 mutex_enter(&stpdown->sd_lock);
1921                 stpdown->sd_flag &= ~STRPLUMB;
1922                 /* Wakeup anyone waiting for STRPLUMB to clear. */
1923                 cv_broadcast(&stpdown->sd_monitor);
1924                 mutex_exit(&stpdown->sd_lock);
1925 
1926                 mutex_exit(&muxifier);
1927                 netstack_rele(ss->ss_netstack);
1928                 return (error);
1929         }
1930         mutex_enter(&fpdown->f_tlock);
1931         fpdown->f_count++;
1932         mutex_exit(&fpdown->f_tlock);
1933 
1934         /*
1935          * if we've made it here the linkage is all set up so we should also
1936          * set up the layered driver linkages
1937          */
1938 
1939         ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1940         if (cmd == I_LINK) {
1941                 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1942         } else {
1943                 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1944         }
1945 
1946         link_rempassthru(passq);
1947 
1948         mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1949 
1950         /*
1951          * Mark the upper stream as having dependent links
1952          * so that strclose can clean it up.
1953          */
1954         if (cmd == I_LINK) {
1955                 mutex_enter(&stp->sd_lock);
1956                 stp->sd_flag |= STRHASLINKS;
1957                 mutex_exit(&stp->sd_lock);
1958         }
1959         /*
1960          * Wake up any other processes that may have been
1961          * waiting on the lower stream. These will all
1962          * error out.
1963          */
1964         mutex_enter(&stpdown->sd_lock);
1965         /* The passthru module is removed so we may release STRPLUMB */
1966         stpdown->sd_flag &= ~STRPLUMB;
1967         cv_broadcast(&rq->q_wait);
1968         cv_broadcast(&_WR(rq)->q_wait);
1969         cv_broadcast(&stpdown->sd_monitor);
1970         mutex_exit(&stpdown->sd_lock);
1971         mutex_exit(&muxifier);
1972         *rvalp = linkp->li_lblk.l_index;
1973         netstack_rele(ss->ss_netstack);
1974         return (0);
1975 }
1976 
1977 int
1978 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1979 {
1980         int             ret;
1981         struct file     *fpdown;
1982 
1983         fpdown = getf(arg);
1984         ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1985         if (fpdown != NULL)
1986                 releasef(arg);
1987         return (ret);
1988 }
1989 
1990 /*
1991  * Unlink a multiplexor link. Stp is the controlling stream for the
1992  * link, and linkp points to the link's entry in the linkinfo list.
1993  * The muxifier lock must be held on entry and is dropped on exit.
1994  *
1995  * NOTE : Currently it is assumed that mux would process all the messages
1996  * sitting on it's queue before ACKing the UNLINK. It is the responsibility
1997  * of the mux to handle all the messages that arrive before UNLINK.
1998  * If the mux has to send down messages on its lower stream before
1999  * ACKing I_UNLINK, then it *should* know to handle messages even
2000  * after the UNLINK is acked (actually it should be able to handle till we
2001  * re-block the read side of the pass queue here). If the mux does not
2002  * open up the lower stream, any messages that arrive during UNLINK
2003  * will be put in the stream head. In the case of lower stream opening
2004  * up, some messages might land in the stream head depending on when
2005  * the message arrived and when the read side of the pass queue was
2006  * re-blocked.
2007  */
2008 int
2009 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2010     str_stack_t *ss)
2011 {
2012         struct strioctl strioc;
2013         struct stdata *stpdown;
2014         queue_t *rq, *wrq;
2015         queue_t *passq;
2016         syncq_t *passyncq;
2017         int error = 0;
2018         file_t *fpdown;
2019 
2020         ASSERT(MUTEX_HELD(&muxifier));
2021 
2022         stpdown = linkp->li_fpdown->f_vnode->v_stream;
2023 
2024         /*
2025          * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2026          */
2027         mutex_enter(&stpdown->sd_lock);
2028         stpdown->sd_flag |= STRPLUMB;
2029         mutex_exit(&stpdown->sd_lock);
2030 
2031         /*
2032          * Add passthru queue below lower mux. This will block
2033          * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2034          */
2035         passq = link_addpassthru(stpdown);
2036 
2037         if ((flag & LINKTYPEMASK) == LINKNORMAL)
2038                 strioc.ic_cmd = I_UNLINK;
2039         else
2040                 strioc.ic_cmd = I_PUNLINK;
2041         strioc.ic_timout = INFTIM;
2042         strioc.ic_len = sizeof (struct linkblk);
2043         strioc.ic_dp = (char *)&linkp->li_lblk;
2044 
2045         error = strdoioctl(stp, &strioc, FNATIVE,
2046             K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2047 
2048         /*
2049          * If there was an error and this is not called via strclose,
2050          * return to the user. Otherwise, pretend there was no error
2051          * and close the link.
2052          */
2053         if (error) {
2054                 if (flag & LINKCLOSE) {
2055                         cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2056                             "unlink ioctl, closing anyway (%d)\n", error);
2057                 } else {
2058                         link_rempassthru(passq);
2059                         mutex_enter(&stpdown->sd_lock);
2060                         stpdown->sd_flag &= ~STRPLUMB;
2061                         cv_broadcast(&stpdown->sd_monitor);
2062                         mutex_exit(&stpdown->sd_lock);
2063                         mutex_exit(&muxifier);
2064                         return (error);
2065                 }
2066         }
2067 
2068         mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2069         fpdown = linkp->li_fpdown;
2070         lbfree(linkp);
2071 
2072         /*
2073          * We go ahead and drop muxifier here--it's a nasty global lock that
2074          * can slow others down. It's okay to since attempts to mlink() this
2075          * stream will be stopped because STPLEX is still set in the stdata
2076          * structure, and munlink() is stopped because mux_rmvedge() and
2077          * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2078          * respectively.  Note that we defer the closef() of fpdown until
2079          * after we drop muxifier since strclose() can call munlinkall().
2080          */
2081         mutex_exit(&muxifier);
2082 
2083         wrq = stpdown->sd_wrq;
2084         rq = _RD(wrq);
2085 
2086         /*
2087          * Get rid of outstanding service procedure runs, before we make
2088          * it a stream head, since a stream head doesn't have any service
2089          * procedure.
2090          */
2091         disable_svc(rq);
2092         wait_svc(rq);
2093 
2094         /*
2095          * Since we don't disable the syncq for QPERMOD, we wait for whatever
2096          * is queued up to be finished. mux should take care that nothing is
2097          * send down to this queue. We should do it now as we're going to block
2098          * passyncq if it was unblocked.
2099          */
2100         if (wrq->q_flag & QPERMOD) {
2101                 syncq_t *sq = wrq->q_syncq;
2102 
2103                 mutex_enter(SQLOCK(sq));
2104                 while (wrq->q_sqflags & Q_SQQUEUED) {
2105                         sq->sq_flags |= SQ_WANTWAKEUP;
2106                         cv_wait(&sq->sq_wait, SQLOCK(sq));
2107                 }
2108                 mutex_exit(SQLOCK(sq));
2109         }
2110         passyncq = passq->q_syncq;
2111         if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2112 
2113                 syncq_t *sq, *outer;
2114 
2115                 /*
2116                  * Messages could be flowing from underneath. We will
2117                  * block the read side of the passq. This would be
2118                  * sufficient for QPAIR and QPERQ muxes to ensure
2119                  * that no data is flowing up into this queue
2120                  * and hence no thread active in this instance of
2121                  * lower mux. But for QPERMOD and QMTOUTPERIM there
2122                  * could be messages on the inner and outer/inner
2123                  * syncqs respectively. We will wait for them to drain.
2124                  * Because passq is blocked messages end up in the syncq
2125                  * And qfill_syncq could possibly end up setting QFULL
2126                  * which will access the rq->q_flag. Hence, we have to
2127                  * acquire the QLOCK in setq.
2128                  *
2129                  * XXX Messages can also flow from top into this
2130                  * queue though the unlink is over (Ex. some instance
2131                  * in putnext() called from top that has still not
2132                  * accessed this queue. And also putq(lowerq) ?).
2133                  * Solution : How about blocking the l_qtop queue ?
2134                  * Do we really care about such pure D_MP muxes ?
2135                  */
2136 
2137                 blocksq(passyncq, SQ_BLOCKED, 0);
2138 
2139                 sq = rq->q_syncq;
2140                 if ((outer = sq->sq_outer) != NULL) {
2141 
2142                         /*
2143                          * We have to just wait for the outer sq_count
2144                          * drop to zero. As this does not prevent new
2145                          * messages to enter the outer perimeter, this
2146                          * is subject to starvation.
2147                          *
2148                          * NOTE :Because of blocksq above, messages could
2149                          * be in the inner syncq only because of some
2150                          * thread holding the outer perimeter exclusively.
2151                          * Hence it would be sufficient to wait for the
2152                          * exclusive holder of the outer perimeter to drain
2153                          * the inner and outer syncqs. But we will not depend
2154                          * on this feature and hence check the inner syncqs
2155                          * separately.
2156                          */
2157                         wait_syncq(outer);
2158                 }
2159 
2160 
2161                 /*
2162                  * There could be messages destined for
2163                  * this queue. Let the exclusive holder
2164                  * drain it.
2165                  */
2166 
2167                 wait_syncq(sq);
2168                 ASSERT((rq->q_flag & QPERMOD) ||
2169                     ((rq->q_syncq->sq_head == NULL) &&
2170                     (_WR(rq)->q_syncq->sq_head == NULL)));
2171         }
2172 
2173         /*
2174          * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2175          * case as we don't disable its syncq or remove it off the syncq
2176          * service list.
2177          */
2178         if (rq->q_flag & QPERMOD) {
2179                 syncq_t *sq = rq->q_syncq;
2180 
2181                 mutex_enter(SQLOCK(sq));
2182                 while (rq->q_sqflags & Q_SQQUEUED) {
2183                         sq->sq_flags |= SQ_WANTWAKEUP;
2184                         cv_wait(&sq->sq_wait, SQLOCK(sq));
2185                 }
2186                 mutex_exit(SQLOCK(sq));
2187         }
2188 
2189         /*
2190          * flush_syncq changes states only when there are some messages to
2191          * free, i.e. when it returns non-zero value to return.
2192          */
2193         ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2194         ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2195 
2196         /*
2197          * Nobody else should know about this queue now.
2198          * If the mux did not process the messages before
2199          * acking the I_UNLINK, free them now.
2200          */
2201 
2202         flushq(rq, FLUSHALL);
2203         flushq(_WR(rq), FLUSHALL);
2204 
2205         /*
2206          * Convert the mux lower queue into a stream head queue.
2207          * Turn off STPLEX before we turn on the stream by removing the passq.
2208          */
2209         rq->q_ptr = wrq->q_ptr = stpdown;
2210         setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2211 
2212         ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2213         ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2214 
2215         enable_svc(rq);
2216 
2217         /*
2218          * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2219          * needs to be set to prevent reopen() of the stream - such reopen may
2220          * try to call non-existent pass queue open routine and panic.
2221          */
2222         mutex_enter(&stpdown->sd_lock);
2223         stpdown->sd_flag &= ~STPLEX;
2224         mutex_exit(&stpdown->sd_lock);
2225 
2226         ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2227             ((flag & LINKTYPEMASK) == LINKPERSIST));
2228 
2229         /* clean up the layered driver linkages */
2230         if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2231                 ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2232         } else {
2233                 ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2234         }
2235 
2236         link_rempassthru(passq);
2237 
2238         /*
2239          * Now all plumbing changes are finished and STRPLUMB is no
2240          * longer needed.
2241          */
2242         mutex_enter(&stpdown->sd_lock);
2243         stpdown->sd_flag &= ~STRPLUMB;
2244         cv_broadcast(&stpdown->sd_monitor);
2245         mutex_exit(&stpdown->sd_lock);
2246 
2247         (void) closef(fpdown);
2248         return (0);
2249 }
2250 
2251 /*
2252  * Unlink all multiplexor links for which stp is the controlling stream.
2253  * Return 0, or a non-zero errno on failure.
2254  */
2255 int
2256 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2257 {
2258         linkinfo_t *linkp;
2259         int error = 0;
2260 
2261         mutex_enter(&muxifier);
2262         while (linkp = findlinks(stp, 0, flag, ss)) {
2263                 /*
2264                  * munlink() releases the muxifier lock.
2265                  */
2266                 if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2267                         return (error);
2268                 mutex_enter(&muxifier);
2269         }
2270         mutex_exit(&muxifier);
2271         return (0);
2272 }
2273 
2274 /*
2275  * A multiplexor link has been made. Add an
2276  * edge to the directed graph.
2277  */
2278 void
2279 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2280 {
2281         struct mux_node *np;
2282         struct mux_edge *ep;
2283         major_t upmaj;
2284         major_t lomaj;
2285 
2286         upmaj = getmajor(upstp->sd_vnode->v_rdev);
2287         lomaj = getmajor(lostp->sd_vnode->v_rdev);
2288         np = &ss->ss_mux_nodes[upmaj];
2289         if (np->mn_outp) {
2290                 ep = np->mn_outp;
2291                 while (ep->me_nextp)
2292                         ep = ep->me_nextp;
2293                 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2294                 ep = ep->me_nextp;
2295         } else {
2296                 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2297                 ep = np->mn_outp;
2298         }
2299         ep->me_nextp = NULL;
2300         ep->me_muxid = muxid;
2301         /*
2302          * Save the dev_t for the purposes of str_stack_shutdown.
2303          * str_stack_shutdown assumes that the device allows reopen, since
2304          * this dev_t is the one after any cloning by xx_open().
2305          * Would prefer finding the dev_t from before any cloning,
2306          * but specfs doesn't retain that.
2307          */
2308         ep->me_dev = upstp->sd_vnode->v_rdev;
2309         if (lostp->sd_vnode->v_type == VFIFO)
2310                 ep->me_nodep = NULL;
2311         else
2312                 ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2313 }
2314 
2315 /*
2316  * A multiplexor link has been removed. Remove the
2317  * edge in the directed graph.
2318  */
2319 void
2320 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2321 {
2322         struct mux_node *np;
2323         struct mux_edge *ep;
2324         struct mux_edge *pep = NULL;
2325         major_t upmaj;
2326 
2327         upmaj = getmajor(upstp->sd_vnode->v_rdev);
2328         np = &ss->ss_mux_nodes[upmaj];
2329         ASSERT(np->mn_outp != NULL);
2330         ep = np->mn_outp;
2331         while (ep) {
2332                 if (ep->me_muxid == muxid) {
2333                         if (pep)
2334                                 pep->me_nextp = ep->me_nextp;
2335                         else
2336                                 np->mn_outp = ep->me_nextp;
2337                         kmem_free(ep, sizeof (struct mux_edge));
2338                         return;
2339                 }
2340                 pep = ep;
2341                 ep = ep->me_nextp;
2342         }
2343         ASSERT(0);      /* should not reach here */
2344 }
2345 
2346 /*
2347  * Translate the device flags (from conf.h) to the corresponding
2348  * qflag and sq_flag (type) values.
2349  */
2350 int
2351 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2352         uint32_t *sqtypep)
2353 {
2354         uint32_t qflag = 0;
2355         uint32_t sqtype = 0;
2356 
2357         if (devflag & _D_OLD)
2358                 goto bad;
2359 
2360         /* Inner perimeter presence and scope */
2361         switch (devflag & D_MTINNER_MASK) {
2362         case D_MP:
2363                 qflag |= QMTSAFE;
2364                 sqtype |= SQ_CI;
2365                 break;
2366         case D_MTPERQ|D_MP:
2367                 qflag |= QPERQ;
2368                 break;
2369         case D_MTQPAIR|D_MP:
2370                 qflag |= QPAIR;
2371                 break;
2372         case D_MTPERMOD|D_MP:
2373                 qflag |= QPERMOD;
2374                 break;
2375         default:
2376                 goto bad;
2377         }
2378 
2379         /* Outer perimeter */
2380         if (devflag & D_MTOUTPERIM) {
2381                 switch (devflag & D_MTINNER_MASK) {
2382                 case D_MP:
2383                 case D_MTPERQ|D_MP:
2384                 case D_MTQPAIR|D_MP:
2385                         break;
2386                 default:
2387                         goto bad;
2388                 }
2389                 qflag |= QMTOUTPERIM;
2390         }
2391 
2392         /* Inner perimeter modifiers */
2393         if (devflag & D_MTINNER_MOD) {
2394                 switch (devflag & D_MTINNER_MASK) {
2395                 case D_MP:
2396                         goto bad;
2397                 default:
2398                         break;
2399                 }
2400                 if (devflag & D_MTPUTSHARED)
2401                         sqtype |= SQ_CIPUT;
2402                 if (devflag & _D_MTOCSHARED) {
2403                         /*
2404                          * The code in putnext assumes that it has the
2405                          * highest concurrency by not checking sq_count.
2406                          * Thus _D_MTOCSHARED can only be supported when
2407                          * D_MTPUTSHARED is set.
2408                          */
2409                         if (!(devflag & D_MTPUTSHARED))
2410                                 goto bad;
2411                         sqtype |= SQ_CIOC;
2412                 }
2413                 if (devflag & _D_MTCBSHARED) {
2414                         /*
2415                          * The code in putnext assumes that it has the
2416                          * highest concurrency by not checking sq_count.
2417                          * Thus _D_MTCBSHARED can only be supported when
2418                          * D_MTPUTSHARED is set.
2419                          */
2420                         if (!(devflag & D_MTPUTSHARED))
2421                                 goto bad;
2422                         sqtype |= SQ_CICB;
2423                 }
2424                 if (devflag & _D_MTSVCSHARED) {
2425                         /*
2426                          * The code in putnext assumes that it has the
2427                          * highest concurrency by not checking sq_count.
2428                          * Thus _D_MTSVCSHARED can only be supported when
2429                          * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2430                          * supported only for QPERMOD.
2431                          */
2432                         if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2433                                 goto bad;
2434                         sqtype |= SQ_CISVC;
2435                 }
2436         }
2437 
2438         /* Default outer perimeter concurrency */
2439         sqtype |= SQ_CO;
2440 
2441         /* Outer perimeter modifiers */
2442         if (devflag & D_MTOCEXCL) {
2443                 if (!(devflag & D_MTOUTPERIM)) {
2444                         /* No outer perimeter */
2445                         goto bad;
2446                 }
2447                 sqtype &= ~SQ_COOC;
2448         }
2449 
2450         /* Synchronous Streams extended qinit structure */
2451         if (devflag & D_SYNCSTR)
2452                 qflag |= QSYNCSTR;
2453 
2454         /*
2455          * Private flag used by a transport module to indicate
2456          * to sockfs that it supports direct-access mode without
2457          * having to go through STREAMS.
2458          */
2459         if (devflag & _D_DIRECT) {
2460                 /* Reject unless the module is fully-MT (no perimeter) */
2461                 if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2462                         goto bad;
2463                 qflag |= _QDIRECT;
2464         }
2465 
2466         *qflagp = qflag;
2467         *sqtypep = sqtype;
2468         return (0);
2469 
2470 bad:
2471         cmn_err(CE_WARN,
2472             "stropen: bad MT flags (0x%x) in driver '%s'",
2473             (int)(qflag & D_MTSAFETY_MASK),
2474             stp->st_rdinit->qi_minfo->mi_idname);
2475 
2476         return (EINVAL);
2477 }
2478 
2479 /*
2480  * Set the interface values for a pair of queues (qinit structure,
2481  * packet sizes, water marks).
2482  * setq assumes that the caller does not have a claim (entersq or claimq)
2483  * on the queue.
2484  */
2485 void
2486 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2487     perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2488 {
2489         queue_t *wq;
2490         syncq_t *sq, *outer;
2491 
2492         ASSERT(rq->q_flag & QREADR);
2493         ASSERT((qflag & QMT_TYPEMASK) != 0);
2494         IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2495 
2496         wq = _WR(rq);
2497         rq->q_qinfo = rinit;
2498         rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2499         rq->q_lowat = rinit->qi_minfo->mi_lowat;
2500         rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2501         rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2502         wq->q_qinfo = winit;
2503         wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2504         wq->q_lowat = winit->qi_minfo->mi_lowat;
2505         wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2506         wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2507 
2508         /* Remove old syncqs */
2509         sq = rq->q_syncq;
2510         outer = sq->sq_outer;
2511         if (outer != NULL) {
2512                 ASSERT(wq->q_syncq->sq_outer == outer);
2513                 outer_remove(outer, rq->q_syncq);
2514                 if (wq->q_syncq != rq->q_syncq)
2515                         outer_remove(outer, wq->q_syncq);
2516         }
2517         ASSERT(sq->sq_outer == NULL);
2518         ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2519 
2520         if (sq != SQ(rq)) {
2521                 if (!(rq->q_flag & QPERMOD))
2522                         free_syncq(sq);
2523                 if (wq->q_syncq == rq->q_syncq)
2524                         wq->q_syncq = NULL;
2525                 rq->q_syncq = NULL;
2526         }
2527         if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2528             wq->q_syncq != SQ(rq)) {
2529                 free_syncq(wq->q_syncq);
2530                 wq->q_syncq = NULL;
2531         }
2532         ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2533             rq->q_syncq->sq_tail == NULL));
2534         ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2535             wq->q_syncq->sq_tail == NULL));
2536 
2537         if (!(rq->q_flag & QPERMOD) &&
2538             rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2539                 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2540                 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2541                     rq->q_syncq->sq_nciputctrl, 0);
2542                 ASSERT(ciputctrl_cache != NULL);
2543                 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2544                 rq->q_syncq->sq_ciputctrl = NULL;
2545                 rq->q_syncq->sq_nciputctrl = 0;
2546         }
2547 
2548         if (!(wq->q_flag & QPERMOD) &&
2549             wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2550                 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2551                 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2552                     wq->q_syncq->sq_nciputctrl, 0);
2553                 ASSERT(ciputctrl_cache != NULL);
2554                 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2555                 wq->q_syncq->sq_ciputctrl = NULL;
2556                 wq->q_syncq->sq_nciputctrl = 0;
2557         }
2558 
2559         sq = SQ(rq);
2560         ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2561         ASSERT(sq->sq_outer == NULL);
2562         ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2563 
2564         /*
2565          * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2566          * bits in sq_flag based on the sqtype.
2567          */
2568         ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2569 
2570         rq->q_syncq = wq->q_syncq = sq;
2571         sq->sq_type = sqtype;
2572         sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2573 
2574         /*
2575          *  We are making sq_svcflags zero,
2576          *  resetting SQ_DISABLED in case it was set by
2577          *  wait_svc() in the munlink path.
2578          *
2579          */
2580         ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2581         sq->sq_svcflags = 0;
2582 
2583         /*
2584          * We need to acquire the lock here for the mlink and munlink case,
2585          * where canputnext, backenable, etc can access the q_flag.
2586          */
2587         if (lock_needed) {
2588                 mutex_enter(QLOCK(rq));
2589                 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2590                 mutex_exit(QLOCK(rq));
2591                 mutex_enter(QLOCK(wq));
2592                 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2593                 mutex_exit(QLOCK(wq));
2594         } else {
2595                 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2596                 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2597         }
2598 
2599         if (qflag & QPERQ) {
2600                 /* Allocate a separate syncq for the write side */
2601                 sq = new_syncq();
2602                 sq->sq_type = rq->q_syncq->sq_type;
2603                 sq->sq_flags = rq->q_syncq->sq_flags;
2604                 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2605                     sq->sq_oprev == NULL);
2606                 wq->q_syncq = sq;
2607         }
2608         if (qflag & QPERMOD) {
2609                 sq = dmp->dm_sq;
2610 
2611                 /*
2612                  * Assert that we do have an inner perimeter syncq and that it
2613                  * does not have an outer perimeter associated with it.
2614                  */
2615                 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2616                     sq->sq_oprev == NULL);
2617                 rq->q_syncq = wq->q_syncq = sq;
2618         }
2619         if (qflag & QMTOUTPERIM) {
2620                 outer = dmp->dm_sq;
2621 
2622                 ASSERT(outer->sq_outer == NULL);
2623                 outer_insert(outer, rq->q_syncq);
2624                 if (wq->q_syncq != rq->q_syncq)
2625                         outer_insert(outer, wq->q_syncq);
2626         }
2627         ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2628             (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2629         ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2630             (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2631         ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2632 
2633         /*
2634          * Initialize struio() types.
2635          */
2636         rq->q_struiot =
2637             (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2638         wq->q_struiot =
2639             (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2640 }
2641 
2642 perdm_t *
2643 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2644 {
2645         syncq_t *sq;
2646         perdm_t **pp;
2647         perdm_t *p;
2648         perdm_t *dmp;
2649 
2650         ASSERT(str != NULL);
2651         ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2652 
2653         rw_enter(&perdm_rwlock, RW_READER);
2654         for (p = perdm_list; p != NULL; p = p->dm_next) {
2655                 if (p->dm_str == str) {      /* found one */
2656                         atomic_inc_32(&(p->dm_ref));
2657                         rw_exit(&perdm_rwlock);
2658                         return (p);
2659                 }
2660         }
2661         rw_exit(&perdm_rwlock);
2662 
2663         sq = new_syncq();
2664         if (qflag & QPERMOD) {
2665                 sq->sq_type = sqtype | SQ_PERMOD;
2666                 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2667         } else {
2668                 ASSERT(qflag & QMTOUTPERIM);
2669                 sq->sq_onext = sq->sq_oprev = sq;
2670         }
2671 
2672         dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2673         dmp->dm_sq = sq;
2674         dmp->dm_str = str;
2675         dmp->dm_ref = 1;
2676         dmp->dm_next = NULL;
2677 
2678         rw_enter(&perdm_rwlock, RW_WRITER);
2679         for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2680                 if (p->dm_str == str) {      /* already present */
2681                         p->dm_ref++;
2682                         rw_exit(&perdm_rwlock);
2683                         free_syncq(sq);
2684                         kmem_free(dmp, sizeof (perdm_t));
2685                         return (p);
2686                 }
2687         }
2688 
2689         *pp = dmp;
2690         rw_exit(&perdm_rwlock);
2691         return (dmp);
2692 }
2693 
2694 void
2695 rele_dm(perdm_t *dmp)
2696 {
2697         perdm_t **pp;
2698         perdm_t *p;
2699 
2700         rw_enter(&perdm_rwlock, RW_WRITER);
2701         ASSERT(dmp->dm_ref > 0);
2702 
2703         if (--dmp->dm_ref > 0) {
2704                 rw_exit(&perdm_rwlock);
2705                 return;
2706         }
2707 
2708         for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2709                 if (p == dmp)
2710                         break;
2711         ASSERT(p == dmp);
2712         *pp = p->dm_next;
2713         rw_exit(&perdm_rwlock);
2714 
2715         /*
2716          * Wait for any background processing that relies on the
2717          * syncq to complete before it is freed.
2718          */
2719         wait_sq_svc(p->dm_sq);
2720         free_syncq(p->dm_sq);
2721         kmem_free(p, sizeof (perdm_t));
2722 }
2723 
2724 /*
2725  * Make a protocol message given control and data buffers.
2726  * n.b., this can block; be careful of what locks you hold when calling it.
2727  *
2728  * If sd_maxblk is less than *iosize this routine can fail part way through
2729  * (due to an allocation failure). In this case on return *iosize will contain
2730  * the amount that was consumed. Otherwise *iosize will not be modified
2731  * i.e. it will contain the amount that was consumed.
2732  */
2733 int
2734 strmakemsg(
2735         struct strbuf *mctl,
2736         ssize_t *iosize,
2737         struct uio *uiop,
2738         stdata_t *stp,
2739         int32_t flag,
2740         mblk_t **mpp)
2741 {
2742         mblk_t *mpctl = NULL;
2743         mblk_t *mpdata = NULL;
2744         int error;
2745 
2746         ASSERT(uiop != NULL);
2747 
2748         *mpp = NULL;
2749         /* Create control part, if any */
2750         if ((mctl != NULL) && (mctl->len >= 0)) {
2751                 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2752                 if (error)
2753                         return (error);
2754         }
2755         /* Create data part, if any */
2756         if (*iosize >= 0) {
2757                 error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2758                 if (error) {
2759                         freemsg(mpctl);
2760                         return (error);
2761                 }
2762         }
2763         if (mpctl != NULL) {
2764                 if (mpdata != NULL)
2765                         linkb(mpctl, mpdata);
2766                 *mpp = mpctl;
2767         } else {
2768                 *mpp = mpdata;
2769         }
2770         return (0);
2771 }
2772 
2773 /*
2774  * Make the control part of a protocol message given a control buffer.
2775  * n.b., this can block; be careful of what locks you hold when calling it.
2776  */
2777 int
2778 strmakectl(
2779         struct strbuf *mctl,
2780         int32_t flag,
2781         int32_t fflag,
2782         mblk_t **mpp)
2783 {
2784         mblk_t *bp = NULL;
2785         unsigned char msgtype;
2786         int error = 0;
2787         cred_t *cr = CRED();
2788 
2789         /* We do not support interrupt threads using the stream head to send */
2790         ASSERT(cr != NULL);
2791 
2792         *mpp = NULL;
2793         /*
2794          * Create control part of message, if any.
2795          */
2796         if ((mctl != NULL) && (mctl->len >= 0)) {
2797                 caddr_t base;
2798                 int ctlcount;
2799                 int allocsz;
2800 
2801                 if (flag & RS_HIPRI)
2802                         msgtype = M_PCPROTO;
2803                 else
2804                         msgtype = M_PROTO;
2805 
2806                 ctlcount = mctl->len;
2807                 base = mctl->buf;
2808 
2809                 /*
2810                  * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2811                  * blocks by increasing the size to something more usable.
2812                  */
2813                 allocsz = MAX(ctlcount, 64);
2814 
2815                 /*
2816                  * Range checking has already been done; simply try
2817                  * to allocate a message block for the ctl part.
2818                  */
2819                 while ((bp = allocb_cred(allocsz, cr,
2820                     curproc->p_pid)) == NULL) {
2821                         if (fflag & (FNDELAY|FNONBLOCK))
2822                                 return (EAGAIN);
2823                         if (error = strwaitbuf(allocsz, BPRI_MED))
2824                                 return (error);
2825                 }
2826 
2827                 bp->b_datap->db_type = msgtype;
2828                 if (copyin(base, bp->b_wptr, ctlcount)) {
2829                         freeb(bp);
2830                         return (EFAULT);
2831                 }
2832                 bp->b_wptr += ctlcount;
2833         }
2834         *mpp = bp;
2835         return (0);
2836 }
2837 
2838 /*
2839  * Make a protocol message given data buffers.
2840  * n.b., this can block; be careful of what locks you hold when calling it.
2841  *
2842  * If sd_maxblk is less than *iosize this routine can fail part way through
2843  * (due to an allocation failure). In this case on return *iosize will contain
2844  * the amount that was consumed. Otherwise *iosize will not be modified
2845  * i.e. it will contain the amount that was consumed.
2846  */
2847 int
2848 strmakedata(
2849         ssize_t   *iosize,
2850         struct uio *uiop,
2851         stdata_t *stp,
2852         int32_t flag,
2853         mblk_t **mpp)
2854 {
2855         mblk_t *mp = NULL;
2856         mblk_t *bp;
2857         int wroff = (int)stp->sd_wroff;
2858         int tail_len = (int)stp->sd_tail;
2859         int extra = wroff + tail_len;
2860         int error = 0;
2861         ssize_t maxblk;
2862         ssize_t count = *iosize;
2863         cred_t *cr;
2864 
2865         *mpp = NULL;
2866         if (count < 0)
2867                 return (0);
2868 
2869         /* We do not support interrupt threads using the stream head to send */
2870         cr = CRED();
2871         ASSERT(cr != NULL);
2872 
2873         maxblk = stp->sd_maxblk;
2874         if (maxblk == INFPSZ)
2875                 maxblk = count;
2876 
2877         /*
2878          * Create data part of message, if any.
2879          */
2880         do {
2881                 ssize_t size;
2882                 dblk_t  *dp;
2883 
2884                 ASSERT(uiop);
2885 
2886                 size = MIN(count, maxblk);
2887 
2888                 while ((bp = allocb_cred(size + extra, cr,
2889                     curproc->p_pid)) == NULL) {
2890                         error = EAGAIN;
2891                         if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2892                             (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2893                                 if (count == *iosize) {
2894                                         freemsg(mp);
2895                                         return (error);
2896                                 } else {
2897                                         *iosize -= count;
2898                                         *mpp = mp;
2899                                         return (0);
2900                                 }
2901                         }
2902                 }
2903                 dp = bp->b_datap;
2904                 dp->db_cpid = curproc->p_pid;
2905                 ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2906                 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2907 
2908                 if (flag & STRUIO_POSTPONE) {
2909                         /*
2910                          * Setup the stream uio portion of the
2911                          * dblk for subsequent use by struioget().
2912                          */
2913                         dp->db_struioflag = STRUIO_SPEC;
2914                         dp->db_cksumstart = 0;
2915                         dp->db_cksumstuff = 0;
2916                         dp->db_cksumend = size;
2917                         *(long long *)dp->db_struioun.data = 0ll;
2918                         bp->b_wptr += size;
2919                 } else {
2920                         if (stp->sd_copyflag & STRCOPYCACHED)
2921                                 uiop->uio_extflg |= UIO_COPY_CACHED;
2922 
2923                         if (size != 0) {
2924                                 error = uiomove(bp->b_wptr, size, UIO_WRITE,
2925                                     uiop);
2926                                 if (error != 0) {
2927                                         freeb(bp);
2928                                         freemsg(mp);
2929                                         return (error);
2930                                 }
2931                         }
2932                         bp->b_wptr += size;
2933 
2934                         if (stp->sd_wputdatafunc != NULL) {
2935                                 mblk_t *newbp;
2936 
2937                                 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2938                                     bp, NULL, NULL, NULL, NULL);
2939                                 if (newbp == NULL) {
2940                                         freeb(bp);
2941                                         freemsg(mp);
2942                                         return (ECOMM);
2943                                 }
2944                                 bp = newbp;
2945                         }
2946                 }
2947 
2948                 count -= size;
2949 
2950                 if (mp == NULL)
2951                         mp = bp;
2952                 else
2953                         linkb(mp, bp);
2954         } while (count > 0);
2955 
2956         *mpp = mp;
2957         return (0);
2958 }
2959 
2960 /*
2961  * Wait for a buffer to become available. Return non-zero errno
2962  * if not able to wait, 0 if buffer is probably there.
2963  */
2964 int
2965 strwaitbuf(size_t size, int pri)
2966 {
2967         bufcall_id_t id;
2968 
2969         mutex_enter(&bcall_monitor);
2970         if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2971             &ttoproc(curthread)->p_flag_cv)) == 0) {
2972                 mutex_exit(&bcall_monitor);
2973                 return (ENOSR);
2974         }
2975         if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2976                 unbufcall(id);
2977                 mutex_exit(&bcall_monitor);
2978                 return (EINTR);
2979         }
2980         unbufcall(id);
2981         mutex_exit(&bcall_monitor);
2982         return (0);
2983 }
2984 
2985 /*
2986  * This function waits for a read or write event to happen on a stream.
2987  * fmode can specify FNDELAY and/or FNONBLOCK.
2988  * The timeout is in ms with -1 meaning infinite.
2989  * The flag values work as follows:
2990  *      READWAIT        Check for read side errors, send M_READ
2991  *      GETWAIT         Check for read side errors, no M_READ
2992  *      WRITEWAIT       Check for write side errors.
2993  *      NOINTR          Do not return error if nonblocking or timeout.
2994  *      STR_NOERROR     Ignore all errors except STPLEX.
2995  *      STR_NOSIG       Ignore/hold signals during the duration of the call.
2996  *      STR_PEEK        Pass through the strgeterr().
2997  */
2998 int
2999 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3000     int *done)
3001 {
3002         int slpflg, errs;
3003         int error;
3004         kcondvar_t *sleepon;
3005         mblk_t *mp;
3006         ssize_t *rd_count;
3007         clock_t rval;
3008 
3009         ASSERT(MUTEX_HELD(&stp->sd_lock));
3010         if ((flag & READWAIT) || (flag & GETWAIT)) {
3011                 slpflg = RSLEEP;
3012                 sleepon = &_RD(stp->sd_wrq)->q_wait;
3013                 errs = STRDERR|STPLEX;
3014         } else {
3015                 slpflg = WSLEEP;
3016                 sleepon = &stp->sd_wrq->q_wait;
3017                 errs = STWRERR|STRHUP|STPLEX;
3018         }
3019         if (flag & STR_NOERROR)
3020                 errs = STPLEX;
3021 
3022         if (stp->sd_wakeq & slpflg) {
3023                 /*
3024                  * A strwakeq() is pending, no need to sleep.
3025                  */
3026                 stp->sd_wakeq &= ~slpflg;
3027                 *done = 0;
3028                 return (0);
3029         }
3030 
3031         if (stp->sd_flag & errs) {
3032                 /*
3033                  * Check for errors before going to sleep since the
3034                  * caller might not have checked this while holding
3035                  * sd_lock.
3036                  */
3037                 error = strgeterr(stp, errs, (flag & STR_PEEK));
3038                 if (error != 0) {
3039                         *done = 1;
3040                         return (error);
3041                 }
3042         }
3043 
3044         /*
3045          * If any module downstream has requested read notification
3046          * by setting SNDMREAD flag using M_SETOPTS, send a message
3047          * down stream.
3048          */
3049         if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3050                 mutex_exit(&stp->sd_lock);
3051                 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3052                     (flag & STR_NOSIG), &error))) {
3053                         mutex_enter(&stp->sd_lock);
3054                         *done = 1;
3055                         return (error);
3056                 }
3057                 mp->b_datap->db_type = M_READ;
3058                 rd_count = (ssize_t *)mp->b_wptr;
3059                 *rd_count = count;
3060                 mp->b_wptr += sizeof (ssize_t);
3061                 /*
3062                  * Send the number of bytes requested by the
3063                  * read as the argument to M_READ.
3064                  */
3065                 stream_willservice(stp);
3066                 putnext(stp->sd_wrq, mp);
3067                 stream_runservice(stp);
3068                 mutex_enter(&stp->sd_lock);
3069 
3070                 /*
3071                  * If any data arrived due to inline processing
3072                  * of putnext(), don't sleep.
3073                  */
3074                 if (_RD(stp->sd_wrq)->q_first != NULL) {
3075                         *done = 0;
3076                         return (0);
3077                 }
3078         }
3079 
3080         if (fmode & (FNDELAY|FNONBLOCK)) {
3081                 if (!(flag & NOINTR))
3082                         error = EAGAIN;
3083                 else
3084                         error = 0;
3085                 *done = 1;
3086                 return (error);
3087         }
3088 
3089         stp->sd_flag |= slpflg;
3090         TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3091             "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3092             stp, flag, count, fmode, done);
3093 
3094         rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3095         if (rval > 0) {
3096                 /* EMPTY */
3097                 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3098                     "strwaitq awakes(2):%X, %X, %X, %X, %X",
3099                     stp, flag, count, fmode, done);
3100         } else if (rval == 0) {
3101                 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3102                     "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3103                     stp, flag, count, fmode, done);
3104                 stp->sd_flag &= ~slpflg;
3105                 cv_broadcast(sleepon);
3106                 if (!(flag & NOINTR))
3107                         error = EINTR;
3108                 else
3109                         error = 0;
3110                 *done = 1;
3111                 return (error);
3112         } else {
3113                 /* timeout */
3114                 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3115                     "strwaitq timeout:%p, %X, %lX, %X, %p",
3116                     stp, flag, count, fmode, done);
3117                 *done = 1;
3118                 if (!(flag & NOINTR))
3119                         return (ETIME);
3120                 else
3121                         return (0);
3122         }
3123         /*
3124          * If the caller implements delayed errors (i.e. queued after data)
3125          * we can not check for errors here since data as well as an
3126          * error might have arrived at the stream head. We return to
3127          * have the caller check the read queue before checking for errors.
3128          */
3129         if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3130                 error = strgeterr(stp, errs, (flag & STR_PEEK));
3131                 if (error != 0) {
3132                         *done = 1;
3133                         return (error);
3134                 }
3135         }
3136         *done = 0;
3137         return (0);
3138 }
3139 
3140 /*
3141  * Perform job control discipline access checks.
3142  * Return 0 for success and the errno for failure.
3143  */
3144 
3145 #define cantsend(p, t, sig) \
3146         (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3147 
3148 int
3149 straccess(struct stdata *stp, enum jcaccess mode)
3150 {
3151         extern kcondvar_t lbolt_cv;     /* XXX: should be in a header file */
3152         kthread_t *t = curthread;
3153         proc_t *p = ttoproc(t);
3154         sess_t *sp;
3155 
3156         ASSERT(mutex_owned(&stp->sd_lock));
3157 
3158         if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3159                 return (0);
3160 
3161         mutex_enter(&p->p_lock);         /* protects p_pgidp */
3162 
3163         for (;;) {
3164                 mutex_enter(&p->p_splock);       /* protects p->p_sessp */
3165                 sp = p->p_sessp;
3166                 mutex_enter(&sp->s_lock);        /* protects sp->* */
3167 
3168                 /*
3169                  * If this is not the calling process's controlling terminal
3170                  * or if the calling process is already in the foreground
3171                  * then allow access.
3172                  */
3173                 if (sp->s_dev != stp->sd_vnode->v_rdev ||
3174                     p->p_pgidp == stp->sd_pgidp) {
3175                         mutex_exit(&sp->s_lock);
3176                         mutex_exit(&p->p_splock);
3177                         mutex_exit(&p->p_lock);
3178                         return (0);
3179                 }
3180 
3181                 /*
3182                  * Check to see if controlling terminal has been deallocated.
3183                  */
3184                 if (sp->s_vp == NULL) {
3185                         if (!cantsend(p, t, SIGHUP))
3186                                 sigtoproc(p, t, SIGHUP);
3187                         mutex_exit(&sp->s_lock);
3188                         mutex_exit(&p->p_splock);
3189                         mutex_exit(&p->p_lock);
3190                         return (EIO);
3191                 }
3192 
3193                 mutex_exit(&sp->s_lock);
3194                 mutex_exit(&p->p_splock);
3195 
3196                 if (mode == JCGETP) {
3197                         mutex_exit(&p->p_lock);
3198                         return (0);
3199                 }
3200 
3201                 if (mode == JCREAD) {
3202                         if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3203                                 mutex_exit(&p->p_lock);
3204                                 return (EIO);
3205                         }
3206                         mutex_exit(&p->p_lock);
3207                         mutex_exit(&stp->sd_lock);
3208                         pgsignal(p->p_pgidp, SIGTTIN);
3209                         mutex_enter(&stp->sd_lock);
3210                         mutex_enter(&p->p_lock);
3211                 } else {  /* mode == JCWRITE or JCSETP */
3212                         if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3213                             cantsend(p, t, SIGTTOU)) {
3214                                 mutex_exit(&p->p_lock);
3215                                 return (0);
3216                         }
3217                         if (p->p_detached) {
3218                                 mutex_exit(&p->p_lock);
3219                                 return (EIO);
3220                         }
3221                         mutex_exit(&p->p_lock);
3222                         mutex_exit(&stp->sd_lock);
3223                         pgsignal(p->p_pgidp, SIGTTOU);
3224                         mutex_enter(&stp->sd_lock);
3225                         mutex_enter(&p->p_lock);
3226                 }
3227 
3228                 /*
3229                  * We call cv_wait_sig_swap() to cause the appropriate
3230                  * action for the jobcontrol signal to take place.
3231                  * If the signal is being caught, we will take the
3232                  * EINTR error return.  Otherwise, the default action
3233                  * of causing the process to stop will take place.
3234                  * In this case, we rely on the periodic cv_broadcast() on
3235                  * &lbolt_cv to wake us up to loop around and test again.
3236                  * We can't get here if the signal is ignored or
3237                  * if the current thread is blocking the signal.
3238                  */
3239                 mutex_exit(&stp->sd_lock);
3240                 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3241                         mutex_exit(&p->p_lock);
3242                         mutex_enter(&stp->sd_lock);
3243                         return (EINTR);
3244                 }
3245                 mutex_exit(&p->p_lock);
3246                 mutex_enter(&stp->sd_lock);
3247                 mutex_enter(&p->p_lock);
3248         }
3249 }
3250 
3251 /*
3252  * Return size of message of block type (bp->b_datap->db_type)
3253  */
3254 size_t
3255 xmsgsize(mblk_t *bp)
3256 {
3257         unsigned char type;
3258         size_t count = 0;
3259 
3260         type = bp->b_datap->db_type;
3261 
3262         for (; bp; bp = bp->b_cont) {
3263                 if (type != bp->b_datap->db_type)
3264                         break;
3265                 ASSERT(bp->b_wptr >= bp->b_rptr);
3266                 count += bp->b_wptr - bp->b_rptr;
3267         }
3268         return (count);
3269 }
3270 
3271 /*
3272  * Allocate a stream head.
3273  */
3274 struct stdata *
3275 shalloc(queue_t *qp)
3276 {
3277         stdata_t *stp;
3278 
3279         stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3280 
3281         stp->sd_wrq = _WR(qp);
3282         stp->sd_strtab = NULL;
3283         stp->sd_iocid = 0;
3284         stp->sd_mate = NULL;
3285         stp->sd_freezer = NULL;
3286         stp->sd_refcnt = 0;
3287         stp->sd_wakeq = 0;
3288         stp->sd_anchor = 0;
3289         stp->sd_struiowrq = NULL;
3290         stp->sd_struiordq = NULL;
3291         stp->sd_struiodnak = 0;
3292         stp->sd_struionak = NULL;
3293         stp->sd_t_audit_data = NULL;
3294         stp->sd_rput_opt = 0;
3295         stp->sd_wput_opt = 0;
3296         stp->sd_read_opt = 0;
3297         stp->sd_rprotofunc = strrput_proto;
3298         stp->sd_rmiscfunc = strrput_misc;
3299         stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3300         stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3301         stp->sd_ciputctrl = NULL;
3302         stp->sd_nciputctrl = 0;
3303         stp->sd_qhead = NULL;
3304         stp->sd_qtail = NULL;
3305         stp->sd_servid = NULL;
3306         stp->sd_nqueues = 0;
3307         stp->sd_svcflags = 0;
3308         stp->sd_copyflag = 0;
3309 
3310         return (stp);
3311 }
3312 
3313 /*
3314  * Free a stream head.
3315  */
3316 void
3317 shfree(stdata_t *stp)
3318 {
3319         ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3320 
3321         stp->sd_wrq = NULL;
3322 
3323         mutex_enter(&stp->sd_qlock);
3324         while (stp->sd_svcflags & STRS_SCHEDULED) {
3325                 STRSTAT(strwaits);
3326                 cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3327         }
3328         mutex_exit(&stp->sd_qlock);
3329 
3330         if (stp->sd_ciputctrl != NULL) {
3331                 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3332                 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3333                     stp->sd_nciputctrl, 0);
3334                 ASSERT(ciputctrl_cache != NULL);
3335                 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3336                 stp->sd_ciputctrl = NULL;
3337                 stp->sd_nciputctrl = 0;
3338         }
3339         ASSERT(stp->sd_qhead == NULL);
3340         ASSERT(stp->sd_qtail == NULL);
3341         ASSERT(stp->sd_nqueues == 0);
3342         kmem_cache_free(stream_head_cache, stp);
3343 }
3344 
3345 /*
3346  * Allocate a pair of queues and a syncq for the pair
3347  */
3348 queue_t *
3349 allocq(void)
3350 {
3351         queinfo_t *qip;
3352         queue_t *qp, *wqp;
3353         syncq_t *sq;
3354 
3355         qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3356 
3357         qp = &qip->qu_rqueue;
3358         wqp = &qip->qu_wqueue;
3359         sq = &qip->qu_syncq;
3360 
3361         qp->q_last   = NULL;
3362         qp->q_next   = NULL;
3363         qp->q_ptr    = NULL;
3364         qp->q_flag   = QUSE | QREADR;
3365         qp->q_bandp  = NULL;
3366         qp->q_stream = NULL;
3367         qp->q_syncq  = sq;
3368         qp->q_nband  = 0;
3369         qp->q_nfsrv  = NULL;
3370         qp->q_draining       = 0;
3371         qp->q_syncqmsgs      = 0;
3372         qp->q_spri   = 0;
3373         qp->q_qtstamp        = 0;
3374         qp->q_sqtstamp       = 0;
3375         qp->q_fp     = NULL;
3376 
3377         wqp->q_last  = NULL;
3378         wqp->q_next  = NULL;
3379         wqp->q_ptr   = NULL;
3380         wqp->q_flag  = QUSE;
3381         wqp->q_bandp = NULL;
3382         wqp->q_stream        = NULL;
3383         wqp->q_syncq = sq;
3384         wqp->q_nband = 0;
3385         wqp->q_nfsrv = NULL;
3386         wqp->q_draining      = 0;
3387         wqp->q_syncqmsgs = 0;
3388         wqp->q_qtstamp       = 0;
3389         wqp->q_sqtstamp      = 0;
3390         wqp->q_spri  = 0;
3391 
3392         sq->sq_count = 0;
3393         sq->sq_rmqcount      = 0;
3394         sq->sq_flags = 0;
3395         sq->sq_type  = 0;
3396         sq->sq_callbflags = 0;
3397         sq->sq_cancelid      = 0;
3398         sq->sq_ciputctrl = NULL;
3399         sq->sq_nciputctrl = 0;
3400         sq->sq_needexcl = 0;
3401         sq->sq_svcflags = 0;
3402 
3403         return (qp);
3404 }
3405 
3406 /*
3407  * Free a pair of queues and the "attached" syncq.
3408  * Discard any messages left on the syncq(s), remove the syncq(s) from the
3409  * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3410  */
3411 void
3412 freeq(queue_t *qp)
3413 {
3414         qband_t *qbp, *nqbp;
3415         syncq_t *sq, *outer;
3416         queue_t *wqp = _WR(qp);
3417 
3418         ASSERT(qp->q_flag & QREADR);
3419 
3420         /*
3421          * If a previously dispatched taskq job is scheduled to run
3422          * sync_service() or a service routine is scheduled for the
3423          * queues about to be freed, wait here until all service is
3424          * done on the queue and all associated queues and syncqs.
3425          */
3426         wait_svc(qp);
3427 
3428         (void) flush_syncq(qp->q_syncq, qp);
3429         (void) flush_syncq(wqp->q_syncq, wqp);
3430         ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3431 
3432         /*
3433          * Flush the queues before q_next is set to NULL This is needed
3434          * in order to backenable any downstream queue before we go away.
3435          * Note: we are already removed from the stream so that the
3436          * backenabling will not cause any messages to be delivered to our
3437          * put procedures.
3438          */
3439         flushq(qp, FLUSHALL);
3440         flushq(wqp, FLUSHALL);
3441 
3442         /* Tidy up - removeq only does a half-remove from stream */
3443         qp->q_next = wqp->q_next = NULL;
3444         ASSERT(!(qp->q_flag & QENAB));
3445         ASSERT(!(wqp->q_flag & QENAB));
3446 
3447         outer = qp->q_syncq->sq_outer;
3448         if (outer != NULL) {
3449                 outer_remove(outer, qp->q_syncq);
3450                 if (wqp->q_syncq != qp->q_syncq)
3451                         outer_remove(outer, wqp->q_syncq);
3452         }
3453         /*
3454          * Free any syncqs that are outside what allocq returned.
3455          */
3456         if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3457                 free_syncq(qp->q_syncq);
3458         if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3459                 free_syncq(wqp->q_syncq);
3460 
3461         ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3462         ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3463         ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3464         ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3465         sq = SQ(qp);
3466         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3467         ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3468         ASSERT(sq->sq_outer == NULL);
3469         ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3470         ASSERT(sq->sq_callbpend == NULL);
3471         ASSERT(sq->sq_needexcl == 0);
3472 
3473         if (sq->sq_ciputctrl != NULL) {
3474                 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3475                 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3476                     sq->sq_nciputctrl, 0);
3477                 ASSERT(ciputctrl_cache != NULL);
3478                 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3479                 sq->sq_ciputctrl = NULL;
3480                 sq->sq_nciputctrl = 0;
3481         }
3482 
3483         ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3484         ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3485         ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3486 
3487         qp->q_flag &= ~QUSE;
3488         wqp->q_flag &= ~QUSE;
3489 
3490         /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3491         /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3492 
3493         qbp = qp->q_bandp;
3494         while (qbp) {
3495                 nqbp = qbp->qb_next;
3496                 freeband(qbp);
3497                 qbp = nqbp;
3498         }
3499         qbp = wqp->q_bandp;
3500         while (qbp) {
3501                 nqbp = qbp->qb_next;
3502                 freeband(qbp);
3503                 qbp = nqbp;
3504         }
3505         kmem_cache_free(queue_cache, qp);
3506 }
3507 
3508 /*
3509  * Allocate a qband structure.
3510  */
3511 qband_t *
3512 allocband(void)
3513 {
3514         qband_t *qbp;
3515 
3516         qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3517         if (qbp == NULL)
3518                 return (NULL);
3519 
3520         qbp->qb_next = NULL;
3521         qbp->qb_count        = 0;
3522         qbp->qb_mblkcnt      = 0;
3523         qbp->qb_first        = NULL;
3524         qbp->qb_last = NULL;
3525         qbp->qb_flag = 0;
3526 
3527         return (qbp);
3528 }
3529 
3530 /*
3531  * Free a qband structure.
3532  */
3533 void
3534 freeband(qband_t *qbp)
3535 {
3536         kmem_cache_free(qband_cache, qbp);
3537 }
3538 
3539 /*
3540  * Just like putnextctl(9F), except that allocb_wait() is used.
3541  *
3542  * Consolidation Private, and of course only callable from the stream head or
3543  * routines that may block.
3544  */
3545 int
3546 putnextctl_wait(queue_t *q, int type)
3547 {
3548         mblk_t *bp;
3549         int error;
3550 
3551         if ((datamsg(type) && (type != M_DELAY)) ||
3552             (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3553                 return (0);
3554 
3555         bp->b_datap->db_type = (unsigned char)type;
3556         putnext(q, bp);
3557         return (1);
3558 }
3559 
3560 /*
3561  * Run any possible bufcalls.
3562  */
3563 void
3564 runbufcalls(void)
3565 {
3566         strbufcall_t *bcp;
3567 
3568         mutex_enter(&bcall_monitor);
3569         mutex_enter(&strbcall_lock);
3570 
3571         if (strbcalls.bc_head) {
3572                 size_t count;
3573                 int nevent;
3574 
3575                 /*
3576                  * count how many events are on the list
3577                  * now so we can check to avoid looping
3578                  * in low memory situations
3579                  */
3580                 nevent = 0;
3581                 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3582                         nevent++;
3583 
3584                 /*
3585                  * get estimate of available memory from kmem_avail().
3586                  * awake all bufcall functions waiting for
3587                  * memory whose request could be satisfied
3588                  * by 'count' memory and let 'em fight for it.
3589                  */
3590                 count = kmem_avail();
3591                 while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3592                         STRSTAT(bufcalls);
3593                         --nevent;
3594                         if (bcp->bc_size <= count) {
3595                                 bcp->bc_executor = curthread;
3596                                 mutex_exit(&strbcall_lock);
3597                                 (*bcp->bc_func)(bcp->bc_arg);
3598                                 mutex_enter(&strbcall_lock);
3599                                 bcp->bc_executor = NULL;
3600                                 cv_broadcast(&bcall_cv);
3601                                 strbcalls.bc_head = bcp->bc_next;
3602                                 kmem_free(bcp, sizeof (strbufcall_t));
3603                         } else {
3604                                 /*
3605                                  * too big, try again later - note
3606                                  * that nevent was decremented above
3607                                  * so we won't retry this one on this
3608                                  * iteration of the loop
3609                                  */
3610                                 if (bcp->bc_next != NULL) {
3611                                         strbcalls.bc_head = bcp->bc_next;
3612                                         bcp->bc_next = NULL;
3613                                         strbcalls.bc_tail->bc_next = bcp;
3614                                         strbcalls.bc_tail = bcp;
3615                                 }
3616                         }
3617                 }
3618                 if (strbcalls.bc_head == NULL)
3619                         strbcalls.bc_tail = NULL;
3620         }
3621 
3622         mutex_exit(&strbcall_lock);
3623         mutex_exit(&bcall_monitor);
3624 }
3625 
3626 
3627 /*
3628  * Actually run queue's service routine.
3629  */
3630 static void
3631 runservice(queue_t *q)
3632 {
3633         qband_t *qbp;
3634 
3635         ASSERT(q->q_qinfo->qi_srvp);
3636 again:
3637         entersq(q->q_syncq, SQ_SVC);
3638         TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3639             "runservice starts:%p", q);
3640 
3641         if (!(q->q_flag & QWCLOSE))
3642                 (*q->q_qinfo->qi_srvp)(q);
3643 
3644         TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3645             "runservice ends:(%p)", q);
3646 
3647         leavesq(q->q_syncq, SQ_SVC);
3648 
3649         mutex_enter(QLOCK(q));
3650         if (q->q_flag & QENAB) {
3651                 q->q_flag &= ~QENAB;
3652                 mutex_exit(QLOCK(q));
3653                 goto again;
3654         }
3655         q->q_flag &= ~QINSERVICE;
3656         q->q_flag &= ~QBACK;
3657         for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3658                 qbp->qb_flag &= ~QB_BACK;
3659         /*
3660          * Wakeup thread waiting for the service procedure
3661          * to be run (strclose and qdetach).
3662          */
3663         cv_broadcast(&q->q_wait);
3664 
3665         mutex_exit(QLOCK(q));
3666 }
3667 
3668 /*
3669  * Background processing of bufcalls.
3670  */
3671 void
3672 streams_bufcall_service(void)
3673 {
3674         callb_cpr_t     cprinfo;
3675 
3676         CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3677             "streams_bufcall_service");
3678 
3679         mutex_enter(&strbcall_lock);
3680 
3681         for (;;) {
3682                 if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3683                         mutex_exit(&strbcall_lock);
3684                         runbufcalls();
3685                         mutex_enter(&strbcall_lock);
3686                 }
3687                 if (strbcalls.bc_head != NULL) {
3688                         STRSTAT(bcwaits);
3689                         /* Wait for memory to become available */
3690                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3691                         (void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3692                             SEC_TO_TICK(60), TR_CLOCK_TICK);
3693                         CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3694                 }
3695 
3696                 /* Wait for new work to arrive */
3697                 if (strbcalls.bc_head == NULL) {
3698                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3699                         cv_wait(&strbcall_cv, &strbcall_lock);
3700                         CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3701                 }
3702         }
3703 }
3704 
3705 /*
3706  * Background processing of streams background tasks which failed
3707  * taskq_dispatch.
3708  */
3709 static void
3710 streams_qbkgrnd_service(void)
3711 {
3712         callb_cpr_t cprinfo;
3713         queue_t *q;
3714 
3715         CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3716             "streams_bkgrnd_service");
3717 
3718         mutex_enter(&service_queue);
3719 
3720         for (;;) {
3721                 /*
3722                  * Wait for work to arrive.
3723                  */
3724                 while ((freebs_list == NULL) && (qhead == NULL)) {
3725                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3726                         cv_wait(&services_to_run, &service_queue);
3727                         CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3728                 }
3729                 /*
3730                  * Handle all pending freebs requests to free memory.
3731                  */
3732                 while (freebs_list != NULL) {
3733                         mblk_t *mp = freebs_list;
3734                         freebs_list = mp->b_next;
3735                         mutex_exit(&service_queue);
3736                         mblk_free(mp);
3737                         mutex_enter(&service_queue);
3738                 }
3739                 /*
3740                  * Run pending queues.
3741                  */
3742                 while (qhead != NULL) {
3743                         DQ(q, qhead, qtail, q_link);
3744                         ASSERT(q != NULL);
3745                         mutex_exit(&service_queue);
3746                         queue_service(q);
3747                         mutex_enter(&service_queue);
3748                 }
3749                 ASSERT(qhead == NULL && qtail == NULL);
3750         }
3751 }
3752 
3753 /*
3754  * Background processing of streams background tasks which failed
3755  * taskq_dispatch.
3756  */
3757 static void
3758 streams_sqbkgrnd_service(void)
3759 {
3760         callb_cpr_t cprinfo;
3761         syncq_t *sq;
3762 
3763         CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3764             "streams_sqbkgrnd_service");
3765 
3766         mutex_enter(&service_queue);
3767 
3768         for (;;) {
3769                 /*
3770                  * Wait for work to arrive.
3771                  */
3772                 while (sqhead == NULL) {
3773                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3774                         cv_wait(&syncqs_to_run, &service_queue);
3775                         CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3776                 }
3777 
3778                 /*
3779                  * Run pending syncqs.
3780                  */
3781                 while (sqhead != NULL) {
3782                         DQ(sq, sqhead, sqtail, sq_next);
3783                         ASSERT(sq != NULL);
3784                         ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3785                         mutex_exit(&service_queue);
3786                         syncq_service(sq);
3787                         mutex_enter(&service_queue);
3788                 }
3789         }
3790 }
3791 
3792 /*
3793  * Disable the syncq and wait for background syncq processing to complete.
3794  * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3795  * list.
3796  */
3797 void
3798 wait_sq_svc(syncq_t *sq)
3799 {
3800         mutex_enter(SQLOCK(sq));
3801         sq->sq_svcflags |= SQ_DISABLED;
3802         if (sq->sq_svcflags & SQ_BGTHREAD) {
3803                 syncq_t *sq_chase;
3804                 syncq_t *sq_curr;
3805                 int removed;
3806 
3807                 ASSERT(sq->sq_servcount == 1);
3808                 mutex_enter(&service_queue);
3809                 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3810                 mutex_exit(&service_queue);
3811                 if (removed) {
3812                         sq->sq_svcflags &= ~SQ_BGTHREAD;
3813                         sq->sq_servcount = 0;
3814                         STRSTAT(sqremoved);
3815                         goto done;
3816                 }
3817         }
3818         while (sq->sq_servcount != 0) {
3819                 sq->sq_flags |= SQ_WANTWAKEUP;
3820                 cv_wait(&sq->sq_wait, SQLOCK(sq));
3821         }
3822 done:
3823         mutex_exit(SQLOCK(sq));
3824 }
3825 
3826 /*
3827  * Put a syncq on the list of syncq's to be serviced by the sqthread.
3828  * Add the argument to the end of the sqhead list and set the flag
3829  * indicating this syncq has been enabled.  If it has already been
3830  * enabled, don't do anything.
3831  * This routine assumes that SQLOCK is held.
3832  * NOTE that the lock order is to have the SQLOCK first,
3833  * so if the service_syncq lock is held, we need to release it
3834  * before acquiring the SQLOCK (mostly relevant for the background
3835  * thread, and this seems to be common among the STREAMS global locks).
3836  * Note that the sq_svcflags are protected by the SQLOCK.
3837  */
3838 void
3839 sqenable(syncq_t *sq)
3840 {
3841         /*
3842          * This is probably not important except for where I believe it
3843          * is being called.  At that point, it should be held (and it
3844          * is a pain to release it just for this routine, so don't do
3845          * it).
3846          */
3847         ASSERT(MUTEX_HELD(SQLOCK(sq)));
3848 
3849         IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3850         IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3851 
3852         /*
3853          * Do not put on list if background thread is scheduled or
3854          * syncq is disabled.
3855          */
3856         if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3857                 return;
3858 
3859         /*
3860          * Check whether we should enable sq at all.
3861          * Non PERMOD syncqs may be drained by at most one thread.
3862          * PERMOD syncqs may be drained by several threads but we limit the
3863          * total amount to the lesser of
3864          *      Number of queues on the squeue and
3865          *      Number of CPUs.
3866          */
3867         if (sq->sq_servcount != 0) {
3868                 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3869                     (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3870                         STRSTAT(sqtoomany);
3871                         return;
3872                 }
3873         }
3874 
3875         sq->sq_tstamp = ddi_get_lbolt();
3876         STRSTAT(sqenables);
3877 
3878         /* Attempt a taskq dispatch */
3879         sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3880             (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3881         if (sq->sq_servid != NULL) {
3882                 sq->sq_servcount++;
3883                 return;
3884         }
3885 
3886         /*
3887          * This taskq dispatch failed, but a previous one may have succeeded.
3888          * Don't try to schedule on the background thread whilst there is
3889          * outstanding taskq processing.
3890          */
3891         if (sq->sq_servcount != 0)
3892                 return;
3893 
3894         /*
3895          * System is low on resources and can't perform a non-sleeping
3896          * dispatch. Schedule the syncq for a background thread and mark the
3897          * syncq to avoid any further taskq dispatch attempts.
3898          */
3899         mutex_enter(&service_queue);
3900         STRSTAT(taskqfails);
3901         ENQUEUE(sq, sqhead, sqtail, sq_next);
3902         sq->sq_svcflags |= SQ_BGTHREAD;
3903         sq->sq_servcount = 1;
3904         cv_signal(&syncqs_to_run);
3905         mutex_exit(&service_queue);
3906 }
3907 
3908 /*
3909  * Note: fifo_close() depends on the mblk_t on the queue being freed
3910  * asynchronously. The asynchronous freeing of messages breaks the
3911  * recursive call chain of fifo_close() while there are I_SENDFD type of
3912  * messages referring to other file pointers on the queue. Then when
3913  * closing pipes it can avoid stack overflow in case of daisy-chained
3914  * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3915  * share the same fifolock_t).
3916  *
3917  * No need to kpreempt_disable to access cpu_seqid.  If we migrate and
3918  * the esb queue does not match the new CPU, that is OK.
3919  */
3920 void
3921 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3922 {
3923         int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3924         esb_queue_t *eqp;
3925 
3926         ASSERT(dbp->db_mblk == mp);
3927         ASSERT(qindex < esbq_nelem);
3928 
3929         eqp = system_esbq_array;
3930         if (eqp != NULL) {
3931                 eqp += qindex;
3932         } else {
3933                 mutex_enter(&esbq_lock);
3934                 if (kmem_ready && system_esbq_array == NULL)
3935                         system_esbq_array = (esb_queue_t *)kmem_zalloc(
3936                             esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3937                 mutex_exit(&esbq_lock);
3938                 eqp = system_esbq_array;
3939                 if (eqp != NULL)
3940                         eqp += qindex;
3941                 else
3942                         eqp = &system_esbq;
3943         }
3944 
3945         /*
3946          * Check data sanity. The dblock should have non-empty free function.
3947          * It is better to panic here then later when the dblock is freed
3948          * asynchronously when the context is lost.
3949          */
3950         if (dbp->db_frtnp->free_func == NULL) {
3951                 panic("freebs_enqueue: dblock %p has a NULL free callback",
3952                     (void *)dbp);
3953         }
3954 
3955         mutex_enter(&eqp->eq_lock);
3956         /* queue the new mblk on the esballoc queue */
3957         if (eqp->eq_head == NULL) {
3958                 eqp->eq_head = eqp->eq_tail = mp;
3959         } else {
3960                 eqp->eq_tail->b_next = mp;
3961                 eqp->eq_tail = mp;
3962         }
3963         eqp->eq_len++;
3964 
3965         /* If we're the first thread to reach the threshold, process */
3966         if (eqp->eq_len >= esbq_max_qlen &&
3967             !(eqp->eq_flags & ESBQ_PROCESSING))
3968                 esballoc_process_queue(eqp);
3969 
3970         esballoc_set_timer(eqp, esbq_timeout);
3971         mutex_exit(&eqp->eq_lock);
3972 }
3973 
3974 static void
3975 esballoc_process_queue(esb_queue_t *eqp)
3976 {
3977         mblk_t  *mp;
3978 
3979         ASSERT(MUTEX_HELD(&eqp->eq_lock));
3980 
3981         eqp->eq_flags |= ESBQ_PROCESSING;
3982 
3983         do {
3984                 /*
3985                  * Detach the message chain for processing.
3986                  */
3987                 mp = eqp->eq_head;
3988                 eqp->eq_tail->b_next = NULL;
3989                 eqp->eq_head = eqp->eq_tail = NULL;
3990                 eqp->eq_len = 0;
3991                 mutex_exit(&eqp->eq_lock);
3992 
3993                 /*
3994                  * Process the message chain.
3995                  */
3996                 esballoc_enqueue_mblk(mp);
3997                 mutex_enter(&eqp->eq_lock);
3998         } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
3999 
4000         eqp->eq_flags &= ~ESBQ_PROCESSING;
4001 }
4002 
4003 /*
4004  * taskq callback routine to free esballoced mblk's
4005  */
4006 static void
4007 esballoc_mblk_free(mblk_t *mp)
4008 {
4009         mblk_t  *nextmp;
4010 
4011         for (; mp != NULL; mp = nextmp) {
4012                 nextmp = mp->b_next;
4013                 mp->b_next = NULL;
4014                 mblk_free(mp);
4015         }
4016 }
4017 
4018 static void
4019 esballoc_enqueue_mblk(mblk_t *mp)
4020 {
4021 
4022         if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4023             TQ_NOSLEEP) == NULL) {
4024                 mblk_t *first_mp = mp;
4025                 /*
4026                  * System is low on resources and can't perform a non-sleeping
4027                  * dispatch. Schedule for a background thread.
4028                  */
4029                 mutex_enter(&service_queue);
4030                 STRSTAT(taskqfails);
4031 
4032                 while (mp->b_next != NULL)
4033                         mp = mp->b_next;
4034 
4035                 mp->b_next = freebs_list;
4036                 freebs_list = first_mp;
4037                 cv_signal(&services_to_run);
4038                 mutex_exit(&service_queue);
4039         }
4040 }
4041 
4042 static void
4043 esballoc_timer(void *arg)
4044 {
4045         esb_queue_t *eqp = arg;
4046 
4047         mutex_enter(&eqp->eq_lock);
4048         eqp->eq_flags &= ~ESBQ_TIMER;
4049 
4050         if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4051             eqp->eq_len > 0)
4052                 esballoc_process_queue(eqp);
4053 
4054         esballoc_set_timer(eqp, esbq_timeout);
4055         mutex_exit(&eqp->eq_lock);
4056 }
4057 
4058 static void
4059 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4060 {
4061         ASSERT(MUTEX_HELD(&eqp->eq_lock));
4062 
4063         if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4064                 (void) timeout(esballoc_timer, eqp, eq_timeout);
4065                 eqp->eq_flags |= ESBQ_TIMER;
4066         }
4067 }
4068 
4069 /*
4070  * Setup esbq array length based upon NCPU scaled by CPUs per
4071  * queue. Use static system_esbq until kmem_ready and we can
4072  * create an array in freebs_enqueue().
4073  */
4074 void
4075 esballoc_queue_init(void)
4076 {
4077         esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4078         esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4079         esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4080         system_esbq.eq_len = 0;
4081         system_esbq.eq_head = system_esbq.eq_tail = NULL;
4082         system_esbq.eq_flags = 0;
4083 }
4084 
4085 /*
4086  * Set the QBACK or QB_BACK flag in the given queue for
4087  * the given priority band.
4088  */
4089 void
4090 setqback(queue_t *q, unsigned char pri)
4091 {
4092         int i;
4093         qband_t *qbp;
4094         qband_t **qbpp;
4095 
4096         ASSERT(MUTEX_HELD(QLOCK(q)));
4097         if (pri != 0) {
4098                 if (pri > q->q_nband) {
4099                         qbpp = &q->q_bandp;
4100                         while (*qbpp)
4101                                 qbpp = &(*qbpp)->qb_next;
4102                         while (pri > q->q_nband) {
4103                                 if ((*qbpp = allocband()) == NULL) {
4104                                         cmn_err(CE_WARN,
4105                                             "setqback: can't allocate qband\n");
4106                                         return;
4107                                 }
4108                                 (*qbpp)->qb_hiwat = q->q_hiwat;
4109                                 (*qbpp)->qb_lowat = q->q_lowat;
4110                                 q->q_nband++;
4111                                 qbpp = &(*qbpp)->qb_next;
4112                         }
4113                 }
4114                 qbp = q->q_bandp;
4115                 i = pri;
4116                 while (--i)
4117                         qbp = qbp->qb_next;
4118                 qbp->qb_flag |= QB_BACK;
4119         } else {
4120                 q->q_flag |= QBACK;
4121         }
4122 }
4123 
4124 int
4125 strcopyin(void *from, void *to, size_t len, int copyflag)
4126 {
4127         if (copyflag & U_TO_K) {
4128                 ASSERT((copyflag & K_TO_K) == 0);
4129                 if (copyin(from, to, len))
4130                         return (EFAULT);
4131         } else {
4132                 ASSERT(copyflag & K_TO_K);
4133                 bcopy(from, to, len);
4134         }
4135         return (0);
4136 }
4137 
4138 int
4139 strcopyout(void *from, void *to, size_t len, int copyflag)
4140 {
4141         if (copyflag & U_TO_K) {
4142                 if (copyout(from, to, len))
4143                         return (EFAULT);
4144         } else {
4145                 ASSERT(copyflag & K_TO_K);
4146                 bcopy(from, to, len);
4147         }
4148         return (0);
4149 }
4150 
4151 /*
4152  * strsignal_nolock() posts a signal to the process(es) at the stream head.
4153  * It assumes that the stream head lock is already held, whereas strsignal()
4154  * acquires the lock first.  This routine was created because a few callers
4155  * release the stream head lock before calling only to re-acquire it after
4156  * it returns.
4157  */
4158 void
4159 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4160 {
4161         ASSERT(MUTEX_HELD(&stp->sd_lock));
4162         switch (sig) {
4163         case SIGPOLL:
4164                 if (stp->sd_sigflags & S_MSG)
4165                         strsendsig(stp->sd_siglist, S_MSG, band, 0);
4166                 break;
4167         default:
4168                 if (stp->sd_pgidp)
4169                         pgsignal(stp->sd_pgidp, sig);
4170                 break;
4171         }
4172 }
4173 
4174 void
4175 strsignal(stdata_t *stp, int sig, int32_t band)
4176 {
4177         TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4178             "strsignal:%p, %X, %X", stp, sig, band);
4179 
4180         mutex_enter(&stp->sd_lock);
4181         switch (sig) {
4182         case SIGPOLL:
4183                 if (stp->sd_sigflags & S_MSG)
4184                         strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4185                 break;
4186 
4187         default:
4188                 if (stp->sd_pgidp) {
4189                         pgsignal(stp->sd_pgidp, sig);
4190                 }
4191                 break;
4192         }
4193         mutex_exit(&stp->sd_lock);
4194 }
4195 
4196 void
4197 strhup(stdata_t *stp)
4198 {
4199         ASSERT(mutex_owned(&stp->sd_lock));
4200         pollwakeup(&stp->sd_pollist, POLLHUP);
4201         if (stp->sd_sigflags & S_HANGUP)
4202                 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4203 }
4204 
4205 /*
4206  * Backenable the first queue upstream from `q' with a service procedure.
4207  */
4208 void
4209 backenable(queue_t *q, uchar_t pri)
4210 {
4211         queue_t *nq;
4212 
4213         /*
4214          * Our presence might not prevent other modules in our own
4215          * stream from popping/pushing since the caller of getq might not
4216          * have a claim on the queue (some drivers do a getq on somebody
4217          * else's queue - they know that the queue itself is not going away
4218          * but the framework has to guarantee q_next in that stream).
4219          */
4220         claimstr(q);
4221 
4222         /* Find nearest back queue with service proc */
4223         for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4224                 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4225         }
4226 
4227         if (nq) {
4228                 kthread_t *freezer;
4229                 /*
4230                  * backenable can be called either with no locks held
4231                  * or with the stream frozen (the latter occurs when a module
4232                  * calls rmvq with the stream frozen). If the stream is frozen
4233                  * by the caller the caller will hold all qlocks in the stream.
4234                  * Note that a frozen stream doesn't freeze a mated stream,
4235                  * so we explicitly check for that.
4236                  */
4237                 freezer = STREAM(q)->sd_freezer;
4238                 if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4239                         mutex_enter(QLOCK(nq));
4240                 }
4241 #ifdef DEBUG
4242                 else {
4243                         ASSERT(frozenstr(q));
4244                         ASSERT(MUTEX_HELD(QLOCK(q)));
4245                         ASSERT(MUTEX_HELD(QLOCK(nq)));
4246                 }
4247 #endif
4248                 setqback(nq, pri);
4249                 qenable_locked(nq);
4250                 if (freezer != curthread || STREAM(q) != STREAM(nq))
4251                         mutex_exit(QLOCK(nq));
4252         }
4253         releasestr(q);
4254 }
4255 
4256 /*
4257  * Return the appropriate errno when one of flags_to_check is set
4258  * in sd_flags. Uses the exported error routines if they are set.
4259  * Will return 0 if non error is set (or if the exported error routines
4260  * do not return an error).
4261  *
4262  * If there is both a read and write error to check, we prefer the read error.
4263  * Also, give preference to recorded errno's over the error functions.
4264  * The flags that are handled are:
4265  *      STPLEX          return EINVAL
4266  *      STRDERR         return sd_rerror (and clear if STRDERRNONPERSIST)
4267  *      STWRERR         return sd_werror (and clear if STWRERRNONPERSIST)
4268  *      STRHUP          return sd_werror
4269  *
4270  * If the caller indicates that the operation is a peek, a nonpersistent error
4271  * is not cleared.
4272  */
4273 int
4274 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4275 {
4276         int32_t sd_flag = stp->sd_flag & flags_to_check;
4277         int error = 0;
4278 
4279         ASSERT(MUTEX_HELD(&stp->sd_lock));
4280         ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4281         if (sd_flag & STPLEX)
4282                 error = EINVAL;
4283         else if (sd_flag & STRDERR) {
4284                 error = stp->sd_rerror;
4285                 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4286                         /*
4287                          * Read errors are non-persistent i.e. discarded once
4288                          * returned to a non-peeking caller,
4289                          */
4290                         stp->sd_rerror = 0;
4291                         stp->sd_flag &= ~STRDERR;
4292                 }
4293                 if (error == 0 && stp->sd_rderrfunc != NULL) {
4294                         int clearerr = 0;
4295 
4296                         error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4297                             &clearerr);
4298                         if (clearerr) {
4299                                 stp->sd_flag &= ~STRDERR;
4300                                 stp->sd_rderrfunc = NULL;
4301                         }
4302                 }
4303         } else if (sd_flag & STWRERR) {
4304                 error = stp->sd_werror;
4305                 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4306                         /*
4307                          * Write errors are non-persistent i.e. discarded once
4308                          * returned to a non-peeking caller,
4309                          */
4310                         stp->sd_werror = 0;
4311                         stp->sd_flag &= ~STWRERR;
4312                 }
4313                 if (error == 0 && stp->sd_wrerrfunc != NULL) {
4314                         int clearerr = 0;
4315 
4316                         error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4317                             &clearerr);
4318                         if (clearerr) {
4319                                 stp->sd_flag &= ~STWRERR;
4320                                 stp->sd_wrerrfunc = NULL;
4321                         }
4322                 }
4323         } else if (sd_flag & STRHUP) {
4324                 /* sd_werror set when STRHUP */
4325                 error = stp->sd_werror;
4326         }
4327         return (error);
4328 }
4329 
4330 
4331 /*
4332  * Single-thread open/close/push/pop
4333  * for twisted streams also
4334  */
4335 int
4336 strstartplumb(stdata_t *stp, int flag, int cmd)
4337 {
4338         int waited = 1;
4339         int error = 0;
4340 
4341         if (STRMATED(stp)) {
4342                 struct stdata *stmatep = stp->sd_mate;
4343 
4344                 STRLOCKMATES(stp);
4345                 while (waited) {
4346                         waited = 0;
4347                         while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4348                                 if ((cmd == I_POP) &&
4349                                     (flag & (FNDELAY|FNONBLOCK))) {
4350                                         STRUNLOCKMATES(stp);
4351                                         return (EAGAIN);
4352                                 }
4353                                 waited = 1;
4354                                 mutex_exit(&stp->sd_lock);
4355                                 if (!cv_wait_sig(&stmatep->sd_monitor,
4356                                     &stmatep->sd_lock)) {
4357                                         mutex_exit(&stmatep->sd_lock);
4358                                         return (EINTR);
4359                                 }
4360                                 mutex_exit(&stmatep->sd_lock);
4361                                 STRLOCKMATES(stp);
4362                         }
4363                         while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4364                                 if ((cmd == I_POP) &&
4365                                     (flag & (FNDELAY|FNONBLOCK))) {
4366                                         STRUNLOCKMATES(stp);
4367                                         return (EAGAIN);
4368                                 }
4369                                 waited = 1;
4370                                 mutex_exit(&stmatep->sd_lock);
4371                                 if (!cv_wait_sig(&stp->sd_monitor,
4372                                     &stp->sd_lock)) {
4373                                         mutex_exit(&stp->sd_lock);
4374                                         return (EINTR);
4375                                 }
4376                                 mutex_exit(&stp->sd_lock);
4377                                 STRLOCKMATES(stp);
4378                         }
4379                         if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4380                                 error = strgeterr(stp,
4381                                     STRDERR|STWRERR|STRHUP|STPLEX, 0);
4382                                 if (error != 0) {
4383                                         STRUNLOCKMATES(stp);
4384                                         return (error);
4385                                 }
4386                         }
4387                 }
4388                 stp->sd_flag |= STRPLUMB;
4389                 STRUNLOCKMATES(stp);
4390         } else {
4391                 mutex_enter(&stp->sd_lock);
4392                 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4393                         if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4394                             (flag & (FNDELAY|FNONBLOCK))) {
4395                                 mutex_exit(&stp->sd_lock);
4396                                 return (EAGAIN);
4397                         }
4398                         if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4399                                 mutex_exit(&stp->sd_lock);
4400                                 return (EINTR);
4401                         }
4402                         if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4403                                 error = strgeterr(stp,
4404                                     STRDERR|STWRERR|STRHUP|STPLEX, 0);
4405                                 if (error != 0) {
4406                                         mutex_exit(&stp->sd_lock);
4407                                         return (error);
4408                                 }
4409                         }
4410                 }
4411                 stp->sd_flag |= STRPLUMB;
4412                 mutex_exit(&stp->sd_lock);
4413         }
4414         return (0);
4415 }
4416 
4417 /*
4418  * Complete the plumbing operation associated with stream `stp'.
4419  */
4420 void
4421 strendplumb(stdata_t *stp)
4422 {
4423         ASSERT(MUTEX_HELD(&stp->sd_lock));
4424         ASSERT(stp->sd_flag & STRPLUMB);
4425         stp->sd_flag &= ~STRPLUMB;
4426         cv_broadcast(&stp->sd_monitor);
4427 }
4428 
4429 /*
4430  * This describes how the STREAMS framework handles synchronization
4431  * during open/push and close/pop.
4432  * The key interfaces for open and close are qprocson and qprocsoff,
4433  * respectively. While the close case in general is harder both open
4434  * have close have significant similarities.
4435  *
4436  * During close the STREAMS framework has to both ensure that there
4437  * are no stale references to the queue pair (and syncq) that
4438  * are being closed and also provide the guarantees that are documented
4439  * in qprocsoff(9F).
4440  * If there are stale references to the queue that is closing it can
4441  * result in kernel memory corruption or kernel panics.
4442  *
4443  * Note that is it up to the module/driver to ensure that it itself
4444  * does not have any stale references to the closing queues once its close
4445  * routine returns. This includes:
4446  *  - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4447  *    associated with the queues. For timeout and bufcall callbacks the
4448  *    module/driver also has to ensure (or wait for) any callbacks that
4449  *    are in progress.
4450  *  - If the module/driver is using esballoc it has to ensure that any
4451  *    esballoc free functions do not refer to a queue that has closed.
4452  *    (Note that in general the close routine can not wait for the esballoc'ed
4453  *    messages to be freed since that can cause a deadlock.)
4454  *  - Cancelling any interrupts that refer to the closing queues and
4455  *    also ensuring that there are no interrupts in progress that will
4456  *    refer to the closing queues once the close routine returns.
4457  *  - For multiplexors removing any driver global state that refers to
4458  *    the closing queue and also ensuring that there are no threads in
4459  *    the multiplexor that has picked up a queue pointer but not yet
4460  *    finished using it.
4461  *
4462  * In addition, a driver/module can only reference the q_next pointer
4463  * in its open, close, put, or service procedures or in a
4464  * qtimeout/qbufcall callback procedure executing "on" the correct
4465  * stream. Thus it can not reference the q_next pointer in an interrupt
4466  * routine or a timeout, bufcall or esballoc callback routine. Likewise
4467  * it can not reference q_next of a different queue e.g. in a mux that
4468  * passes messages from one queues put/service procedure to another queue.
4469  * In all the cases when the driver/module can not access the q_next
4470  * field it must use the *next* versions e.g. canputnext instead of
4471  * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4472  *
4473  *
4474  * Assuming that the driver/module conforms to the above constraints
4475  * the STREAMS framework has to avoid stale references to q_next for all
4476  * the framework internal cases which include (but are not limited to):
4477  *  - Threads in canput/canputnext/backenable and elsewhere that are
4478  *    walking q_next.
4479  *  - Messages on a syncq that have a reference to the queue through b_queue.
4480  *  - Messages on an outer perimeter (syncq) that have a reference to the
4481  *    queue through b_queue.
4482  *  - Threads that use q_nfsrv (e.g. canput) to find a queue.
4483  *    Note that only canput and bcanput use q_nfsrv without any locking.
4484  *
4485  * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4486  * after qprocsoff returns, the framework has to ensure that no threads can
4487  * enter the put or service routines for the closing read or write-side queue.
4488  * In addition to preventing "direct" entry into the put procedures
4489  * the framework also has to prevent messages being drained from
4490  * the syncq or the outer perimeter.
4491  * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4492  * mechanism to prevent qwriter(PERIM_OUTER) from running after
4493  * qprocsoff has returned.
4494  * Note that if a module/driver uses put(9F) on one of its own queues
4495  * it is up to the module/driver to ensure that the put() doesn't
4496  * get called when the queue is closing.
4497  *
4498  *
4499  * The framework aspects of the above "contract" is implemented by
4500  * qprocsoff, removeq, and strlock:
4501  *  - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4502  *    entering the service procedures.
4503  *  - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4504  *    canputnext, backenable etc from dereferencing the q_next that will
4505  *    soon change.
4506  *  - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4507  *    or other q_next walker that uses claimstr/releasestr to finish.
4508  *  - optionally for every syncq in the stream strlock acquires all the
4509  *    sq_lock's and waits for all sq_counts to drop to a value that indicates
4510  *    that no thread executes in the put or service procedures and that no
4511  *    thread is draining into the module/driver. This ensures that no
4512  *    open, close, put, service, or qtimeout/qbufcall callback procedure is
4513  *    currently executing hence no such thread can end up with the old stale
4514  *    q_next value and no canput/backenable can have the old stale
4515  *    q_nfsrv/q_next.
4516  *  - qdetach (wait_svc) makes sure that any scheduled or running threads
4517  *    have either finished or observed the QWCLOSE flag and gone away.
4518  */
4519 
4520 
4521 /*
4522  * Get all the locks necessary to change q_next.
4523  *
4524  * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4525  * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4526  * the only threads inside the syncq are threads currently calling removeq().
4527  * Since threads calling removeq() are in the process of removing their queues
4528  * from the stream, we do not need to worry about them accessing a stale q_next
4529  * pointer and thus we do not need to wait for them to exit (in fact, waiting
4530  * for them can cause deadlock).
4531  *
4532  * This routine is subject to starvation since it does not set any flag to
4533  * prevent threads from entering a module in the stream (i.e. sq_count can
4534  * increase on some syncq while it is waiting on some other syncq).
4535  *
4536  * Assumes that only one thread attempts to call strlock for a given
4537  * stream. If this is not the case the two threads would deadlock.
4538  * This assumption is guaranteed since strlock is only called by insertq
4539  * and removeq and streams plumbing changes are single-threaded for
4540  * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4541  *
4542  * For pipes, it is not difficult to atomically designate a pair of streams
4543  * to be mated. Once mated atomically by the framework the twisted pair remain
4544  * configured that way until dismantled atomically by the framework.
4545  * When plumbing takes place on a twisted stream it is necessary to ensure that
4546  * this operation is done exclusively on the twisted stream since two such
4547  * operations, each initiated on different ends of the pipe will deadlock
4548  * waiting for each other to complete.
4549  *
4550  * On entry, no locks should be held.
4551  * The locks acquired and held by strlock depends on a few factors.
4552  * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4553  *   and held on exit and all sq_count are at an acceptable level.
4554  * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4555  *   sd_refcnt being zero.
4556  */
4557 
4558 static void
4559 strlock(struct stdata *stp, sqlist_t *sqlist)
4560 {
4561         syncql_t *sql, *sql2;
4562 retry:
4563         /*
4564          * Wait for any claimstr to go away.
4565          */
4566         if (STRMATED(stp)) {
4567                 struct stdata *stp1, *stp2;
4568 
4569                 STRLOCKMATES(stp);
4570                 /*
4571                  * Note that the selection of locking order is not
4572                  * important, just that they are always acquired in
4573                  * the same order.  To assure this, we choose this
4574                  * order based on the value of the pointer, and since
4575                  * the pointer will not change for the life of this
4576                  * pair, we will always grab the locks in the same
4577                  * order (and hence, prevent deadlocks).
4578                  */
4579                 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4580                         stp1 = stp;
4581                         stp2 = stp->sd_mate;
4582                 } else {
4583                         stp2 = stp;
4584                         stp1 = stp->sd_mate;
4585                 }
4586                 mutex_enter(&stp1->sd_reflock);
4587                 if (stp1->sd_refcnt > 0) {
4588                         STRUNLOCKMATES(stp);
4589                         cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4590                         mutex_exit(&stp1->sd_reflock);
4591                         goto retry;
4592                 }
4593                 mutex_enter(&stp2->sd_reflock);
4594                 if (stp2->sd_refcnt > 0) {
4595                         STRUNLOCKMATES(stp);
4596                         mutex_exit(&stp1->sd_reflock);
4597                         cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4598                         mutex_exit(&stp2->sd_reflock);
4599                         goto retry;
4600                 }
4601                 STREAM_PUTLOCKS_ENTER(stp1);
4602                 STREAM_PUTLOCKS_ENTER(stp2);
4603         } else {
4604                 mutex_enter(&stp->sd_lock);
4605                 mutex_enter(&stp->sd_reflock);
4606                 while (stp->sd_refcnt > 0) {
4607                         mutex_exit(&stp->sd_lock);
4608                         cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4609                         if (mutex_tryenter(&stp->sd_lock) == 0) {
4610                                 mutex_exit(&stp->sd_reflock);
4611                                 mutex_enter(&stp->sd_lock);
4612                                 mutex_enter(&stp->sd_reflock);
4613                         }
4614                 }
4615                 STREAM_PUTLOCKS_ENTER(stp);
4616         }
4617 
4618         if (sqlist == NULL)
4619                 return;
4620 
4621         for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4622                 syncq_t *sq = sql->sql_sq;
4623                 uint16_t count;
4624 
4625                 mutex_enter(SQLOCK(sq));
4626                 count = sq->sq_count;
4627                 ASSERT(sq->sq_rmqcount <= count);
4628                 SQ_PUTLOCKS_ENTER(sq);
4629                 SUM_SQ_PUTCOUNTS(sq, count);
4630                 if (count == sq->sq_rmqcount)
4631                         continue;
4632 
4633                 /* Failed - drop all locks that we have acquired so far */
4634                 if (STRMATED(stp)) {
4635                         STREAM_PUTLOCKS_EXIT(stp);
4636                         STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4637                         STRUNLOCKMATES(stp);
4638                         mutex_exit(&stp->sd_reflock);
4639                         mutex_exit(&stp->sd_mate->sd_reflock);
4640                 } else {
4641                         STREAM_PUTLOCKS_EXIT(stp);
4642                         mutex_exit(&stp->sd_lock);
4643                         mutex_exit(&stp->sd_reflock);
4644                 }
4645                 for (sql2 = sqlist->sqlist_head; sql2 != sql;
4646                     sql2 = sql2->sql_next) {
4647                         SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4648                         mutex_exit(SQLOCK(sql2->sql_sq));
4649                 }
4650 
4651                 /*
4652                  * The wait loop below may starve when there are many threads
4653                  * claiming the syncq. This is especially a problem with permod
4654                  * syncqs (IP). To lessen the impact of the problem we increment
4655                  * sq_needexcl and clear fastbits so that putnexts will slow
4656                  * down and call sqenable instead of draining right away.
4657                  */
4658                 sq->sq_needexcl++;
4659                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4660                 while (count > sq->sq_rmqcount) {
4661                         sq->sq_flags |= SQ_WANTWAKEUP;
4662                         SQ_PUTLOCKS_EXIT(sq);
4663                         cv_wait(&sq->sq_wait, SQLOCK(sq));
4664                         count = sq->sq_count;
4665                         SQ_PUTLOCKS_ENTER(sq);
4666                         SUM_SQ_PUTCOUNTS(sq, count);
4667                 }
4668                 sq->sq_needexcl--;
4669                 if (sq->sq_needexcl == 0)
4670                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4671                 SQ_PUTLOCKS_EXIT(sq);
4672                 ASSERT(count == sq->sq_rmqcount);
4673                 mutex_exit(SQLOCK(sq));
4674                 goto retry;
4675         }
4676 }
4677 
4678 /*
4679  * Drop all the locks that strlock acquired.
4680  */
4681 static void
4682 strunlock(struct stdata *stp, sqlist_t *sqlist)
4683 {
4684         syncql_t *sql;
4685 
4686         if (STRMATED(stp)) {
4687                 STREAM_PUTLOCKS_EXIT(stp);
4688                 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4689                 STRUNLOCKMATES(stp);
4690                 mutex_exit(&stp->sd_reflock);
4691                 mutex_exit(&stp->sd_mate->sd_reflock);
4692         } else {
4693                 STREAM_PUTLOCKS_EXIT(stp);
4694                 mutex_exit(&stp->sd_lock);
4695                 mutex_exit(&stp->sd_reflock);
4696         }
4697 
4698         if (sqlist == NULL)
4699                 return;
4700 
4701         for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4702                 SQ_PUTLOCKS_EXIT(sql->sql_sq);
4703                 mutex_exit(SQLOCK(sql->sql_sq));
4704         }
4705 }
4706 
4707 /*
4708  * When the module has service procedure, we need check if the next
4709  * module which has service procedure is in flow control to trigger
4710  * the backenable.
4711  */
4712 static void
4713 backenable_insertedq(queue_t *q)
4714 {
4715         qband_t *qbp;
4716 
4717         claimstr(q);
4718         if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4719                 if (q->q_next->q_nfsrv->q_flag & QWANTW)
4720                         backenable(q, 0);
4721 
4722                 qbp = q->q_next->q_nfsrv->q_bandp;
4723                 for (; qbp != NULL; qbp = qbp->qb_next)
4724                         if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4725                                 backenable(q, qbp->qb_first->b_band);
4726         }
4727         releasestr(q);
4728 }
4729 
4730 /*
4731  * Given two read queues, insert a new single one after another.
4732  *
4733  * This routine acquires all the necessary locks in order to change
4734  * q_next and related pointer using strlock().
4735  * It depends on the stream head ensuring that there are no concurrent
4736  * insertq or removeq on the same stream. The stream head ensures this
4737  * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4738  *
4739  * Note that no syncq locks are held during the q_next change. This is
4740  * applied to all streams since, unlike removeq, there is no problem of stale
4741  * pointers when adding a module to the stream. Thus drivers/modules that do a
4742  * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4743  * applied this optimization to all streams.
4744  */
4745 void
4746 insertq(struct stdata *stp, queue_t *new)
4747 {
4748         queue_t *after;
4749         queue_t *wafter;
4750         queue_t *wnew = _WR(new);
4751         boolean_t have_fifo = B_FALSE;
4752 
4753         if (new->q_flag & _QINSERTING) {
4754                 ASSERT(stp->sd_vnode->v_type != VFIFO);
4755                 after = new->q_next;
4756                 wafter = _WR(new->q_next);
4757         } else {
4758                 after = _RD(stp->sd_wrq);
4759                 wafter = stp->sd_wrq;
4760         }
4761 
4762         TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4763             "insertq:%p, %p", after, new);
4764         ASSERT(after->q_flag & QREADR);
4765         ASSERT(new->q_flag & QREADR);
4766 
4767         strlock(stp, NULL);
4768 
4769         /* Do we have a FIFO? */
4770         if (wafter->q_next == after) {
4771                 have_fifo = B_TRUE;
4772                 wnew->q_next = new;
4773         } else {
4774                 wnew->q_next = wafter->q_next;
4775         }
4776         new->q_next = after;
4777 
4778         set_nfsrv_ptr(new, wnew, after, wafter);
4779         /*
4780          * set_nfsrv_ptr() needs to know if this is an insertion or not,
4781          * so only reset this flag after calling it.
4782          */
4783         new->q_flag &= ~_QINSERTING;
4784 
4785         if (have_fifo) {
4786                 wafter->q_next = wnew;
4787         } else {
4788                 if (wafter->q_next)
4789                         _OTHERQ(wafter->q_next)->q_next = new;
4790                 wafter->q_next = wnew;
4791         }
4792 
4793         set_qend(new);
4794         /* The QEND flag might have to be updated for the upstream guy */
4795         set_qend(after);
4796 
4797         ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4798         ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4799         ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4800         ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4801         strsetuio(stp);
4802 
4803         /*
4804          * If this was a module insertion, bump the push count.
4805          */
4806         if (!(new->q_flag & QISDRV))
4807                 stp->sd_pushcnt++;
4808 
4809         strunlock(stp, NULL);
4810 
4811         /* check if the write Q needs backenable */
4812         backenable_insertedq(wnew);
4813 
4814         /* check if the read Q needs backenable */
4815         backenable_insertedq(new);
4816 }
4817 
4818 /*
4819  * Given a read queue, unlink it from any neighbors.
4820  *
4821  * This routine acquires all the necessary locks in order to
4822  * change q_next and related pointers and also guard against
4823  * stale references (e.g. through q_next) to the queue that
4824  * is being removed. It also plays part of the role in ensuring
4825  * that the module's/driver's put procedure doesn't get called
4826  * after qprocsoff returns.
4827  *
4828  * Removeq depends on the stream head ensuring that there are
4829  * no concurrent insertq or removeq on the same stream. The
4830  * stream head ensures this using the flags STWOPEN, STRCLOSE and
4831  * STRPLUMB.
4832  *
4833  * The set of locks needed to remove the queue is different in
4834  * different cases:
4835  *
4836  * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4837  * waiting for the syncq reference count to drop to 0 indicating that no
4838  * non-close threads are present anywhere in the stream. This ensures that any
4839  * module/driver can reference q_next in its open, close, put, or service
4840  * procedures.
4841  *
4842  * The sq_rmqcount counter tracks the number of threads inside removeq().
4843  * strlock() ensures that there is either no threads executing inside perimeter
4844  * or there is only a thread calling qprocsoff().
4845  *
4846  * strlock() compares the value of sq_count with the number of threads inside
4847  * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4848  * any threads waiting in strlock() when the sq_rmqcount increases.
4849  */
4850 
4851 void
4852 removeq(queue_t *qp)
4853 {
4854         queue_t *wqp = _WR(qp);
4855         struct stdata *stp = STREAM(qp);
4856         sqlist_t *sqlist = NULL;
4857         boolean_t isdriver;
4858         int moved;
4859         syncq_t *sq = qp->q_syncq;
4860         syncq_t *wsq = wqp->q_syncq;
4861 
4862         ASSERT(stp);
4863 
4864         TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4865             "removeq:%p %p", qp, wqp);
4866         ASSERT(qp->q_flag&QREADR);
4867 
4868         /*
4869          * For queues using Synchronous streams, we must wait for all threads in
4870          * rwnext() to drain out before proceeding.
4871          */
4872         if (qp->q_flag & QSYNCSTR) {
4873                 /* First, we need wakeup any threads blocked in rwnext() */
4874                 mutex_enter(SQLOCK(sq));
4875                 if (sq->sq_flags & SQ_WANTWAKEUP) {
4876                         sq->sq_flags &= ~SQ_WANTWAKEUP;
4877                         cv_broadcast(&sq->sq_wait);
4878                 }
4879                 mutex_exit(SQLOCK(sq));
4880 
4881                 if (wsq != sq) {
4882                         mutex_enter(SQLOCK(wsq));
4883                         if (wsq->sq_flags & SQ_WANTWAKEUP) {
4884                                 wsq->sq_flags &= ~SQ_WANTWAKEUP;
4885                                 cv_broadcast(&wsq->sq_wait);
4886                         }
4887                         mutex_exit(SQLOCK(wsq));
4888                 }
4889 
4890                 mutex_enter(QLOCK(qp));
4891                 while (qp->q_rwcnt > 0) {
4892                         qp->q_flag |= QWANTRMQSYNC;
4893                         cv_wait(&qp->q_wait, QLOCK(qp));
4894                 }
4895                 mutex_exit(QLOCK(qp));
4896 
4897                 mutex_enter(QLOCK(wqp));
4898                 while (wqp->q_rwcnt > 0) {
4899                         wqp->q_flag |= QWANTRMQSYNC;
4900                         cv_wait(&wqp->q_wait, QLOCK(wqp));
4901                 }
4902                 mutex_exit(QLOCK(wqp));
4903         }
4904 
4905         mutex_enter(SQLOCK(sq));
4906         sq->sq_rmqcount++;
4907         if (sq->sq_flags & SQ_WANTWAKEUP) {
4908                 sq->sq_flags &= ~SQ_WANTWAKEUP;
4909                 cv_broadcast(&sq->sq_wait);
4910         }
4911         mutex_exit(SQLOCK(sq));
4912 
4913         isdriver = (qp->q_flag & QISDRV);
4914 
4915         sqlist = sqlist_build(qp, stp, STRMATED(stp));
4916         strlock(stp, sqlist);
4917 
4918         reset_nfsrv_ptr(qp, wqp);
4919 
4920         ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4921         ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4922         /* Do we have a FIFO? */
4923         if (wqp->q_next == qp) {
4924                 stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4925         } else {
4926                 if (wqp->q_next)
4927                         backq(qp)->q_next = qp->q_next;
4928                 if (qp->q_next)
4929                         backq(wqp)->q_next = wqp->q_next;
4930         }
4931 
4932         /* The QEND flag might have to be updated for the upstream guy */
4933         if (qp->q_next)
4934                 set_qend(qp->q_next);
4935 
4936         ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4937         ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4938 
4939         /*
4940          * Move any messages destined for the put procedures to the next
4941          * syncq in line. Otherwise free them.
4942          */
4943         moved = 0;
4944         /*
4945          * Quick check to see whether there are any messages or events.
4946          */
4947         if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4948                 moved += propagate_syncq(qp);
4949         if (wqp->q_syncqmsgs != 0 ||
4950             (wqp->q_syncq->sq_flags & SQ_EVENTS))
4951                 moved += propagate_syncq(wqp);
4952 
4953         strsetuio(stp);
4954 
4955         /*
4956          * If this was a module removal, decrement the push count.
4957          */
4958         if (!isdriver)
4959                 stp->sd_pushcnt--;
4960 
4961         strunlock(stp, sqlist);
4962         sqlist_free(sqlist);
4963 
4964         /*
4965          * Make sure any messages that were propagated are drained.
4966          * Also clear any QFULL bit caused by messages that were propagated.
4967          */
4968 
4969         if (qp->q_next != NULL) {
4970                 clr_qfull(qp);
4971                 /*
4972                  * For the driver calling qprocsoff, propagate_syncq
4973                  * frees all the messages instead of putting it in
4974                  * the stream head
4975                  */
4976                 if (!isdriver && (moved > 0))
4977                         emptysq(qp->q_next->q_syncq);
4978         }
4979         if (wqp->q_next != NULL) {
4980                 clr_qfull(wqp);
4981                 /*
4982                  * We come here for any pop of a module except for the
4983                  * case of driver being removed. We don't call emptysq
4984                  * if we did not move any messages. This will avoid holding
4985                  * PERMOD syncq locks in emptysq
4986                  */
4987                 if (moved > 0)
4988                         emptysq(wqp->q_next->q_syncq);
4989         }
4990 
4991         mutex_enter(SQLOCK(sq));
4992         sq->sq_rmqcount--;
4993         mutex_exit(SQLOCK(sq));
4994 }
4995 
4996 /*
4997  * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
4998  * SQ_WRITER) on a syncq.
4999  * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5000  * sync queue and waits until sq_count reaches maxcnt.
5001  *
5002  * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5003  * does not care about putnext threads that are in the middle of calling put
5004  * entry points.
5005  *
5006  * This routine is used for both inner and outer syncqs.
5007  */
5008 static void
5009 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5010 {
5011         uint16_t count = 0;
5012 
5013         mutex_enter(SQLOCK(sq));
5014         /*
5015          * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5016          * SQ_FROZEN will be set if there is a frozen stream that has a
5017          * queue which also refers to this "shared" syncq.
5018          * SQ_BLOCKED will be set if there is "off" queue which also
5019          * refers to this "shared" syncq.
5020          */
5021         if (maxcnt != -1) {
5022                 count = sq->sq_count;
5023                 SQ_PUTLOCKS_ENTER(sq);
5024                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5025                 SUM_SQ_PUTCOUNTS(sq, count);
5026         }
5027         sq->sq_needexcl++;
5028         ASSERT(sq->sq_needexcl != 0);        /* wraparound */
5029 
5030         while ((sq->sq_flags & flag) ||
5031             (maxcnt != -1 && count > (unsigned)maxcnt)) {
5032                 sq->sq_flags |= SQ_WANTWAKEUP;
5033                 if (maxcnt != -1) {
5034                         SQ_PUTLOCKS_EXIT(sq);
5035                 }
5036                 cv_wait(&sq->sq_wait, SQLOCK(sq));
5037                 if (maxcnt != -1) {
5038                         count = sq->sq_count;
5039                         SQ_PUTLOCKS_ENTER(sq);
5040                         SUM_SQ_PUTCOUNTS(sq, count);
5041                 }
5042         }
5043         sq->sq_needexcl--;
5044         sq->sq_flags |= flag;
5045         ASSERT(maxcnt == -1 || count == maxcnt);
5046         if (maxcnt != -1) {
5047                 if (sq->sq_needexcl == 0) {
5048                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5049                 }
5050                 SQ_PUTLOCKS_EXIT(sq);
5051         } else if (sq->sq_needexcl == 0) {
5052                 SQ_PUTCOUNT_SETFAST(sq);
5053         }
5054 
5055         mutex_exit(SQLOCK(sq));
5056 }
5057 
5058 /*
5059  * Reset a flag that was set with blocksq.
5060  *
5061  * Can not use this routine to reset SQ_WRITER.
5062  *
5063  * If "isouter" is set then the syncq is assumed to be an outer perimeter
5064  * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5065  * to handle the queued qwriter operations.
5066  *
5067  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5068  * sq_putlocks are used.
5069  */
5070 static void
5071 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5072 {
5073         uint16_t flags;
5074 
5075         mutex_enter(SQLOCK(sq));
5076         ASSERT(resetflag != SQ_WRITER);
5077         ASSERT(sq->sq_flags & resetflag);
5078         flags = sq->sq_flags & ~resetflag;
5079         sq->sq_flags = flags;
5080         if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5081                 if (flags & SQ_WANTWAKEUP) {
5082                         flags &= ~SQ_WANTWAKEUP;
5083                         cv_broadcast(&sq->sq_wait);
5084                 }
5085                 sq->sq_flags = flags;
5086                 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5087                         if (!isouter) {
5088                                 /* drain_syncq drops SQLOCK */
5089                                 drain_syncq(sq);
5090                                 return;
5091                         }
5092                 }
5093         }
5094         mutex_exit(SQLOCK(sq));
5095 }
5096 
5097 /*
5098  * Reset a flag that was set with blocksq.
5099  * Does not drain the syncq. Use emptysq() for that.
5100  * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5101  *
5102  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5103  * sq_putlocks are used.
5104  */
5105 static int
5106 dropsq(syncq_t *sq, uint16_t resetflag)
5107 {
5108         uint16_t flags;
5109 
5110         mutex_enter(SQLOCK(sq));
5111         ASSERT(sq->sq_flags & resetflag);
5112         flags = sq->sq_flags & ~resetflag;
5113         if (flags & SQ_WANTWAKEUP) {
5114                 flags &= ~SQ_WANTWAKEUP;
5115                 cv_broadcast(&sq->sq_wait);
5116         }
5117         sq->sq_flags = flags;
5118         mutex_exit(SQLOCK(sq));
5119         if (flags & SQ_QUEUED)
5120                 return (1);
5121         return (0);
5122 }
5123 
5124 /*
5125  * Empty all the messages on a syncq.
5126  *
5127  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5128  * sq_putlocks are used.
5129  */
5130 static void
5131 emptysq(syncq_t *sq)
5132 {
5133         uint16_t flags;
5134 
5135         mutex_enter(SQLOCK(sq));
5136         flags = sq->sq_flags;
5137         if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5138                 /*
5139                  * To prevent potential recursive invocation of drain_syncq we
5140                  * do not call drain_syncq if count is non-zero.
5141                  */
5142                 if (sq->sq_count == 0) {
5143                         /* drain_syncq() drops SQLOCK */
5144                         drain_syncq(sq);
5145                         return;
5146                 } else
5147                         sqenable(sq);
5148         }
5149         mutex_exit(SQLOCK(sq));
5150 }
5151 
5152 /*
5153  * Ordered insert while removing duplicates.
5154  */
5155 static void
5156 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5157 {
5158         syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5159 
5160         prev_sqlpp = &sqlist->sqlist_head;
5161         while ((sqlp = *prev_sqlpp) != NULL) {
5162                 if (sqlp->sql_sq >= sqp) {
5163                         if (sqlp->sql_sq == sqp)     /* duplicate */
5164                                 return;
5165                         break;
5166                 }
5167                 prev_sqlpp = &sqlp->sql_next;
5168         }
5169         new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5170         ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5171         new_sqlp->sql_next = sqlp;
5172         new_sqlp->sql_sq = sqp;
5173         *prev_sqlpp = new_sqlp;
5174 }
5175 
5176 /*
5177  * Walk the write side queues until we hit either the driver
5178  * or a twist in the stream (_SAMESTR will return false in both
5179  * these cases) then turn around and walk the read side queues
5180  * back up to the stream head.
5181  */
5182 static void
5183 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5184 {
5185         while (q != NULL) {
5186                 sqlist_insert(sqlist, q->q_syncq);
5187 
5188                 if (_SAMESTR(q))
5189                         q = q->q_next;
5190                 else if (!(q->q_flag & QREADR))
5191                         q = _RD(q);
5192                 else
5193                         q = NULL;
5194         }
5195 }
5196 
5197 /*
5198  * Allocate and build a list of all syncqs in a stream and the syncq(s)
5199  * associated with the "q" parameter. The resulting list is sorted in a
5200  * canonical order and is free of duplicates.
5201  * Assumes the passed queue is a _RD(q).
5202  */
5203 static sqlist_t *
5204 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5205 {
5206         sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5207 
5208         /*
5209          * start with the current queue/qpair
5210          */
5211         ASSERT(q->q_flag & QREADR);
5212 
5213         sqlist_insert(sqlist, q->q_syncq);
5214         sqlist_insert(sqlist, _WR(q)->q_syncq);
5215 
5216         sqlist_insertall(sqlist, stp->sd_wrq);
5217         if (do_twist)
5218                 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5219 
5220         return (sqlist);
5221 }
5222 
5223 static sqlist_t *
5224 sqlist_alloc(struct stdata *stp, int kmflag)
5225 {
5226         size_t sqlist_size;
5227         sqlist_t *sqlist;
5228 
5229         /*
5230          * Allocate 2 syncql_t's for each pushed module. Note that
5231          * the sqlist_t structure already has 4 syncql_t's built in:
5232          * 2 for the stream head, and 2 for the driver/other stream head.
5233          */
5234         sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5235             sizeof (sqlist_t);
5236         if (STRMATED(stp))
5237                 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5238         sqlist = kmem_alloc(sqlist_size, kmflag);
5239 
5240         sqlist->sqlist_head = NULL;
5241         sqlist->sqlist_size = sqlist_size;
5242         sqlist->sqlist_index = 0;
5243 
5244         return (sqlist);
5245 }
5246 
5247 /*
5248  * Free the list created by sqlist_alloc()
5249  */
5250 static void
5251 sqlist_free(sqlist_t *sqlist)
5252 {
5253         kmem_free(sqlist, sqlist->sqlist_size);
5254 }
5255 
5256 /*
5257  * Prevent any new entries into any syncq in this stream.
5258  * Used by freezestr.
5259  */
5260 void
5261 strblock(queue_t *q)
5262 {
5263         struct stdata   *stp;
5264         syncql_t        *sql;
5265         sqlist_t        *sqlist;
5266 
5267         q = _RD(q);
5268 
5269         stp = STREAM(q);
5270         ASSERT(stp != NULL);
5271 
5272         /*
5273          * Get a sorted list with all the duplicates removed containing
5274          * all the syncqs referenced by this stream.
5275          */
5276         sqlist = sqlist_build(q, stp, B_FALSE);
5277         for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5278                 blocksq(sql->sql_sq, SQ_FROZEN, -1);
5279         sqlist_free(sqlist);
5280 }
5281 
5282 /*
5283  * Release the block on new entries into this stream
5284  */
5285 void
5286 strunblock(queue_t *q)
5287 {
5288         struct stdata   *stp;
5289         syncql_t        *sql;
5290         sqlist_t        *sqlist;
5291         int             drain_needed;
5292 
5293         q = _RD(q);
5294 
5295         /*
5296          * Get a sorted list with all the duplicates removed containing
5297          * all the syncqs referenced by this stream.
5298          * Have to drop the SQ_FROZEN flag on all the syncqs before
5299          * starting to drain them; otherwise the draining might
5300          * cause a freezestr in some module on the stream (which
5301          * would deadlock).
5302          */
5303         stp = STREAM(q);
5304         ASSERT(stp != NULL);
5305         sqlist = sqlist_build(q, stp, B_FALSE);
5306         drain_needed = 0;
5307         for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5308                 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5309         if (drain_needed) {
5310                 for (sql = sqlist->sqlist_head; sql != NULL;
5311                     sql = sql->sql_next)
5312                         emptysq(sql->sql_sq);
5313         }
5314         sqlist_free(sqlist);
5315 }
5316 
5317 #ifdef DEBUG
5318 static int
5319 qprocsareon(queue_t *rq)
5320 {
5321         if (rq->q_next == NULL)
5322                 return (0);
5323         return (_WR(rq->q_next)->q_next == _WR(rq));
5324 }
5325 
5326 int
5327 qclaimed(queue_t *q)
5328 {
5329         uint_t count;
5330 
5331         count = q->q_syncq->sq_count;
5332         SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5333         return (count != 0);
5334 }
5335 
5336 /*
5337  * Check if anyone has frozen this stream with freezestr
5338  */
5339 int
5340 frozenstr(queue_t *q)
5341 {
5342         return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5343 }
5344 #endif /* DEBUG */
5345 
5346 /*
5347  * Enter a queue.
5348  * Obsoleted interface. Should not be used.
5349  */
5350 void
5351 enterq(queue_t *q)
5352 {
5353         entersq(q->q_syncq, SQ_CALLBACK);
5354 }
5355 
5356 void
5357 leaveq(queue_t *q)
5358 {
5359         leavesq(q->q_syncq, SQ_CALLBACK);
5360 }
5361 
5362 /*
5363  * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5364  * to check.
5365  * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5366  * calls and the running of open, close and service procedures.
5367  *
5368  * If c_inner bit is set no need to grab sq_putlocks since we don't care
5369  * if other threads have entered or are entering put entry point.
5370  *
5371  * If c_inner bit is set it might have been possible to use
5372  * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5373  * open/close path for IP) but since the count may need to be decremented in
5374  * qwait() we wouldn't know which counter to decrement. Currently counter is
5375  * selected by current cpu_seqid and current CPU can change at any moment. XXX
5376  * in the future we might use curthread id bits to select the counter and this
5377  * would stay constant across routine calls.
5378  */
5379 void
5380 entersq(syncq_t *sq, int entrypoint)
5381 {
5382         uint16_t        count = 0;
5383         uint16_t        flags;
5384         uint16_t        waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5385         uint16_t        type;
5386         uint_t          c_inner = entrypoint & SQ_CI;
5387         uint_t          c_outer = entrypoint & SQ_CO;
5388 
5389         /*
5390          * Increment ref count to keep closes out of this queue.
5391          */
5392         ASSERT(sq);
5393         ASSERT(c_inner && c_outer);
5394         mutex_enter(SQLOCK(sq));
5395         flags = sq->sq_flags;
5396         type = sq->sq_type;
5397         if (!(type & c_inner)) {
5398                 /* Make sure all putcounts now use slowlock. */
5399                 count = sq->sq_count;
5400                 SQ_PUTLOCKS_ENTER(sq);
5401                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5402                 SUM_SQ_PUTCOUNTS(sq, count);
5403                 sq->sq_needexcl++;
5404                 ASSERT(sq->sq_needexcl != 0);        /* wraparound */
5405                 waitflags |= SQ_MESSAGES;
5406         }
5407         /*
5408          * Wait until we can enter the inner perimeter.
5409          * If we want exclusive access we wait until sq_count is 0.
5410          * We have to do this before entering the outer perimeter in order
5411          * to preserve put/close message ordering.
5412          */
5413         while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5414                 sq->sq_flags = flags | SQ_WANTWAKEUP;
5415                 if (!(type & c_inner)) {
5416                         SQ_PUTLOCKS_EXIT(sq);
5417                 }
5418                 cv_wait(&sq->sq_wait, SQLOCK(sq));
5419                 if (!(type & c_inner)) {
5420                         count = sq->sq_count;
5421                         SQ_PUTLOCKS_ENTER(sq);
5422                         SUM_SQ_PUTCOUNTS(sq, count);
5423                 }
5424                 flags = sq->sq_flags;
5425         }
5426 
5427         if (!(type & c_inner)) {
5428                 ASSERT(sq->sq_needexcl > 0);
5429                 sq->sq_needexcl--;
5430                 if (sq->sq_needexcl == 0) {
5431                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5432                 }
5433         }
5434 
5435         /* Check if we need to enter the outer perimeter */
5436         if (!(type & c_outer)) {
5437                 /*
5438                  * We have to enter the outer perimeter exclusively before
5439                  * we can increment sq_count to avoid deadlock. This implies
5440                  * that we have to re-check sq_flags and sq_count.
5441                  *
5442                  * is it possible to have c_inner set when c_outer is not set?
5443                  */
5444                 if (!(type & c_inner)) {
5445                         SQ_PUTLOCKS_EXIT(sq);
5446                 }
5447                 mutex_exit(SQLOCK(sq));
5448                 outer_enter(sq->sq_outer, SQ_GOAWAY);
5449                 mutex_enter(SQLOCK(sq));
5450                 flags = sq->sq_flags;
5451                 /*
5452                  * there should be no need to recheck sq_putcounts
5453                  * because outer_enter() has already waited for them to clear
5454                  * after setting SQ_WRITER.
5455                  */
5456                 count = sq->sq_count;
5457 #ifdef DEBUG
5458                 /*
5459                  * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5460                  * of doing an ASSERT internally. Others should do
5461                  * something like
5462                  *       ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5463                  * without the need to #ifdef DEBUG it.
5464                  */
5465                 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5466 #endif
5467                 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5468                     (!(type & c_inner) && count != 0)) {
5469                         sq->sq_flags = flags | SQ_WANTWAKEUP;
5470                         cv_wait(&sq->sq_wait, SQLOCK(sq));
5471                         count = sq->sq_count;
5472                         flags = sq->sq_flags;
5473                 }
5474         }
5475 
5476         sq->sq_count++;
5477         ASSERT(sq->sq_count != 0);   /* Wraparound */
5478         if (!(type & c_inner)) {
5479                 /* Exclusive entry */
5480                 ASSERT(sq->sq_count == 1);
5481                 sq->sq_flags |= SQ_EXCL;
5482                 if (type & c_outer) {
5483                         SQ_PUTLOCKS_EXIT(sq);
5484                 }
5485         }
5486         mutex_exit(SQLOCK(sq));
5487 }
5488 
5489 /*
5490  * Leave a syncq. Announce to framework that closes may proceed.
5491  * c_inner and c_outer specify which concurrency bits to check.
5492  *
5493  * Must never be called from driver or module put entry point.
5494  *
5495  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5496  * sq_putlocks are used.
5497  */
5498 void
5499 leavesq(syncq_t *sq, int entrypoint)
5500 {
5501         uint16_t        flags;
5502         uint16_t        type;
5503         uint_t          c_outer = entrypoint & SQ_CO;
5504 #ifdef DEBUG
5505         uint_t          c_inner = entrypoint & SQ_CI;
5506 #endif
5507 
5508         /*
5509          * Decrement ref count, drain the syncq if possible, and wake up
5510          * any waiting close.
5511          */
5512         ASSERT(sq);
5513         ASSERT(c_inner && c_outer);
5514         mutex_enter(SQLOCK(sq));
5515         flags = sq->sq_flags;
5516         type = sq->sq_type;
5517         if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5518 
5519                 if (flags & SQ_WANTWAKEUP) {
5520                         flags &= ~SQ_WANTWAKEUP;
5521                         cv_broadcast(&sq->sq_wait);
5522                 }
5523                 if (flags & SQ_WANTEXWAKEUP) {
5524                         flags &= ~SQ_WANTEXWAKEUP;
5525                         cv_broadcast(&sq->sq_exitwait);
5526                 }
5527 
5528                 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5529                         /*
5530                          * The syncq needs to be drained. "Exit" the syncq
5531                          * before calling drain_syncq.
5532                          */
5533                         ASSERT(sq->sq_count != 0);
5534                         sq->sq_count--;
5535                         ASSERT((flags & SQ_EXCL) || (type & c_inner));
5536                         sq->sq_flags = flags & ~SQ_EXCL;
5537                         drain_syncq(sq);
5538                         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5539                         /* Check if we need to exit the outer perimeter */
5540                         /* XXX will this ever be true? */
5541                         if (!(type & c_outer))
5542                                 outer_exit(sq->sq_outer);
5543                         return;
5544                 }
5545         }
5546         ASSERT(sq->sq_count != 0);
5547         sq->sq_count--;
5548         ASSERT((flags & SQ_EXCL) || (type & c_inner));
5549         sq->sq_flags = flags & ~SQ_EXCL;
5550         mutex_exit(SQLOCK(sq));
5551 
5552         /* Check if we need to exit the outer perimeter */
5553         if (!(sq->sq_type & c_outer))
5554                 outer_exit(sq->sq_outer);
5555 }
5556 
5557 /*
5558  * Prevent q_next from changing in this stream by incrementing sq_count.
5559  *
5560  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5561  * sq_putlocks are used.
5562  */
5563 void
5564 claimq(queue_t *qp)
5565 {
5566         syncq_t *sq = qp->q_syncq;
5567 
5568         mutex_enter(SQLOCK(sq));
5569         sq->sq_count++;
5570         ASSERT(sq->sq_count != 0);   /* Wraparound */
5571         mutex_exit(SQLOCK(sq));
5572 }
5573 
5574 /*
5575  * Undo claimq.
5576  *
5577  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5578  * sq_putlocks are used.
5579  */
5580 void
5581 releaseq(queue_t *qp)
5582 {
5583         syncq_t *sq = qp->q_syncq;
5584         uint16_t flags;
5585 
5586         mutex_enter(SQLOCK(sq));
5587         ASSERT(sq->sq_count > 0);
5588         sq->sq_count--;
5589 
5590         flags = sq->sq_flags;
5591         if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5592                 if (flags & SQ_WANTWAKEUP) {
5593                         flags &= ~SQ_WANTWAKEUP;
5594                         cv_broadcast(&sq->sq_wait);
5595                 }
5596                 sq->sq_flags = flags;
5597                 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5598                         /*
5599                          * To prevent potential recursive invocation of
5600                          * drain_syncq we do not call drain_syncq if count is
5601                          * non-zero.
5602                          */
5603                         if (sq->sq_count == 0) {
5604                                 drain_syncq(sq);
5605                                 return;
5606                         } else
5607                                 sqenable(sq);
5608                 }
5609         }
5610         mutex_exit(SQLOCK(sq));
5611 }
5612 
5613 /*
5614  * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5615  */
5616 void
5617 claimstr(queue_t *qp)
5618 {
5619         struct stdata *stp = STREAM(qp);
5620 
5621         mutex_enter(&stp->sd_reflock);
5622         stp->sd_refcnt++;
5623         ASSERT(stp->sd_refcnt != 0); /* Wraparound */
5624         mutex_exit(&stp->sd_reflock);
5625 }
5626 
5627 /*
5628  * Undo claimstr.
5629  */
5630 void
5631 releasestr(queue_t *qp)
5632 {
5633         struct stdata *stp = STREAM(qp);
5634 
5635         mutex_enter(&stp->sd_reflock);
5636         ASSERT(stp->sd_refcnt != 0);
5637         if (--stp->sd_refcnt == 0)
5638                 cv_broadcast(&stp->sd_refmonitor);
5639         mutex_exit(&stp->sd_reflock);
5640 }
5641 
5642 static syncq_t *
5643 new_syncq(void)
5644 {
5645         return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5646 }
5647 
5648 static void
5649 free_syncq(syncq_t *sq)
5650 {
5651         ASSERT(sq->sq_head == NULL);
5652         ASSERT(sq->sq_outer == NULL);
5653         ASSERT(sq->sq_callbpend == NULL);
5654         ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5655             (sq->sq_onext == sq && sq->sq_oprev == sq));
5656 
5657         if (sq->sq_ciputctrl != NULL) {
5658                 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5659                 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5660                     sq->sq_nciputctrl, 0);
5661                 ASSERT(ciputctrl_cache != NULL);
5662                 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5663         }
5664 
5665         sq->sq_tail = NULL;
5666         sq->sq_evhead = NULL;
5667         sq->sq_evtail = NULL;
5668         sq->sq_ciputctrl = NULL;
5669         sq->sq_nciputctrl = 0;
5670         sq->sq_count = 0;
5671         sq->sq_rmqcount = 0;
5672         sq->sq_callbflags = 0;
5673         sq->sq_cancelid = 0;
5674         sq->sq_next = NULL;
5675         sq->sq_needexcl = 0;
5676         sq->sq_svcflags = 0;
5677         sq->sq_nqueues = 0;
5678         sq->sq_pri = 0;
5679         sq->sq_onext = NULL;
5680         sq->sq_oprev = NULL;
5681         sq->sq_flags = 0;
5682         sq->sq_type = 0;
5683         sq->sq_servcount = 0;
5684 
5685         kmem_cache_free(syncq_cache, sq);
5686 }
5687 
5688 /* Outer perimeter code */
5689 
5690 /*
5691  * The outer syncq uses the fields and flags in the syncq slightly
5692  * differently from the inner syncqs.
5693  *      sq_count        Incremented when there are pending or running
5694  *                      writers at the outer perimeter to prevent the set of
5695  *                      inner syncqs that belong to the outer perimeter from
5696  *                      changing.
5697  *      sq_head/tail    List of deferred qwriter(OUTER) operations.
5698  *
5699  *      SQ_BLOCKED      Set to prevent traversing of sq_next,sq_prev while
5700  *                      inner syncqs are added to or removed from the
5701  *                      outer perimeter.
5702  *      SQ_QUEUED       sq_head/tail has messages or events queued.
5703  *
5704  *      SQ_WRITER       A thread is currently traversing all the inner syncqs
5705  *                      setting the SQ_WRITER flag.
5706  */
5707 
5708 /*
5709  * Get write access at the outer perimeter.
5710  * Note that read access is done by entersq, putnext, and put by simply
5711  * incrementing sq_count in the inner syncq.
5712  *
5713  * Waits until "flags" is no longer set in the outer to prevent multiple
5714  * threads from having write access at the same time. SQ_WRITER has to be part
5715  * of "flags".
5716  *
5717  * Increases sq_count on the outer syncq to keep away outer_insert/remove
5718  * until the outer_exit is finished.
5719  *
5720  * outer_enter is vulnerable to starvation since it does not prevent new
5721  * threads from entering the inner syncqs while it is waiting for sq_count to
5722  * go to zero.
5723  */
5724 void
5725 outer_enter(syncq_t *outer, uint16_t flags)
5726 {
5727         syncq_t *sq;
5728         int     wait_needed;
5729         uint16_t        count;
5730 
5731         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5732             outer->sq_oprev != NULL);
5733         ASSERT(flags & SQ_WRITER);
5734 
5735 retry:
5736         mutex_enter(SQLOCK(outer));
5737         while (outer->sq_flags & flags) {
5738                 outer->sq_flags |= SQ_WANTWAKEUP;
5739                 cv_wait(&outer->sq_wait, SQLOCK(outer));
5740         }
5741 
5742         ASSERT(!(outer->sq_flags & SQ_WRITER));
5743         outer->sq_flags |= SQ_WRITER;
5744         outer->sq_count++;
5745         ASSERT(outer->sq_count != 0);        /* wraparound */
5746         wait_needed = 0;
5747         /*
5748          * Set SQ_WRITER on all the inner syncqs while holding
5749          * the SQLOCK on the outer syncq. This ensures that the changing
5750          * of SQ_WRITER is atomic under the outer SQLOCK.
5751          */
5752         for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5753                 mutex_enter(SQLOCK(sq));
5754                 count = sq->sq_count;
5755                 SQ_PUTLOCKS_ENTER(sq);
5756                 sq->sq_flags |= SQ_WRITER;
5757                 SUM_SQ_PUTCOUNTS(sq, count);
5758                 if (count != 0)
5759                         wait_needed = 1;
5760                 SQ_PUTLOCKS_EXIT(sq);
5761                 mutex_exit(SQLOCK(sq));
5762         }
5763         mutex_exit(SQLOCK(outer));
5764 
5765         /*
5766          * Get everybody out of the syncqs sequentially.
5767          * Note that we don't actually need to acquire the PUTLOCKS, since
5768          * we have already cleared the fastbit, and set QWRITER.  By
5769          * definition, the count can not increase since putnext will
5770          * take the slowlock path (and the purpose of acquiring the
5771          * putlocks was to make sure it didn't increase while we were
5772          * waiting).
5773          *
5774          * Note that we still acquire the PUTLOCKS to be safe.
5775          */
5776         if (wait_needed) {
5777                 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5778                         mutex_enter(SQLOCK(sq));
5779                         count = sq->sq_count;
5780                         SQ_PUTLOCKS_ENTER(sq);
5781                         SUM_SQ_PUTCOUNTS(sq, count);
5782                         while (count != 0) {
5783                                 sq->sq_flags |= SQ_WANTWAKEUP;
5784                                 SQ_PUTLOCKS_EXIT(sq);
5785                                 cv_wait(&sq->sq_wait, SQLOCK(sq));
5786                                 count = sq->sq_count;
5787                                 SQ_PUTLOCKS_ENTER(sq);
5788                                 SUM_SQ_PUTCOUNTS(sq, count);
5789                         }
5790                         SQ_PUTLOCKS_EXIT(sq);
5791                         mutex_exit(SQLOCK(sq));
5792                 }
5793                 /*
5794                  * Verify that none of the flags got set while we
5795                  * were waiting for the sq_counts to drop.
5796                  * If this happens we exit and retry entering the
5797                  * outer perimeter.
5798                  */
5799                 mutex_enter(SQLOCK(outer));
5800                 if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5801                         mutex_exit(SQLOCK(outer));
5802                         outer_exit(outer);
5803                         goto retry;
5804                 }
5805                 mutex_exit(SQLOCK(outer));
5806         }
5807 }
5808 
5809 /*
5810  * Drop the write access at the outer perimeter.
5811  * Read access is dropped implicitly (by putnext, put, and leavesq) by
5812  * decrementing sq_count.
5813  */
5814 void
5815 outer_exit(syncq_t *outer)
5816 {
5817         syncq_t *sq;
5818         int      drain_needed;
5819         uint16_t flags;
5820 
5821         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5822             outer->sq_oprev != NULL);
5823         ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5824 
5825         /*
5826          * Atomically (from the perspective of threads calling become_writer)
5827          * drop the write access at the outer perimeter by holding
5828          * SQLOCK(outer) across all the dropsq calls and the resetting of
5829          * SQ_WRITER.
5830          * This defines a locking order between the outer perimeter
5831          * SQLOCK and the inner perimeter SQLOCKs.
5832          */
5833         mutex_enter(SQLOCK(outer));
5834         flags = outer->sq_flags;
5835         ASSERT(outer->sq_flags & SQ_WRITER);
5836         if (flags & SQ_QUEUED) {
5837                 write_now(outer);
5838                 flags = outer->sq_flags;
5839         }
5840 
5841         /*
5842          * sq_onext is stable since sq_count has not yet been decreased.
5843          * Reset the SQ_WRITER flags in all syncqs.
5844          * After dropping SQ_WRITER on the outer syncq we empty all the
5845          * inner syncqs.
5846          */
5847         drain_needed = 0;
5848         for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5849                 drain_needed += dropsq(sq, SQ_WRITER);
5850         ASSERT(!(outer->sq_flags & SQ_QUEUED));
5851         flags &= ~SQ_WRITER;
5852         if (drain_needed) {
5853                 outer->sq_flags = flags;
5854                 mutex_exit(SQLOCK(outer));
5855                 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5856                         emptysq(sq);
5857                 mutex_enter(SQLOCK(outer));
5858                 flags = outer->sq_flags;
5859         }
5860         if (flags & SQ_WANTWAKEUP) {
5861                 flags &= ~SQ_WANTWAKEUP;
5862                 cv_broadcast(&outer->sq_wait);
5863         }
5864         outer->sq_flags = flags;
5865         ASSERT(outer->sq_count > 0);
5866         outer->sq_count--;
5867         mutex_exit(SQLOCK(outer));
5868 }
5869 
5870 /*
5871  * Add another syncq to an outer perimeter.
5872  * Block out all other access to the outer perimeter while it is being
5873  * changed using blocksq.
5874  * Assumes that the caller has *not* done an outer_enter.
5875  *
5876  * Vulnerable to starvation in blocksq.
5877  */
5878 static void
5879 outer_insert(syncq_t *outer, syncq_t *sq)
5880 {
5881         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5882             outer->sq_oprev != NULL);
5883         ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5884             sq->sq_oprev == NULL);   /* Can't be in an outer perimeter */
5885 
5886         /* Get exclusive access to the outer perimeter list */
5887         blocksq(outer, SQ_BLOCKED, 0);
5888         ASSERT(outer->sq_flags & SQ_BLOCKED);
5889         ASSERT(!(outer->sq_flags & SQ_WRITER));
5890 
5891         mutex_enter(SQLOCK(sq));
5892         sq->sq_outer = outer;
5893         outer->sq_onext->sq_oprev = sq;
5894         sq->sq_onext = outer->sq_onext;
5895         outer->sq_onext = sq;
5896         sq->sq_oprev = outer;
5897         mutex_exit(SQLOCK(sq));
5898         unblocksq(outer, SQ_BLOCKED, 1);
5899 }
5900 
5901 /*
5902  * Remove a syncq from an outer perimeter.
5903  * Block out all other access to the outer perimeter while it is being
5904  * changed using blocksq.
5905  * Assumes that the caller has *not* done an outer_enter.
5906  *
5907  * Vulnerable to starvation in blocksq.
5908  */
5909 static void
5910 outer_remove(syncq_t *outer, syncq_t *sq)
5911 {
5912         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5913             outer->sq_oprev != NULL);
5914         ASSERT(sq->sq_outer == outer);
5915 
5916         /* Get exclusive access to the outer perimeter list */
5917         blocksq(outer, SQ_BLOCKED, 0);
5918         ASSERT(outer->sq_flags & SQ_BLOCKED);
5919         ASSERT(!(outer->sq_flags & SQ_WRITER));
5920 
5921         mutex_enter(SQLOCK(sq));
5922         sq->sq_outer = NULL;
5923         sq->sq_onext->sq_oprev = sq->sq_oprev;
5924         sq->sq_oprev->sq_onext = sq->sq_onext;
5925         sq->sq_oprev = sq->sq_onext = NULL;
5926         mutex_exit(SQLOCK(sq));
5927         unblocksq(outer, SQ_BLOCKED, 1);
5928 }
5929 
5930 /*
5931  * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5932  * If this is the first callback for this outer perimeter then add
5933  * this outer perimeter to the list of outer perimeters that
5934  * the qwriter_outer_thread will process.
5935  *
5936  * Increments sq_count in the outer syncq to prevent the membership
5937  * of the outer perimeter (in terms of inner syncqs) to change while
5938  * the callback is pending.
5939  */
5940 static void
5941 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5942 {
5943         ASSERT(MUTEX_HELD(SQLOCK(outer)));
5944 
5945         mp->b_prev = (mblk_t *)func;
5946         mp->b_queue = q;
5947         mp->b_next = NULL;
5948         outer->sq_count++;   /* Decremented when dequeued */
5949         ASSERT(outer->sq_count != 0);        /* Wraparound */
5950         if (outer->sq_evhead == NULL) {
5951                 /* First message. */
5952                 outer->sq_evhead = outer->sq_evtail = mp;
5953                 outer->sq_flags |= SQ_EVENTS;
5954                 mutex_exit(SQLOCK(outer));
5955                 STRSTAT(qwr_outer);
5956                 (void) taskq_dispatch(streams_taskq,
5957                     (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5958         } else {
5959                 ASSERT(outer->sq_flags & SQ_EVENTS);
5960                 outer->sq_evtail->b_next = mp;
5961                 outer->sq_evtail = mp;
5962                 mutex_exit(SQLOCK(outer));
5963         }
5964 }
5965 
5966 /*
5967  * Try and upgrade to write access at the outer perimeter. If this can
5968  * not be done without blocking then queue the callback to be done
5969  * by the qwriter_outer_thread.
5970  *
5971  * This routine can only be called from put or service procedures plus
5972  * asynchronous callback routines that have properly entered the queue (with
5973  * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5974  * associated with q.
5975  */
5976 void
5977 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
5978 {
5979         syncq_t *osq, *sq, *outer;
5980         int     failed;
5981         uint16_t flags;
5982 
5983         osq = q->q_syncq;
5984         outer = osq->sq_outer;
5985         if (outer == NULL)
5986                 panic("qwriter(PERIM_OUTER): no outer perimeter");
5987         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5988             outer->sq_oprev != NULL);
5989 
5990         mutex_enter(SQLOCK(outer));
5991         flags = outer->sq_flags;
5992         /*
5993          * If some thread is traversing sq_next, or if we are blocked by
5994          * outer_insert or outer_remove, or if the we already have queued
5995          * callbacks, then queue this callback for later processing.
5996          *
5997          * Also queue the qwriter for an interrupt thread in order
5998          * to reduce the time spent running at high IPL.
5999          * to identify there are events.
6000          */
6001         if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6002                 /*
6003                  * Queue the become_writer request.
6004                  * The queueing is atomic under SQLOCK(outer) in order
6005                  * to synchronize with outer_exit.
6006                  * queue_writer will drop the outer SQLOCK
6007                  */
6008                 if (flags & SQ_BLOCKED) {
6009                         /* Must set SQ_WRITER on inner perimeter */
6010                         mutex_enter(SQLOCK(osq));
6011                         osq->sq_flags |= SQ_WRITER;
6012                         mutex_exit(SQLOCK(osq));
6013                 } else {
6014                         if (!(flags & SQ_WRITER)) {
6015                                 /*
6016                                  * The outer could have been SQ_BLOCKED thus
6017                                  * SQ_WRITER might not be set on the inner.
6018                                  */
6019                                 mutex_enter(SQLOCK(osq));
6020                                 osq->sq_flags |= SQ_WRITER;
6021                                 mutex_exit(SQLOCK(osq));
6022                         }
6023                         ASSERT(osq->sq_flags & SQ_WRITER);
6024                 }
6025                 queue_writer(outer, func, q, mp);
6026                 return;
6027         }
6028         /*
6029          * We are half-way to exclusive access to the outer perimeter.
6030          * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6031          * while the inner syncqs are traversed.
6032          */
6033         outer->sq_count++;
6034         ASSERT(outer->sq_count != 0);        /* wraparound */
6035         flags |= SQ_WRITER;
6036         /*
6037          * Check if we can run the function immediately. Mark all
6038          * syncqs with the writer flag to prevent new entries into
6039          * put and service procedures.
6040          *
6041          * Set SQ_WRITER on all the inner syncqs while holding
6042          * the SQLOCK on the outer syncq. This ensures that the changing
6043          * of SQ_WRITER is atomic under the outer SQLOCK.
6044          */
6045         failed = 0;
6046         for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6047                 uint16_t count;
6048                 uint_t  maxcnt = (sq == osq) ? 1 : 0;
6049 
6050                 mutex_enter(SQLOCK(sq));
6051                 count = sq->sq_count;
6052                 SQ_PUTLOCKS_ENTER(sq);
6053                 SUM_SQ_PUTCOUNTS(sq, count);
6054                 if (sq->sq_count > maxcnt)
6055                         failed = 1;
6056                 sq->sq_flags |= SQ_WRITER;
6057                 SQ_PUTLOCKS_EXIT(sq);
6058                 mutex_exit(SQLOCK(sq));
6059         }
6060         if (failed) {
6061                 /*
6062                  * Some other thread has a read claim on the outer perimeter.
6063                  * Queue the callback for deferred processing.
6064                  *
6065                  * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6066                  * so that other qwriter(OUTER) calls will queue their
6067                  * callbacks as well. queue_writer increments sq_count so we
6068                  * decrement to compensate for the our increment.
6069                  *
6070                  * Dropping SQ_WRITER enables the writer thread to work
6071                  * on this outer perimeter.
6072                  */
6073                 outer->sq_flags = flags;
6074                 queue_writer(outer, func, q, mp);
6075                 /* queue_writer dropper the lock */
6076                 mutex_enter(SQLOCK(outer));
6077                 ASSERT(outer->sq_count > 0);
6078                 outer->sq_count--;
6079                 ASSERT(outer->sq_flags & SQ_WRITER);
6080                 flags = outer->sq_flags;
6081                 flags &= ~SQ_WRITER;
6082                 if (flags & SQ_WANTWAKEUP) {
6083                         flags &= ~SQ_WANTWAKEUP;
6084                         cv_broadcast(&outer->sq_wait);
6085                 }
6086                 outer->sq_flags = flags;
6087                 mutex_exit(SQLOCK(outer));
6088                 return;
6089         } else {
6090                 outer->sq_flags = flags;
6091                 mutex_exit(SQLOCK(outer));
6092         }
6093 
6094         /* Can run it immediately */
6095         (*func)(q, mp);
6096 
6097         outer_exit(outer);
6098 }
6099 
6100 /*
6101  * Dequeue all writer callbacks from the outer perimeter and run them.
6102  */
6103 static void
6104 write_now(syncq_t *outer)
6105 {
6106         mblk_t          *mp;
6107         queue_t         *q;
6108         void    (*func)();
6109 
6110         ASSERT(MUTEX_HELD(SQLOCK(outer)));
6111         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6112             outer->sq_oprev != NULL);
6113         while ((mp = outer->sq_evhead) != NULL) {
6114                 /*
6115                  * queues cannot be placed on the queuelist on the outer
6116                  * perimeter.
6117                  */
6118                 ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6119                 ASSERT((outer->sq_flags & SQ_EVENTS));
6120 
6121                 outer->sq_evhead = mp->b_next;
6122                 if (outer->sq_evhead == NULL) {
6123                         outer->sq_evtail = NULL;
6124                         outer->sq_flags &= ~SQ_EVENTS;
6125                 }
6126                 ASSERT(outer->sq_count != 0);
6127                 outer->sq_count--;   /* Incremented when enqueued. */
6128                 mutex_exit(SQLOCK(outer));
6129                 /*
6130                  * Drop the message if the queue is closing.
6131                  * Make sure that the queue is "claimed" when the callback
6132                  * is run in order to satisfy various ASSERTs.
6133                  */
6134                 q = mp->b_queue;
6135                 func = (void (*)())mp->b_prev;
6136                 ASSERT(func != NULL);
6137                 mp->b_next = mp->b_prev = NULL;
6138                 if (q->q_flag & QWCLOSE) {
6139                         freemsg(mp);
6140                 } else {
6141                         claimq(q);
6142                         (*func)(q, mp);
6143                         releaseq(q);
6144                 }
6145                 mutex_enter(SQLOCK(outer));
6146         }
6147         ASSERT(MUTEX_HELD(SQLOCK(outer)));
6148 }
6149 
6150 /*
6151  * The list of messages on the inner syncq is effectively hashed
6152  * by destination queue.  These destination queues are doubly
6153  * linked lists (hopefully) in priority order.  Messages are then
6154  * put on the queue referenced by the q_sqhead/q_sqtail elements.
6155  * Additional messages are linked together by the b_next/b_prev
6156  * elements in the mblk, with (similar to putq()) the first message
6157  * having a NULL b_prev and the last message having a NULL b_next.
6158  *
6159  * Events, such as qwriter callbacks, are put onto a list in FIFO
6160  * order referenced by sq_evhead, and sq_evtail.  This is a singly
6161  * linked list, and messages here MUST be processed in the order queued.
6162  */
6163 
6164 /*
6165  * Run the events on the syncq event list (sq_evhead).
6166  * Assumes there is only one claim on the syncq, it is
6167  * already exclusive (SQ_EXCL set), and the SQLOCK held.
6168  * Messages here are processed in order, with the SQ_EXCL bit
6169  * held all the way through till the last message is processed.
6170  */
6171 void
6172 sq_run_events(syncq_t *sq)
6173 {
6174         mblk_t          *bp;
6175         queue_t         *qp;
6176         uint16_t        flags = sq->sq_flags;
6177         void            (*func)();
6178 
6179         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6180         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6181             sq->sq_oprev == NULL) ||
6182             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6183             sq->sq_oprev != NULL));
6184 
6185         ASSERT(flags & SQ_EXCL);
6186         ASSERT(sq->sq_count == 1);
6187 
6188         /*
6189          * We need to process all of the events on this list.  It
6190          * is possible that new events will be added while we are
6191          * away processing a callback, so on every loop, we start
6192          * back at the beginning of the list.
6193          */
6194         /*
6195          * We have to reaccess sq_evhead since there is a
6196          * possibility of a new entry while we were running
6197          * the callback.
6198          */
6199         for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6200                 ASSERT(bp->b_queue->q_syncq == sq);
6201                 ASSERT(sq->sq_flags & SQ_EVENTS);
6202 
6203                 qp = bp->b_queue;
6204                 func = (void (*)())bp->b_prev;
6205                 ASSERT(func != NULL);
6206 
6207                 /*
6208                  * Messages from the event queue must be taken off in
6209                  * FIFO order.
6210                  */
6211                 ASSERT(sq->sq_evhead == bp);
6212                 sq->sq_evhead = bp->b_next;
6213 
6214                 if (bp->b_next == NULL) {
6215                         /* Deleting last */
6216                         ASSERT(sq->sq_evtail == bp);
6217                         sq->sq_evtail = NULL;
6218                         sq->sq_flags &= ~SQ_EVENTS;
6219                 }
6220                 bp->b_prev = bp->b_next = NULL;
6221                 ASSERT(bp->b_datap->db_ref != 0);
6222 
6223                 mutex_exit(SQLOCK(sq));
6224 
6225                 (*func)(qp, bp);
6226 
6227                 mutex_enter(SQLOCK(sq));
6228                 /*
6229                  * re-read the flags, since they could have changed.
6230                  */
6231                 flags = sq->sq_flags;
6232                 ASSERT(flags & SQ_EXCL);
6233         }
6234         ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6235         ASSERT(!(sq->sq_flags & SQ_EVENTS));
6236 
6237         if (flags & SQ_WANTWAKEUP) {
6238                 flags &= ~SQ_WANTWAKEUP;
6239                 cv_broadcast(&sq->sq_wait);
6240         }
6241         if (flags & SQ_WANTEXWAKEUP) {
6242                 flags &= ~SQ_WANTEXWAKEUP;
6243                 cv_broadcast(&sq->sq_exitwait);
6244         }
6245         sq->sq_flags = flags;
6246 }
6247 
6248 /*
6249  * Put messages on the event list.
6250  * If we can go exclusive now, do so and process the event list, otherwise
6251  * let the last claim service this list (or wake the sqthread).
6252  * This procedure assumes SQLOCK is held.  To run the event list, it
6253  * must be called with no claims.
6254  */
6255 static void
6256 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6257 {
6258         uint16_t count;
6259 
6260         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6261         ASSERT(func != NULL);
6262 
6263         /*
6264          * This is a callback.  Add it to the list of callbacks
6265          * and see about upgrading.
6266          */
6267         mp->b_prev = (mblk_t *)func;
6268         mp->b_queue = q;
6269         mp->b_next = NULL;
6270         if (sq->sq_evhead == NULL) {
6271                 sq->sq_evhead = sq->sq_evtail = mp;
6272                 sq->sq_flags |= SQ_EVENTS;
6273         } else {
6274                 ASSERT(sq->sq_evtail != NULL);
6275                 ASSERT(sq->sq_evtail->b_next == NULL);
6276                 ASSERT(sq->sq_flags & SQ_EVENTS);
6277                 sq->sq_evtail->b_next = mp;
6278                 sq->sq_evtail = mp;
6279         }
6280         /*
6281          * We have set SQ_EVENTS, so threads will have to
6282          * unwind out of the perimeter, and new entries will
6283          * not grab a putlock.  But we still need to know
6284          * how many threads have already made a claim to the
6285          * syncq, so grab the putlocks, and sum the counts.
6286          * If there are no claims on the syncq, we can upgrade
6287          * to exclusive, and run the event list.
6288          * NOTE: We hold the SQLOCK, so we can just grab the
6289          * putlocks.
6290          */
6291         count = sq->sq_count;
6292         SQ_PUTLOCKS_ENTER(sq);
6293         SUM_SQ_PUTCOUNTS(sq, count);
6294         /*
6295          * We have no claim, so we need to check if there
6296          * are no others, then we can upgrade.
6297          */
6298         /*
6299          * There are currently no claims on
6300          * the syncq by this thread (at least on this entry). The thread who has
6301          * the claim should drain syncq.
6302          */
6303         if (count > 0) {
6304                 /*
6305                  * Can't upgrade - other threads inside.
6306                  */
6307                 SQ_PUTLOCKS_EXIT(sq);
6308                 mutex_exit(SQLOCK(sq));
6309                 return;
6310         }
6311         /*
6312          * Need to set SQ_EXCL and make a claim on the syncq.
6313          */
6314         ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6315         sq->sq_flags |= SQ_EXCL;
6316         ASSERT(sq->sq_count == 0);
6317         sq->sq_count++;
6318         SQ_PUTLOCKS_EXIT(sq);
6319 
6320         /* Process the events list */
6321         sq_run_events(sq);
6322 
6323         /*
6324          * Release our claim...
6325          */
6326         sq->sq_count--;
6327 
6328         /*
6329          * And release SQ_EXCL.
6330          * We don't need to acquire the putlocks to release
6331          * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6332          */
6333         sq->sq_flags &= ~SQ_EXCL;
6334 
6335         /*
6336          * sq_run_events should have released SQ_EXCL
6337          */
6338         ASSERT(!(sq->sq_flags & SQ_EXCL));
6339 
6340         /*
6341          * If anything happened while we were running the
6342          * events (or was there before), we need to process
6343          * them now.  We shouldn't be exclusive sine we
6344          * released the perimeter above (plus, we asserted
6345          * for it).
6346          */
6347         if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6348                 drain_syncq(sq);
6349         else
6350                 mutex_exit(SQLOCK(sq));
6351 }
6352 
6353 /*
6354  * Perform delayed processing. The caller has to make sure that it is safe
6355  * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6356  * set).
6357  *
6358  * Assume that the caller has NO claims on the syncq.  However, a claim
6359  * on the syncq does not indicate that a thread is draining the syncq.
6360  * There may be more claims on the syncq than there are threads draining
6361  * (i.e.  #_threads_draining <= sq_count)
6362  *
6363  * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6364  * in order to preserve qwriter(OUTER) ordering constraints.
6365  *
6366  * sq_putcount only needs to be checked when dispatching the queued
6367  * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6368  */
6369 void
6370 drain_syncq(syncq_t *sq)
6371 {
6372         queue_t         *qp;
6373         uint16_t        count;
6374         uint16_t        type = sq->sq_type;
6375         uint16_t        flags = sq->sq_flags;
6376         boolean_t       bg_service = sq->sq_svcflags & SQ_SERVICE;
6377 
6378         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6379             "drain_syncq start:%p", sq);
6380         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6381         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6382             sq->sq_oprev == NULL) ||
6383             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6384             sq->sq_oprev != NULL));
6385 
6386         /*
6387          * Drop SQ_SERVICE flag.
6388          */
6389         if (bg_service)
6390                 sq->sq_svcflags &= ~SQ_SERVICE;
6391 
6392         /*
6393          * If SQ_EXCL is set, someone else is processing this syncq - let him
6394          * finish the job.
6395          */
6396         if (flags & SQ_EXCL) {
6397                 if (bg_service) {
6398                         ASSERT(sq->sq_servcount != 0);
6399                         sq->sq_servcount--;
6400                 }
6401                 mutex_exit(SQLOCK(sq));
6402                 return;
6403         }
6404 
6405         /*
6406          * This routine can be called by a background thread if
6407          * it was scheduled by a hi-priority thread.  SO, if there are
6408          * NOT messages queued, return (remember, we have the SQLOCK,
6409          * and it cannot change until we release it). Wakeup any waiters also.
6410          */
6411         if (!(flags & SQ_QUEUED)) {
6412                 if (flags & SQ_WANTWAKEUP) {
6413                         flags &= ~SQ_WANTWAKEUP;
6414                         cv_broadcast(&sq->sq_wait);
6415                 }
6416                 if (flags & SQ_WANTEXWAKEUP) {
6417                         flags &= ~SQ_WANTEXWAKEUP;
6418                         cv_broadcast(&sq->sq_exitwait);
6419                 }
6420                 sq->sq_flags = flags;
6421                 if (bg_service) {
6422                         ASSERT(sq->sq_servcount != 0);
6423                         sq->sq_servcount--;
6424                 }
6425                 mutex_exit(SQLOCK(sq));
6426                 return;
6427         }
6428 
6429         /*
6430          * If this is not a concurrent put perimeter, we need to
6431          * become exclusive to drain.  Also, if not CIPUT, we would
6432          * not have acquired a putlock, so we don't need to check
6433          * the putcounts.  If not entering with a claim, we test
6434          * for sq_count == 0.
6435          */
6436         type = sq->sq_type;
6437         if (!(type & SQ_CIPUT)) {
6438                 if (sq->sq_count > 1) {
6439                         if (bg_service) {
6440                                 ASSERT(sq->sq_servcount != 0);
6441                                 sq->sq_servcount--;
6442                         }
6443                         mutex_exit(SQLOCK(sq));
6444                         return;
6445                 }
6446                 sq->sq_flags |= SQ_EXCL;
6447         }
6448 
6449         /*
6450          * This is where we make a claim to the syncq.
6451          * This can either be done by incrementing a putlock, or
6452          * the sq_count.  But since we already have the SQLOCK
6453          * here, we just bump the sq_count.
6454          *
6455          * Note that after we make a claim, we need to let the code
6456          * fall through to the end of this routine to clean itself
6457          * up.  A return in the while loop will put the syncq in a
6458          * very bad state.
6459          */
6460         sq->sq_count++;
6461         ASSERT(sq->sq_count != 0);   /* wraparound */
6462 
6463         while ((flags = sq->sq_flags) & SQ_QUEUED) {
6464                 /*
6465                  * If we are told to stayaway or went exclusive,
6466                  * we are done.
6467                  */
6468                 if (flags & (SQ_STAYAWAY)) {
6469                         break;
6470                 }
6471 
6472                 /*
6473                  * If there are events to run, do so.
6474                  * We have one claim to the syncq, so if there are
6475                  * more than one, other threads are running.
6476                  */
6477                 if (sq->sq_evhead != NULL) {
6478                         ASSERT(sq->sq_flags & SQ_EVENTS);
6479 
6480                         count = sq->sq_count;
6481                         SQ_PUTLOCKS_ENTER(sq);
6482                         SUM_SQ_PUTCOUNTS(sq, count);
6483                         if (count > 1) {
6484                                 SQ_PUTLOCKS_EXIT(sq);
6485                                 /* Can't upgrade - other threads inside */
6486                                 break;
6487                         }
6488                         ASSERT((flags & SQ_EXCL) == 0);
6489                         sq->sq_flags = flags | SQ_EXCL;
6490                         SQ_PUTLOCKS_EXIT(sq);
6491                         /*
6492                          * we have the only claim, run the events,
6493                          * sq_run_events will clear the SQ_EXCL flag.
6494                          */
6495                         sq_run_events(sq);
6496 
6497                         /*
6498                          * If this is a CIPUT perimeter, we need
6499                          * to drop the SQ_EXCL flag so we can properly
6500                          * continue draining the syncq.
6501                          */
6502                         if (type & SQ_CIPUT) {
6503                                 ASSERT(sq->sq_flags & SQ_EXCL);
6504                                 sq->sq_flags &= ~SQ_EXCL;
6505                         }
6506 
6507                         /*
6508                          * And go back to the beginning just in case
6509                          * anything changed while we were away.
6510                          */
6511                         ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6512                         continue;
6513                 }
6514 
6515                 ASSERT(sq->sq_evhead == NULL);
6516                 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6517 
6518                 /*
6519                  * Find the queue that is not draining.
6520                  *
6521                  * q_draining is protected by QLOCK which we do not hold.
6522                  * But if it was set, then a thread was draining, and if it gets
6523                  * cleared, then it was because the thread has successfully
6524                  * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6525                  * state to happen, a thread needs the SQLOCK which we hold, and
6526                  * if there was such a flag, we would have already seen it.
6527                  */
6528 
6529                 for (qp = sq->sq_head;
6530                     qp != NULL && (qp->q_draining ||
6531                     (qp->q_sqflags & Q_SQDRAINING));
6532                     qp = qp->q_sqnext)
6533                         ;
6534 
6535                 if (qp == NULL)
6536                         break;
6537 
6538                 /*
6539                  * We have a queue to work on, and we hold the
6540                  * SQLOCK and one claim, call qdrain_syncq.
6541                  * This means we need to release the SQLOCK and
6542                  * acquire the QLOCK (OK since we have a claim).
6543                  * Note that qdrain_syncq will actually dequeue
6544                  * this queue from the sq_head list when it is
6545                  * convinced all the work is done and release
6546                  * the QLOCK before returning.
6547                  */
6548                 qp->q_sqflags |= Q_SQDRAINING;
6549                 mutex_exit(SQLOCK(sq));
6550                 mutex_enter(QLOCK(qp));
6551                 qdrain_syncq(sq, qp);
6552                 mutex_enter(SQLOCK(sq));
6553 
6554                 /* The queue is drained */
6555                 ASSERT(qp->q_sqflags & Q_SQDRAINING);
6556                 qp->q_sqflags &= ~Q_SQDRAINING;
6557                 /*
6558                  * NOTE: After this point qp should not be used since it may be
6559                  * closed.
6560                  */
6561         }
6562 
6563         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6564         flags = sq->sq_flags;
6565 
6566         /*
6567          * sq->sq_head cannot change because we hold the
6568          * sqlock. However, a thread CAN decide that it is no longer
6569          * going to drain that queue.  However, this should be due to
6570          * a GOAWAY state, and we should see that here.
6571          *
6572          * This loop is not very efficient. One solution may be adding a second
6573          * pointer to the "draining" queue, but it is difficult to do when
6574          * queues are inserted in the middle due to priority ordering. Another
6575          * possibility is to yank the queue out of the sq list and put it onto
6576          * the "draining list" and then put it back if it can't be drained.
6577          */
6578 
6579         ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6580             (type & SQ_CI) || sq->sq_head->q_draining);
6581 
6582         /* Drop SQ_EXCL for non-CIPUT perimeters */
6583         if (!(type & SQ_CIPUT))
6584                 flags &= ~SQ_EXCL;
6585         ASSERT((flags & SQ_EXCL) == 0);
6586 
6587         /* Wake up any waiters. */
6588         if (flags & SQ_WANTWAKEUP) {
6589                 flags &= ~SQ_WANTWAKEUP;
6590                 cv_broadcast(&sq->sq_wait);
6591         }
6592         if (flags & SQ_WANTEXWAKEUP) {
6593                 flags &= ~SQ_WANTEXWAKEUP;
6594                 cv_broadcast(&sq->sq_exitwait);
6595         }
6596         sq->sq_flags = flags;
6597 
6598         ASSERT(sq->sq_count != 0);
6599         /* Release our claim. */
6600         sq->sq_count--;
6601 
6602         if (bg_service) {
6603                 ASSERT(sq->sq_servcount != 0);
6604                 sq->sq_servcount--;
6605         }
6606 
6607         mutex_exit(SQLOCK(sq));
6608 
6609         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6610             "drain_syncq end:%p", sq);
6611 }
6612 
6613 
6614 /*
6615  *
6616  * qdrain_syncq can be called (currently) from only one of two places:
6617  *      drain_syncq
6618  *      putnext  (or some variation of it).
6619  * and eventually
6620  *      qwait(_sig)
6621  *
6622  * If called from drain_syncq, we found it in the list of queues needing
6623  * service, so there is work to be done (or it wouldn't be in the list).
6624  *
6625  * If called from some putnext variation, it was because the
6626  * perimeter is open, but messages are blocking a putnext and
6627  * there is not a thread working on it.  Now a thread could start
6628  * working on it while we are getting ready to do so ourself, but
6629  * the thread would set the q_draining flag, and we can spin out.
6630  *
6631  * As for qwait(_sig), I think I shall let it continue to call
6632  * drain_syncq directly (after all, it will get here eventually).
6633  *
6634  * qdrain_syncq has to terminate when:
6635  * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6636  * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6637  *
6638  * ASSUMES:
6639  *      One claim
6640  *      QLOCK held
6641  *      SQLOCK not held
6642  *      Will release QLOCK before returning
6643  */
6644 void
6645 qdrain_syncq(syncq_t *sq, queue_t *q)
6646 {
6647         mblk_t          *bp;
6648 #ifdef DEBUG
6649         uint16_t        count;
6650 #endif
6651 
6652         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6653             "drain_syncq start:%p", sq);
6654         ASSERT(q->q_syncq == sq);
6655         ASSERT(MUTEX_HELD(QLOCK(q)));
6656         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6657         /*
6658          * For non-CIPUT perimeters, we should be called with the exclusive bit
6659          * set already. For CIPUT perimeters, we will be doing a concurrent
6660          * drain, so it better not be set.
6661          */
6662         ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6663         ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6664         ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6665         /*
6666          * All outer pointers are set, or none of them are
6667          */
6668         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6669             sq->sq_oprev == NULL) ||
6670             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6671             sq->sq_oprev != NULL));
6672 #ifdef DEBUG
6673         count = sq->sq_count;
6674         /*
6675          * This is OK without the putlocks, because we have one
6676          * claim either from the sq_count, or a putcount.  We could
6677          * get an erroneous value from other counts, but ours won't
6678          * change, so one way or another, we will have at least a
6679          * value of one.
6680          */
6681         SUM_SQ_PUTCOUNTS(sq, count);
6682         ASSERT(count >= 1);
6683 #endif /* DEBUG */
6684 
6685         /*
6686          * The first thing to do is find out if a thread is already draining
6687          * this queue. If so, we are done, just return.
6688          */
6689         if (q->q_draining) {
6690                 mutex_exit(QLOCK(q));
6691                 return;
6692         }
6693 
6694         /*
6695          * If the perimeter is exclusive, there is nothing we can do right now,
6696          * go away. Note that there is nothing to prevent this case from
6697          * changing right after this check, but the spin-out will catch it.
6698          */
6699 
6700         /* Tell other threads that we are draining this queue */
6701         q->q_draining = 1;   /* Protected by QLOCK */
6702 
6703         /*
6704          * If there is nothing to do, clear QFULL as necessary. This caters for
6705          * the case where an empty queue was enqueued onto the syncq.
6706          */
6707         if (q->q_sqhead == NULL) {
6708                 ASSERT(q->q_syncqmsgs == 0);
6709                 mutex_exit(QLOCK(q));
6710                 clr_qfull(q);
6711                 mutex_enter(QLOCK(q));
6712         }
6713 
6714         /*
6715          * Note that q_sqhead must be re-checked here in case another message
6716          * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6717          */
6718         for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6719                 /*
6720                  * Because we can enter this routine just because a putnext is
6721                  * blocked, we need to spin out if the perimeter wants to go
6722                  * exclusive as well as just blocked. We need to spin out also
6723                  * if events are queued on the syncq.
6724                  * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6725                  * set it, and it can't become exclusive while we hold a claim.
6726                  */
6727                 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6728                         break;
6729                 }
6730 
6731 #ifdef DEBUG
6732                 /*
6733                  * Since we are in qdrain_syncq, we already know the queue,
6734                  * but for sanity, we want to check this against the qp that
6735                  * was passed in by bp->b_queue.
6736                  */
6737 
6738                 ASSERT(bp->b_queue == q);
6739                 ASSERT(bp->b_queue->q_syncq == sq);
6740                 bp->b_queue = NULL;
6741 
6742                 /*
6743                  * We would have the following check in the DEBUG code:
6744                  *
6745                  * if (bp->b_prev != NULL)  {
6746                  *      ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6747                  * }
6748                  *
6749                  * This can't be done, however, since IP modifies qinfo
6750                  * structure at run-time (switching between IPv4 qinfo and IPv6
6751                  * qinfo), invalidating the check.
6752                  * So the assignment to func is left here, but the ASSERT itself
6753                  * is removed until the whole issue is resolved.
6754                  */
6755 #endif
6756                 ASSERT(q->q_sqhead == bp);
6757                 q->q_sqhead = bp->b_next;
6758                 bp->b_prev = bp->b_next = NULL;
6759                 ASSERT(q->q_syncqmsgs > 0);
6760                 mutex_exit(QLOCK(q));
6761 
6762                 ASSERT(bp->b_datap->db_ref != 0);
6763 
6764                 (void) (*q->q_qinfo->qi_putp)(q, bp);
6765 
6766                 mutex_enter(QLOCK(q));
6767 
6768                 /*
6769                  * q_syncqmsgs should only be decremented after executing the
6770                  * put procedure to avoid message re-ordering. This is due to an
6771                  * optimisation in putnext() which can call the put procedure
6772                  * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6773                  * being set).
6774                  *
6775                  * We also need to clear QFULL in the next service procedure
6776                  * queue if this is the last message destined for that queue.
6777                  *
6778                  * It would make better sense to have some sort of tunable for
6779                  * the low water mark, but these semantics are not yet defined.
6780                  * So, alas, we use a constant.
6781                  */
6782                 if (--q->q_syncqmsgs == 0) {
6783                         mutex_exit(QLOCK(q));
6784                         clr_qfull(q);
6785                         mutex_enter(QLOCK(q));
6786                 }
6787 
6788                 /*
6789                  * Always clear SQ_EXCL when CIPUT in order to handle
6790                  * qwriter(INNER). The putp() can call qwriter and get exclusive
6791                  * access IFF this is the only claim. So, we need to test for
6792                  * this possibility, acquire the mutex and clear the bit.
6793                  */
6794                 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6795                         mutex_enter(SQLOCK(sq));
6796                         sq->sq_flags &= ~SQ_EXCL;
6797                         mutex_exit(SQLOCK(sq));
6798                 }
6799         }
6800 
6801         /*
6802          * We should either have no messages on this queue, or we were told to
6803          * goaway by a waiter (which we will wake up at the end of this
6804          * function).
6805          */
6806         ASSERT((q->q_sqhead == NULL) ||
6807             (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6808 
6809         ASSERT(MUTEX_HELD(QLOCK(q)));
6810         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6811 
6812         /* Remove the q from the syncq list if all the messages are drained. */
6813         if (q->q_sqhead == NULL) {
6814                 ASSERT(q->q_syncqmsgs == 0);
6815                 mutex_enter(SQLOCK(sq));
6816                 if (q->q_sqflags & Q_SQQUEUED)
6817                         SQRM_Q(sq, q);
6818                 mutex_exit(SQLOCK(sq));
6819                 /*
6820                  * Since the queue is removed from the list, reset its priority.
6821                  */
6822                 q->q_spri = 0;
6823         }
6824 
6825         /*
6826          * Remember, the q_draining flag is used to let another thread know
6827          * that there is a thread currently draining the messages for a queue.
6828          * Since we are now done with this queue (even if there may be messages
6829          * still there), we need to clear this flag so some thread will work on
6830          * it if needed.
6831          */
6832         ASSERT(q->q_draining);
6833         q->q_draining = 0;
6834 
6835         /* Called with a claim, so OK to drop all locks. */
6836         mutex_exit(QLOCK(q));
6837 
6838         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6839             "drain_syncq end:%p", sq);
6840 }
6841 /* END OF QDRAIN_SYNCQ  */
6842 
6843 
6844 /*
6845  * This is the mate to qdrain_syncq, except that it is putting the message onto
6846  * the queue instead of draining. Since the message is destined for the queue
6847  * that is selected, there is no need to identify the function because the
6848  * message is intended for the put routine for the queue. For debug kernels,
6849  * this routine will do it anyway just in case.
6850  *
6851  * After the message is enqueued on the syncq, it calls putnext_tail()
6852  * which will schedule a background thread to actually process the message.
6853  *
6854  * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6855  * SQLOCK(sq) and QLOCK(q) are not held.
6856  */
6857 void
6858 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6859 {
6860         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6861         ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6862         ASSERT(sq->sq_count > 0);
6863         ASSERT(q->q_syncq == sq);
6864         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6865             sq->sq_oprev == NULL) ||
6866             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6867             sq->sq_oprev != NULL));
6868 
6869         mutex_enter(QLOCK(q));
6870 
6871 #ifdef DEBUG
6872         /*
6873          * This is used for debug in the qfill_syncq/qdrain_syncq case
6874          * to trace the queue that the message is intended for.  Note
6875          * that the original use was to identify the queue and function
6876          * to call on the drain.  In the new syncq, we have the context
6877          * of the queue that we are draining, so call it's putproc and
6878          * don't rely on the saved values.  But for debug this is still
6879          * useful information.
6880          */
6881         mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6882         mp->b_queue = q;
6883         mp->b_next = NULL;
6884 #endif
6885         ASSERT(q->q_syncq == sq);
6886         /*
6887          * Enqueue the message on the list.
6888          * SQPUT_MP() accesses q_syncqmsgs.  We are already holding QLOCK to
6889          * protect it.  So it's ok to acquire SQLOCK after SQPUT_MP().
6890          */
6891         SQPUT_MP(q, mp);
6892         mutex_enter(SQLOCK(sq));
6893 
6894         /*
6895          * And queue on syncq for scheduling, if not already queued.
6896          * Note that we need the SQLOCK for this, and for testing flags
6897          * at the end to see if we will drain.  So grab it now, and
6898          * release it before we call qdrain_syncq or return.
6899          */
6900         if (!(q->q_sqflags & Q_SQQUEUED)) {
6901                 q->q_spri = curthread->t_pri;
6902                 SQPUT_Q(sq, q);
6903         }
6904 #ifdef DEBUG
6905         else {
6906                 /*
6907                  * All of these conditions MUST be true!
6908                  */
6909                 ASSERT(sq->sq_tail != NULL);
6910                 if (sq->sq_tail == sq->sq_head) {
6911                         ASSERT((q->q_sqprev == NULL) &&
6912                             (q->q_sqnext == NULL));
6913                 } else {
6914                         ASSERT((q->q_sqprev != NULL) ||
6915                             (q->q_sqnext != NULL));
6916                 }
6917                 ASSERT(sq->sq_flags & SQ_QUEUED);
6918                 ASSERT(q->q_syncqmsgs != 0);
6919                 ASSERT(q->q_sqflags & Q_SQQUEUED);
6920         }
6921 #endif
6922         mutex_exit(QLOCK(q));
6923         /*
6924          * SQLOCK is still held, so sq_count can be safely decremented.
6925          */
6926         sq->sq_count--;
6927 
6928         putnext_tail(sq, q, 0);
6929         /* Should not reference sq or q after this point. */
6930 }
6931 
6932 /*  End of qfill_syncq  */
6933 
6934 /*
6935  * Remove all messages from a syncq (if qp is NULL) or remove all messages
6936  * that would be put into qp by drain_syncq.
6937  * Used when deleting the syncq (qp == NULL) or when detaching
6938  * a queue (qp != NULL).
6939  * Return non-zero if one or more messages were freed.
6940  *
6941  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6942  * sq_putlocks are used.
6943  *
6944  * NOTE: This function assumes that it is called from the close() context and
6945  * that all the queues in the syncq are going away. For this reason it doesn't
6946  * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6947  * currently valid, but it is useful to rethink this function to behave properly
6948  * in other cases.
6949  */
6950 int
6951 flush_syncq(syncq_t *sq, queue_t *qp)
6952 {
6953         mblk_t          *bp, *mp_head, *mp_next, *mp_prev;
6954         queue_t         *q;
6955         int             ret = 0;
6956 
6957         mutex_enter(SQLOCK(sq));
6958 
6959         /*
6960          * Before we leave, we need to make sure there are no
6961          * events listed for this queue.  All events for this queue
6962          * will just be freed.
6963          */
6964         if (qp != NULL && sq->sq_evhead != NULL) {
6965                 ASSERT(sq->sq_flags & SQ_EVENTS);
6966 
6967                 mp_prev = NULL;
6968                 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6969                         mp_next = bp->b_next;
6970                         if (bp->b_queue == qp) {
6971                                 /* Delete this message */
6972                                 if (mp_prev != NULL) {
6973                                         mp_prev->b_next = mp_next;
6974                                         /*
6975                                          * Update sq_evtail if the last element
6976                                          * is removed.
6977                                          */
6978                                         if (bp == sq->sq_evtail) {
6979                                                 ASSERT(mp_next == NULL);
6980                                                 sq->sq_evtail = mp_prev;
6981                                         }
6982                                 } else
6983                                         sq->sq_evhead = mp_next;
6984                                 if (sq->sq_evhead == NULL)
6985                                         sq->sq_flags &= ~SQ_EVENTS;
6986                                 bp->b_prev = bp->b_next = NULL;
6987                                 freemsg(bp);
6988                                 ret++;
6989                         } else {
6990                                 mp_prev = bp;
6991                         }
6992                 }
6993         }
6994 
6995         /*
6996          * Walk sq_head and:
6997          *      - match qp if qp is set, remove it's messages
6998          *      - all if qp is not set
6999          */
7000         q = sq->sq_head;
7001         while (q != NULL) {
7002                 ASSERT(q->q_syncq == sq);
7003                 if ((qp == NULL) || (qp == q)) {
7004                         /*
7005                          * Yank the messages as a list off the queue
7006                          */
7007                         mp_head = q->q_sqhead;
7008                         /*
7009                          * We do not have QLOCK(q) here (which is safe due to
7010                          * assumptions mentioned above). To obtain the lock we
7011                          * need to release SQLOCK which may allow lots of things
7012                          * to change upon us. This place requires more analysis.
7013                          */
7014                         q->q_sqhead = q->q_sqtail = NULL;
7015                         ASSERT(mp_head->b_queue &&
7016                             mp_head->b_queue->q_syncq == sq);
7017 
7018                         /*
7019                          * Free each of the messages.
7020                          */
7021                         for (bp = mp_head; bp != NULL; bp = mp_next) {
7022                                 mp_next = bp->b_next;
7023                                 bp->b_prev = bp->b_next = NULL;
7024                                 freemsg(bp);
7025                                 ret++;
7026                         }
7027                         /*
7028                          * Now remove the queue from the syncq.
7029                          */
7030                         ASSERT(q->q_sqflags & Q_SQQUEUED);
7031                         SQRM_Q(sq, q);
7032                         q->q_spri = 0;
7033                         q->q_syncqmsgs = 0;
7034 
7035                         /*
7036                          * If qp was specified, we are done with it and are
7037                          * going to drop SQLOCK(sq) and return. We wakeup syncq
7038                          * waiters while we still have the SQLOCK.
7039                          */
7040                         if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7041                                 sq->sq_flags &= ~SQ_WANTWAKEUP;
7042                                 cv_broadcast(&sq->sq_wait);
7043                         }
7044                         /* Drop SQLOCK across clr_qfull */
7045                         mutex_exit(SQLOCK(sq));
7046 
7047                         /*
7048                          * We avoid doing the test that drain_syncq does and
7049                          * unconditionally clear qfull for every flushed
7050                          * message. Since flush_syncq is only called during
7051                          * close this should not be a problem.
7052                          */
7053                         clr_qfull(q);
7054                         if (qp != NULL) {
7055                                 return (ret);
7056                         } else {
7057                                 mutex_enter(SQLOCK(sq));
7058                                 /*
7059                                  * The head was removed by SQRM_Q above.
7060                                  * reread the new head and flush it.
7061                                  */
7062                                 q = sq->sq_head;
7063                         }
7064                 } else {
7065                         q = q->q_sqnext;
7066                 }
7067                 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7068         }
7069 
7070         if (sq->sq_flags & SQ_WANTWAKEUP) {
7071                 sq->sq_flags &= ~SQ_WANTWAKEUP;
7072                 cv_broadcast(&sq->sq_wait);
7073         }
7074 
7075         mutex_exit(SQLOCK(sq));
7076         return (ret);
7077 }
7078 
7079 /*
7080  * Propagate all messages from a syncq to the next syncq that are associated
7081  * with the specified queue. If the queue is attached to a driver or if the
7082  * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7083  *
7084  * Assumes that the stream is strlock()'ed. We don't come here if there
7085  * are no messages to propagate.
7086  *
7087  * NOTE : If the queue is attached to a driver, all the messages are freed
7088  * as there is no point in propagating the messages from the driver syncq
7089  * to the closing stream head which will in turn get freed later.
7090  */
7091 static int
7092 propagate_syncq(queue_t *qp)
7093 {
7094         mblk_t          *bp, *head, *tail, *prev, *next;
7095         syncq_t         *sq;
7096         queue_t         *nqp;
7097         syncq_t         *nsq;
7098         boolean_t       isdriver;
7099         int             moved = 0;
7100         uint16_t        flags;
7101         pri_t           priority = curthread->t_pri;
7102 #ifdef DEBUG
7103         void            (*func)();
7104 #endif
7105 
7106         sq = qp->q_syncq;
7107         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7108         /* debug macro */
7109         SQ_PUTLOCKS_HELD(sq);
7110         /*
7111          * As entersq() does not increment the sq_count for
7112          * the write side, check sq_count for non-QPERQ
7113          * perimeters alone.
7114          */
7115         ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7116 
7117         /*
7118          * propagate_syncq() can be called because of either messages on the
7119          * queue syncq or because on events on the queue syncq. Do actual
7120          * message propagations if there are any messages.
7121          */
7122         if (qp->q_syncqmsgs) {
7123                 isdriver = (qp->q_flag & QISDRV);
7124 
7125                 if (!isdriver) {
7126                         nqp = qp->q_next;
7127                         nsq = nqp->q_syncq;
7128                         ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7129                         /* debug macro */
7130                         SQ_PUTLOCKS_HELD(nsq);
7131 #ifdef DEBUG
7132                         func = (void (*)())nqp->q_qinfo->qi_putp;
7133 #endif
7134                 }
7135 
7136                 SQRM_Q(sq, qp);
7137                 priority = MAX(qp->q_spri, priority);
7138                 qp->q_spri = 0;
7139                 head = qp->q_sqhead;
7140                 tail = qp->q_sqtail;
7141                 qp->q_sqhead = qp->q_sqtail = NULL;
7142                 qp->q_syncqmsgs = 0;
7143 
7144                 /*
7145                  * Walk the list of messages, and free them if this is a driver,
7146                  * otherwise reset the b_prev and b_queue value to the new putp.
7147                  * Afterward, we will just add the head to the end of the next
7148                  * syncq, and point the tail to the end of this one.
7149                  */
7150 
7151                 for (bp = head; bp != NULL; bp = next) {
7152                         next = bp->b_next;
7153                         if (isdriver) {
7154                                 bp->b_prev = bp->b_next = NULL;
7155                                 freemsg(bp);
7156                                 continue;
7157                         }
7158                         /* Change the q values for this message */
7159                         bp->b_queue = nqp;
7160 #ifdef DEBUG
7161                         bp->b_prev = (mblk_t *)func;
7162 #endif
7163                         moved++;
7164                 }
7165                 /*
7166                  * Attach list of messages to the end of the new queue (if there
7167                  * is a list of messages).
7168                  */
7169 
7170                 if (!isdriver && head != NULL) {
7171                         ASSERT(tail != NULL);
7172                         if (nqp->q_sqhead == NULL) {
7173                                 nqp->q_sqhead = head;
7174                         } else {
7175                                 ASSERT(nqp->q_sqtail != NULL);
7176                                 nqp->q_sqtail->b_next = head;
7177                         }
7178                         nqp->q_sqtail = tail;
7179                         /*
7180                          * When messages are moved from high priority queue to
7181                          * another queue, the destination queue priority is
7182                          * upgraded.
7183                          */
7184 
7185                         if (priority > nqp->q_spri)
7186                                 nqp->q_spri = priority;
7187 
7188                         SQPUT_Q(nsq, nqp);
7189 
7190                         nqp->q_syncqmsgs += moved;
7191                         ASSERT(nqp->q_syncqmsgs != 0);
7192                 }
7193         }
7194 
7195         /*
7196          * Before we leave, we need to make sure there are no
7197          * events listed for this queue.  All events for this queue
7198          * will just be freed.
7199          */
7200         if (sq->sq_evhead != NULL) {
7201                 ASSERT(sq->sq_flags & SQ_EVENTS);
7202                 prev = NULL;
7203                 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7204                         next = bp->b_next;
7205                         if (bp->b_queue == qp) {
7206                                 /* Delete this message */
7207                                 if (prev != NULL) {
7208                                         prev->b_next = next;
7209                                         /*
7210                                          * Update sq_evtail if the last element
7211                                          * is removed.
7212                                          */
7213                                         if (bp == sq->sq_evtail) {
7214                                                 ASSERT(next == NULL);
7215                                                 sq->sq_evtail = prev;
7216                                         }
7217                                 } else
7218                                         sq->sq_evhead = next;
7219                                 if (sq->sq_evhead == NULL)
7220                                         sq->sq_flags &= ~SQ_EVENTS;
7221                                 bp->b_prev = bp->b_next = NULL;
7222                                 freemsg(bp);
7223                         } else {
7224                                 prev = bp;
7225                         }
7226                 }
7227         }
7228 
7229         flags = sq->sq_flags;
7230 
7231         /* Wake up any waiter before leaving. */
7232         if (flags & SQ_WANTWAKEUP) {
7233                 flags &= ~SQ_WANTWAKEUP;
7234                 cv_broadcast(&sq->sq_wait);
7235         }
7236         sq->sq_flags = flags;
7237 
7238         return (moved);
7239 }
7240 
7241 /*
7242  * Try and upgrade to exclusive access at the inner perimeter. If this can
7243  * not be done without blocking then request will be queued on the syncq
7244  * and drain_syncq will run it later.
7245  *
7246  * This routine can only be called from put or service procedures plus
7247  * asynchronous callback routines that have properly entered the queue (with
7248  * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7249  * associated with q.
7250  */
7251 void
7252 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7253 {
7254         syncq_t *sq = q->q_syncq;
7255         uint16_t count;
7256 
7257         mutex_enter(SQLOCK(sq));
7258         count = sq->sq_count;
7259         SQ_PUTLOCKS_ENTER(sq);
7260         SUM_SQ_PUTCOUNTS(sq, count);
7261         ASSERT(count >= 1);
7262         ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7263 
7264         if (count == 1) {
7265                 /*
7266                  * Can upgrade. This case also handles nested qwriter calls
7267                  * (when the qwriter callback function calls qwriter). In that
7268                  * case SQ_EXCL is already set.
7269                  */
7270                 sq->sq_flags |= SQ_EXCL;
7271                 SQ_PUTLOCKS_EXIT(sq);
7272                 mutex_exit(SQLOCK(sq));
7273                 (*func)(q, mp);
7274                 /*
7275                  * Assumes that leavesq, putnext, and drain_syncq will reset
7276                  * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7277                  * until putnext, leavesq, or drain_syncq drops it.
7278                  * That way we handle nested qwriter(INNER) without dropping
7279                  * SQ_EXCL until the outermost qwriter callback routine is
7280                  * done.
7281                  */
7282                 return;
7283         }
7284         SQ_PUTLOCKS_EXIT(sq);
7285         sqfill_events(sq, q, mp, func);
7286 }
7287 
7288 /*
7289  * Synchronous callback support functions
7290  */
7291 
7292 /*
7293  * Allocate a callback parameter structure.
7294  * Assumes that caller initializes the flags and the id.
7295  * Acquires SQLOCK(sq) if non-NULL is returned.
7296  */
7297 callbparams_t *
7298 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7299 {
7300         callbparams_t *cbp;
7301         size_t size = sizeof (callbparams_t);
7302 
7303         cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7304 
7305         /*
7306          * Only try tryhard allocation if the caller is ready to panic.
7307          * Otherwise just fail.
7308          */
7309         if (cbp == NULL) {
7310                 if (kmflags & KM_PANIC)
7311                         cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7312                             &size, kmflags);
7313                 else
7314                         return (NULL);
7315         }
7316 
7317         ASSERT(size >= sizeof (callbparams_t));
7318         cbp->cbp_size = size;
7319         cbp->cbp_sq = sq;
7320         cbp->cbp_func = func;
7321         cbp->cbp_arg = arg;
7322         mutex_enter(SQLOCK(sq));
7323         cbp->cbp_next = sq->sq_callbpend;
7324         sq->sq_callbpend = cbp;
7325         return (cbp);
7326 }
7327 
7328 void
7329 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7330 {
7331         callbparams_t **pp, *p;
7332 
7333         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7334 
7335         for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7336                 if (p == cbp) {
7337                         *pp = p->cbp_next;
7338                         kmem_free(p, p->cbp_size);
7339                         return;
7340                 }
7341         }
7342         (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7343             "callbparams_free: not found\n"));
7344 }
7345 
7346 void
7347 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7348 {
7349         callbparams_t **pp, *p;
7350 
7351         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7352 
7353         for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7354                 if (p->cbp_id == id && p->cbp_flags == flag) {
7355                         *pp = p->cbp_next;
7356                         kmem_free(p, p->cbp_size);
7357                         return;
7358                 }
7359         }
7360         (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7361             "callbparams_free_id: not found\n"));
7362 }
7363 
7364 /*
7365  * Callback wrapper function used by once-only callbacks that can be
7366  * cancelled (qtimeout and qbufcall)
7367  * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7368  * cancelled by the qun* functions.
7369  */
7370 void
7371 qcallbwrapper(void *arg)
7372 {
7373         callbparams_t *cbp = arg;
7374         syncq_t *sq;
7375         uint16_t count = 0;
7376         uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7377         uint16_t type;
7378 
7379         sq = cbp->cbp_sq;
7380         mutex_enter(SQLOCK(sq));
7381         type = sq->sq_type;
7382         if (!(type & SQ_CICB)) {
7383                 count = sq->sq_count;
7384                 SQ_PUTLOCKS_ENTER(sq);
7385                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7386                 SUM_SQ_PUTCOUNTS(sq, count);
7387                 sq->sq_needexcl++;
7388                 ASSERT(sq->sq_needexcl != 0);        /* wraparound */
7389                 waitflags |= SQ_MESSAGES;
7390         }
7391         /* Can not handle exclusive entry at outer perimeter */
7392         ASSERT(type & SQ_COCB);
7393 
7394         while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7395                 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7396                     (sq->sq_cancelid == cbp->cbp_id)) {
7397                         /* timeout has been cancelled */
7398                         sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7399                         callbparams_free(sq, cbp);
7400                         if (!(type & SQ_CICB)) {
7401                                 ASSERT(sq->sq_needexcl > 0);
7402                                 sq->sq_needexcl--;
7403                                 if (sq->sq_needexcl == 0) {
7404                                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7405                                 }
7406                                 SQ_PUTLOCKS_EXIT(sq);
7407                         }
7408                         mutex_exit(SQLOCK(sq));
7409                         return;
7410                 }
7411                 sq->sq_flags |= SQ_WANTWAKEUP;
7412                 if (!(type & SQ_CICB)) {
7413                         SQ_PUTLOCKS_EXIT(sq);
7414                 }
7415                 cv_wait(&sq->sq_wait, SQLOCK(sq));
7416                 if (!(type & SQ_CICB)) {
7417                         count = sq->sq_count;
7418                         SQ_PUTLOCKS_ENTER(sq);
7419                         SUM_SQ_PUTCOUNTS(sq, count);
7420                 }
7421         }
7422 
7423         sq->sq_count++;
7424         ASSERT(sq->sq_count != 0);   /* Wraparound */
7425         if (!(type & SQ_CICB)) {
7426                 ASSERT(count == 0);
7427                 sq->sq_flags |= SQ_EXCL;
7428                 ASSERT(sq->sq_needexcl > 0);
7429                 sq->sq_needexcl--;
7430                 if (sq->sq_needexcl == 0) {
7431                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7432                 }
7433                 SQ_PUTLOCKS_EXIT(sq);
7434         }
7435 
7436         mutex_exit(SQLOCK(sq));
7437 
7438         cbp->cbp_func(cbp->cbp_arg);
7439 
7440         /*
7441          * We drop the lock only for leavesq to re-acquire it.
7442          * Possible optimization is inline of leavesq.
7443          */
7444         mutex_enter(SQLOCK(sq));
7445         callbparams_free(sq, cbp);
7446         mutex_exit(SQLOCK(sq));
7447         leavesq(sq, SQ_CALLBACK);
7448 }
7449 
7450 /*
7451  * No need to grab sq_putlocks here. See comment in strsubr.h that
7452  * explains when sq_putlocks are used.
7453  *
7454  * sq_count (or one of the sq_putcounts) has already been
7455  * decremented by the caller, and if SQ_QUEUED, we need to call
7456  * drain_syncq (the global syncq drain).
7457  * If putnext_tail is called with the SQ_EXCL bit set, we are in
7458  * one of two states, non-CIPUT perimeter, and we need to clear
7459  * it, or we went exclusive in the put procedure.  In any case,
7460  * we want to clear the bit now, and it is probably easier to do
7461  * this at the beginning of this function (remember, we hold
7462  * the SQLOCK).  Lastly, if there are other messages queued
7463  * on the syncq (and not for our destination), enable the syncq
7464  * for background work.
7465  */
7466 
7467 /* ARGSUSED */
7468 void
7469 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7470 {
7471         uint16_t        flags = sq->sq_flags;
7472 
7473         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7474         ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7475 
7476         /* Clear SQ_EXCL if set in passflags */
7477         if (passflags & SQ_EXCL) {
7478                 flags &= ~SQ_EXCL;
7479         }
7480         if (flags & SQ_WANTWAKEUP) {
7481                 flags &= ~SQ_WANTWAKEUP;
7482                 cv_broadcast(&sq->sq_wait);
7483         }
7484         if (flags & SQ_WANTEXWAKEUP) {
7485                 flags &= ~SQ_WANTEXWAKEUP;
7486                 cv_broadcast(&sq->sq_exitwait);
7487         }
7488         sq->sq_flags = flags;
7489 
7490         /*
7491          * We have cleared SQ_EXCL if we were asked to, and started
7492          * the wakeup process for waiters.  If there are no writers
7493          * then we need to drain the syncq if we were told to, or
7494          * enable the background thread to do it.
7495          */
7496         if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7497                 if ((passflags & SQ_QUEUED) ||
7498                     (sq->sq_svcflags & SQ_DISABLED)) {
7499                         /* drain_syncq will take care of events in the list */
7500                         drain_syncq(sq);
7501                         return;
7502                 } else if (flags & SQ_QUEUED) {
7503                         sqenable(sq);
7504                 }
7505         }
7506         /* Drop the SQLOCK on exit */
7507         mutex_exit(SQLOCK(sq));
7508         TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7509             "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7510 }
7511 
7512 void
7513 set_qend(queue_t *q)
7514 {
7515         mutex_enter(QLOCK(q));
7516         if (!O_SAMESTR(q))
7517                 q->q_flag |= QEND;
7518         else
7519                 q->q_flag &= ~QEND;
7520         mutex_exit(QLOCK(q));
7521         q = _OTHERQ(q);
7522         mutex_enter(QLOCK(q));
7523         if (!O_SAMESTR(q))
7524                 q->q_flag |= QEND;
7525         else
7526                 q->q_flag &= ~QEND;
7527         mutex_exit(QLOCK(q));
7528 }
7529 
7530 /*
7531  * Set QFULL in next service procedure queue (that cares) if not already
7532  * set and if there are already more messages on the syncq than
7533  * sq_max_size.  If sq_max_size is 0, no flow control will be asserted on
7534  * any syncq.
7535  *
7536  * The fq here is the next queue with a service procedure.  This is where
7537  * we would fail canputnext, so this is where we need to set QFULL.
7538  * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7539  *
7540  * We already have QLOCK at this point. To avoid cross-locks with
7541  * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7542  * SQLOCK and sd_reflock, we need to drop respective locks first.
7543  */
7544 void
7545 set_qfull(queue_t *q)
7546 {
7547         queue_t         *fq = NULL;
7548 
7549         ASSERT(MUTEX_HELD(QLOCK(q)));
7550         if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7551             (q->q_syncqmsgs > sq_max_size)) {
7552                 if ((fq = q->q_nfsrv) == q) {
7553                         fq->q_flag |= QFULL;
7554                 } else {
7555                         mutex_exit(QLOCK(q));
7556                         mutex_enter(QLOCK(fq));
7557                         fq->q_flag |= QFULL;
7558                         mutex_exit(QLOCK(fq));
7559                         mutex_enter(QLOCK(q));
7560                 }
7561         }
7562 }
7563 
7564 void
7565 clr_qfull(queue_t *q)
7566 {
7567         queue_t *oq = q;
7568 
7569         q = q->q_nfsrv;
7570         /* Fast check if there is any work to do before getting the lock. */
7571         if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7572                 return;
7573         }
7574 
7575         /*
7576          * Do not reset QFULL (and backenable) if the q_count is the reason
7577          * for QFULL being set.
7578          */
7579         mutex_enter(QLOCK(q));
7580         /*
7581          * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7582          * Hence clear the QFULL.
7583          * If both q_count and q_mblkcnt are less than the hiwat mark,
7584          * clear the QFULL.
7585          */
7586         if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7587             (q->q_mblkcnt < q->q_hiwat))) {
7588                 q->q_flag &= ~QFULL;
7589                 /*
7590                  * A little more confusing, how about this way:
7591                  * if someone wants to write,
7592                  * AND
7593                  *    both counts are less than the lowat mark
7594                  *    OR
7595                  *    the lowat mark is zero
7596                  * THEN
7597                  * backenable
7598                  */
7599                 if ((q->q_flag & QWANTW) &&
7600                     (((q->q_count < q->q_lowat) &&
7601                     (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7602                         q->q_flag &= ~QWANTW;
7603                         mutex_exit(QLOCK(q));
7604                         backenable(oq, 0);
7605                 } else
7606                         mutex_exit(QLOCK(q));
7607         } else
7608                 mutex_exit(QLOCK(q));
7609 }
7610 
7611 /*
7612  * Set the forward service procedure pointer.
7613  *
7614  * Called at insert-time to cache a queue's next forward service procedure in
7615  * q_nfsrv; used by canput() and canputnext().  If the queue to be inserted
7616  * has a service procedure then q_nfsrv points to itself.  If the queue to be
7617  * inserted does not have a service procedure, then q_nfsrv points to the next
7618  * queue forward that has a service procedure.  If the queue is at the logical
7619  * end of the stream (driver for write side, stream head for the read side)
7620  * and does not have a service procedure, then q_nfsrv also points to itself.
7621  */
7622 void
7623 set_nfsrv_ptr(
7624         queue_t  *rnew,         /* read queue pointer to new module */
7625         queue_t  *wnew,         /* write queue pointer to new module */
7626         queue_t  *prev_rq,      /* read queue pointer to the module above */
7627         queue_t  *prev_wq)      /* write queue pointer to the module above */
7628 {
7629         queue_t *qp;
7630 
7631         if (prev_wq->q_next == NULL) {
7632                 /*
7633                  * Insert the driver, initialize the driver and stream head.
7634                  * In this case, prev_rq/prev_wq should be the stream head.
7635                  * _I_INSERT does not allow inserting a driver.  Make sure
7636                  * that it is not an insertion.
7637                  */
7638                 ASSERT(!(rnew->q_flag & _QINSERTING));
7639                 wnew->q_nfsrv = wnew;
7640                 if (rnew->q_qinfo->qi_srvp)
7641                         rnew->q_nfsrv = rnew;
7642                 else
7643                         rnew->q_nfsrv = prev_rq;
7644                 prev_rq->q_nfsrv = prev_rq;
7645                 prev_wq->q_nfsrv = prev_wq;
7646         } else {
7647                 /*
7648                  * set up read side q_nfsrv pointer.  This MUST be done
7649                  * before setting the write side, because the setting of
7650                  * the write side for a fifo may depend on it.
7651                  *
7652                  * Suppose we have a fifo that only has pipemod pushed.
7653                  * pipemod has no read or write service procedures, so
7654                  * nfsrv for both pipemod queues points to prev_rq (the
7655                  * stream read head).  Now push bufmod (which has only a
7656                  * read service procedure).  Doing the write side first,
7657                  * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7658                  * is WRONG; the next queue forward from wnew with a
7659                  * service procedure will be rnew, not the stream read head.
7660                  * Since the downstream queue (which in the case of a fifo
7661                  * is the read queue rnew) can affect upstream queues, it
7662                  * needs to be done first.  Setting up the read side first
7663                  * sets nfsrv for both pipemod queues to rnew and then
7664                  * when the write side is set up, wnew-q_nfsrv will also
7665                  * point to rnew.
7666                  */
7667                 if (rnew->q_qinfo->qi_srvp) {
7668                         /*
7669                          * use _OTHERQ() because, if this is a pipe, next
7670                          * module may have been pushed from other end and
7671                          * q_next could be a read queue.
7672                          */
7673                         qp = _OTHERQ(prev_wq->q_next);
7674                         while (qp && qp->q_nfsrv != qp) {
7675                                 qp->q_nfsrv = rnew;
7676                                 qp = backq(qp);
7677                         }
7678                         rnew->q_nfsrv = rnew;
7679                 } else
7680                         rnew->q_nfsrv = prev_rq->q_nfsrv;
7681 
7682                 /* set up write side q_nfsrv pointer */
7683                 if (wnew->q_qinfo->qi_srvp) {
7684                         wnew->q_nfsrv = wnew;
7685 
7686                         /*
7687                          * For insertion, need to update nfsrv of the modules
7688                          * above which do not have a service routine.
7689                          */
7690                         if (rnew->q_flag & _QINSERTING) {
7691                                 for (qp = prev_wq;
7692                                     qp != NULL && qp->q_nfsrv != qp;
7693                                     qp = backq(qp)) {
7694                                         qp->q_nfsrv = wnew->q_nfsrv;
7695                                 }
7696                         }
7697                 } else {
7698                         if (prev_wq->q_next == prev_rq)
7699                                 /*
7700                                  * Since prev_wq/prev_rq are the middle of a
7701                                  * fifo, wnew/rnew will also be the middle of
7702                                  * a fifo and wnew's nfsrv is same as rnew's.
7703                                  */
7704                                 wnew->q_nfsrv = rnew->q_nfsrv;
7705                         else
7706                                 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7707                 }
7708         }
7709 }
7710 
7711 /*
7712  * Reset the forward service procedure pointer; called at remove-time.
7713  */
7714 void
7715 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7716 {
7717         queue_t *tmp_qp;
7718 
7719         /* Reset the write side q_nfsrv pointer for _I_REMOVE */
7720         if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7721                 for (tmp_qp = backq(wqp);
7722                     tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7723                     tmp_qp = backq(tmp_qp)) {
7724                         tmp_qp->q_nfsrv = wqp->q_nfsrv;
7725                 }
7726         }
7727 
7728         /* reset the read side q_nfsrv pointer */
7729         if (rqp->q_qinfo->qi_srvp) {
7730                 if (wqp->q_next) {   /* non-driver case */
7731                         tmp_qp = _OTHERQ(wqp->q_next);
7732                         while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7733                                 /* Note that rqp->q_next cannot be NULL */
7734                                 ASSERT(rqp->q_next != NULL);
7735                                 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7736                                 tmp_qp = backq(tmp_qp);
7737                         }
7738                 }
7739         }
7740 }
7741 
7742 /*
7743  * This routine should be called after all stream geometry changes to update
7744  * the stream head cached struio() rd/wr queue pointers. Note must be called
7745  * with the streamlock()ed.
7746  *
7747  * Note: only enables Synchronous STREAMS for a side of a Stream which has
7748  *       an explicit synchronous barrier module queue. That is, a queue that
7749  *       has specified a struio() type.
7750  */
7751 static void
7752 strsetuio(stdata_t *stp)
7753 {
7754         queue_t *wrq;
7755 
7756         if (stp->sd_flag & STPLEX) {
7757                 /*
7758                  * Not streamhead, but a mux, so no Synchronous STREAMS.
7759                  */
7760                 stp->sd_struiowrq = NULL;
7761                 stp->sd_struiordq = NULL;
7762                 return;
7763         }
7764         /*
7765          * Scan the write queue(s) while synchronous
7766          * until we find a qinfo uio type specified.
7767          */
7768         wrq = stp->sd_wrq->q_next;
7769         while (wrq) {
7770                 if (wrq->q_struiot == STRUIOT_NONE) {
7771                         wrq = 0;
7772                         break;
7773                 }
7774                 if (wrq->q_struiot != STRUIOT_DONTCARE)
7775                         break;
7776                 if (! _SAMESTR(wrq)) {
7777                         wrq = 0;
7778                         break;
7779                 }
7780                 wrq = wrq->q_next;
7781         }
7782         stp->sd_struiowrq = wrq;
7783         /*
7784          * Scan the read queue(s) while synchronous
7785          * until we find a qinfo uio type specified.
7786          */
7787         wrq = stp->sd_wrq->q_next;
7788         while (wrq) {
7789                 if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7790                         wrq = 0;
7791                         break;
7792                 }
7793                 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7794                         break;
7795                 if (! _SAMESTR(wrq)) {
7796                         wrq = 0;
7797                         break;
7798                 }
7799                 wrq = wrq->q_next;
7800         }
7801         stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7802 }
7803 
7804 /*
7805  * pass_wput, unblocks the passthru queues, so that
7806  * messages can arrive at muxs lower read queue, before
7807  * I_LINK/I_UNLINK is acked/nacked.
7808  */
7809 static void
7810 pass_wput(queue_t *q, mblk_t *mp)
7811 {
7812         syncq_t *sq;
7813 
7814         sq = _RD(q)->q_syncq;
7815         if (sq->sq_flags & SQ_BLOCKED)
7816                 unblocksq(sq, SQ_BLOCKED, 0);
7817         putnext(q, mp);
7818 }
7819 
7820 /*
7821  * Set up queues for the link/unlink.
7822  * Create a new queue and block it and then insert it
7823  * below the stream head on the lower stream.
7824  * This prevents any messages from arriving during the setq
7825  * as well as while the mux is processing the LINK/I_UNLINK.
7826  * The blocked passq is unblocked once the LINK/I_UNLINK has
7827  * been acked or nacked or if a message is generated and sent
7828  * down muxs write put procedure.
7829  * See pass_wput().
7830  *
7831  * After the new queue is inserted, all messages coming from below are
7832  * blocked. The call to strlock will ensure that all activity in the stream head
7833  * read queue syncq is stopped (sq_count drops to zero).
7834  */
7835 static queue_t *
7836 link_addpassthru(stdata_t *stpdown)
7837 {
7838         queue_t *passq;
7839         sqlist_t sqlist;
7840 
7841         passq = allocq();
7842         STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7843         /* setq might sleep in allocator - avoid holding locks. */
7844         setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7845             SQ_CI|SQ_CO, B_FALSE);
7846         claimq(passq);
7847         blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7848         insertq(STREAM(passq), passq);
7849 
7850         /*
7851          * Use strlock() to wait for the stream head sq_count to drop to zero
7852          * since we are going to change q_ptr in the stream head.  Note that
7853          * insertq() doesn't wait for any syncq counts to drop to zero.
7854          */
7855         sqlist.sqlist_head = NULL;
7856         sqlist.sqlist_index = 0;
7857         sqlist.sqlist_size = sizeof (sqlist_t);
7858         sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7859         strlock(stpdown, &sqlist);
7860         strunlock(stpdown, &sqlist);
7861 
7862         releaseq(passq);
7863         return (passq);
7864 }
7865 
7866 /*
7867  * Let messages flow up into the mux by removing
7868  * the passq.
7869  */
7870 static void
7871 link_rempassthru(queue_t *passq)
7872 {
7873         claimq(passq);
7874         removeq(passq);
7875         releaseq(passq);
7876         freeq(passq);
7877 }
7878 
7879 /*
7880  * Wait for the condition variable pointed to by `cvp' to be signaled,
7881  * or for `tim' milliseconds to elapse, whichever comes first.  If `tim'
7882  * is negative, then there is no time limit.  If `nosigs' is non-zero,
7883  * then the wait will be non-interruptible.
7884  *
7885  * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7886  */
7887 clock_t
7888 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7889 {
7890         clock_t ret;
7891 
7892         if (tim < 0) {
7893                 if (nosigs) {
7894                         cv_wait(cvp, mp);
7895                         ret = 1;
7896                 } else {
7897                         ret = cv_wait_sig(cvp, mp);
7898                 }
7899         } else if (tim > 0) {
7900                 /*
7901                  * convert milliseconds to clock ticks
7902                  */
7903                 if (nosigs) {
7904                         ret = cv_reltimedwait(cvp, mp,
7905                             MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7906                 } else {
7907                         ret = cv_reltimedwait_sig(cvp, mp,
7908                             MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7909                 }
7910         } else {
7911                 ret = -1;
7912         }
7913         return (ret);
7914 }
7915 
7916 /*
7917  * Wait until the stream head can determine if it is at the mark but
7918  * don't wait forever to prevent a race condition between the "mark" state
7919  * in the stream head and any mark state in the caller/user of this routine.
7920  *
7921  * This is used by sockets and for a socket it would be incorrect
7922  * to return a failure for SIOCATMARK when there is no data in the receive
7923  * queue and the marked urgent data is traveling up the stream.
7924  *
7925  * This routine waits until the mark is known by waiting for one of these
7926  * three events:
7927  *      The stream head read queue becoming non-empty (including an EOF).
7928  *      The STRATMARK flag being set (due to a MSGMARKNEXT message).
7929  *      The STRNOTATMARK flag being set (which indicates that the transport
7930  *      has sent a MSGNOTMARKNEXT message to indicate that it is not at
7931  *      the mark).
7932  *
7933  * The routine returns 1 if the stream is at the mark; 0 if it can
7934  * be determined that the stream is not at the mark.
7935  * If the wait times out and it can't determine
7936  * whether or not the stream might be at the mark the routine will return -1.
7937  *
7938  * Note: This routine should only be used when a mark is pending i.e.,
7939  * in the socket case the SIGURG has been posted.
7940  * Note2: This can not wakeup just because synchronous streams indicate
7941  * that data is available since it is not possible to use the synchronous
7942  * streams interfaces to determine the b_flag value for the data queued below
7943  * the stream head.
7944  */
7945 int
7946 strwaitmark(vnode_t *vp)
7947 {
7948         struct stdata *stp = vp->v_stream;
7949         queue_t *rq = _RD(stp->sd_wrq);
7950         int mark;
7951 
7952         mutex_enter(&stp->sd_lock);
7953         while (rq->q_first == NULL &&
7954             !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7955                 stp->sd_flag |= RSLEEP;
7956 
7957                 /* Wait for 100 milliseconds for any state change. */
7958                 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7959                         mutex_exit(&stp->sd_lock);
7960                         return (-1);
7961                 }
7962         }
7963         if (stp->sd_flag & STRATMARK)
7964                 mark = 1;
7965         else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7966                 mark = 1;
7967         else
7968                 mark = 0;
7969 
7970         mutex_exit(&stp->sd_lock);
7971         return (mark);
7972 }
7973 
7974 /*
7975  * Set a read side error. If persist is set change the socket error
7976  * to persistent. If errfunc is set install the function as the exported
7977  * error handler.
7978  */
7979 void
7980 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
7981 {
7982         struct stdata *stp = vp->v_stream;
7983 
7984         mutex_enter(&stp->sd_lock);
7985         stp->sd_rerror = error;
7986         if (error == 0 && errfunc == NULL)
7987                 stp->sd_flag &= ~STRDERR;
7988         else
7989                 stp->sd_flag |= STRDERR;
7990         if (persist) {
7991                 stp->sd_flag &= ~STRDERRNONPERSIST;
7992         } else {
7993                 stp->sd_flag |= STRDERRNONPERSIST;
7994         }
7995         stp->sd_rderrfunc = errfunc;
7996         if (error != 0 || errfunc != NULL) {
7997                 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);      /* readers */
7998                 cv_broadcast(&stp->sd_wrq->q_wait);           /* writers */
7999                 cv_broadcast(&stp->sd_monitor);                  /* ioctllers */
8000 
8001                 mutex_exit(&stp->sd_lock);
8002                 pollwakeup(&stp->sd_pollist, POLLERR);
8003                 mutex_enter(&stp->sd_lock);
8004 
8005                 if (stp->sd_sigflags & S_ERROR)
8006                         strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8007         }
8008         mutex_exit(&stp->sd_lock);
8009 }
8010 
8011 /*
8012  * Set a write side error. If persist is set change the socket error
8013  * to persistent.
8014  */
8015 void
8016 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8017 {
8018         struct stdata *stp = vp->v_stream;
8019 
8020         mutex_enter(&stp->sd_lock);
8021         stp->sd_werror = error;
8022         if (error == 0 && errfunc == NULL)
8023                 stp->sd_flag &= ~STWRERR;
8024         else
8025                 stp->sd_flag |= STWRERR;
8026         if (persist) {
8027                 stp->sd_flag &= ~STWRERRNONPERSIST;
8028         } else {
8029                 stp->sd_flag |= STWRERRNONPERSIST;
8030         }
8031         stp->sd_wrerrfunc = errfunc;
8032         if (error != 0 || errfunc != NULL) {
8033                 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);      /* readers */
8034                 cv_broadcast(&stp->sd_wrq->q_wait);           /* writers */
8035                 cv_broadcast(&stp->sd_monitor);                  /* ioctllers */
8036 
8037                 mutex_exit(&stp->sd_lock);
8038                 pollwakeup(&stp->sd_pollist, POLLERR);
8039                 mutex_enter(&stp->sd_lock);
8040 
8041                 if (stp->sd_sigflags & S_ERROR)
8042                         strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8043         }
8044         mutex_exit(&stp->sd_lock);
8045 }
8046 
8047 /*
8048  * Make the stream return 0 (EOF) when all data has been read.
8049  * No effect on write side.
8050  */
8051 void
8052 strseteof(vnode_t *vp, int eof)
8053 {
8054         struct stdata *stp = vp->v_stream;
8055 
8056         mutex_enter(&stp->sd_lock);
8057         if (!eof) {
8058                 stp->sd_flag &= ~STREOF;
8059                 mutex_exit(&stp->sd_lock);
8060                 return;
8061         }
8062         stp->sd_flag |= STREOF;
8063         if (stp->sd_flag & RSLEEP) {
8064                 stp->sd_flag &= ~RSLEEP;
8065                 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8066         }
8067 
8068         mutex_exit(&stp->sd_lock);
8069         pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8070         mutex_enter(&stp->sd_lock);
8071 
8072         if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8073                 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8074         mutex_exit(&stp->sd_lock);
8075 }
8076 
8077 void
8078 strflushrq(vnode_t *vp, int flag)
8079 {
8080         struct stdata *stp = vp->v_stream;
8081 
8082         mutex_enter(&stp->sd_lock);
8083         flushq(_RD(stp->sd_wrq), flag);
8084         mutex_exit(&stp->sd_lock);
8085 }
8086 
8087 void
8088 strsetrputhooks(vnode_t *vp, uint_t flags,
8089                 msgfunc_t protofunc, msgfunc_t miscfunc)
8090 {
8091         struct stdata *stp = vp->v_stream;
8092 
8093         mutex_enter(&stp->sd_lock);
8094 
8095         if (protofunc == NULL)
8096                 stp->sd_rprotofunc = strrput_proto;
8097         else
8098                 stp->sd_rprotofunc = protofunc;
8099 
8100         if (miscfunc == NULL)
8101                 stp->sd_rmiscfunc = strrput_misc;
8102         else
8103                 stp->sd_rmiscfunc = miscfunc;
8104 
8105         if (flags & SH_CONSOL_DATA)
8106                 stp->sd_rput_opt |= SR_CONSOL_DATA;
8107         else
8108                 stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8109 
8110         if (flags & SH_SIGALLDATA)
8111                 stp->sd_rput_opt |= SR_SIGALLDATA;
8112         else
8113                 stp->sd_rput_opt &= ~SR_SIGALLDATA;
8114 
8115         if (flags & SH_IGN_ZEROLEN)
8116                 stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8117         else
8118                 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8119 
8120         mutex_exit(&stp->sd_lock);
8121 }
8122 
8123 void
8124 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8125 {
8126         struct stdata *stp = vp->v_stream;
8127 
8128         mutex_enter(&stp->sd_lock);
8129         stp->sd_closetime = closetime;
8130 
8131         if (flags & SH_SIGPIPE)
8132                 stp->sd_wput_opt |= SW_SIGPIPE;
8133         else
8134                 stp->sd_wput_opt &= ~SW_SIGPIPE;
8135         if (flags & SH_RECHECK_ERR)
8136                 stp->sd_wput_opt |= SW_RECHECK_ERR;
8137         else
8138                 stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8139 
8140         mutex_exit(&stp->sd_lock);
8141 }
8142 
8143 void
8144 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8145 {
8146         struct stdata *stp = vp->v_stream;
8147 
8148         mutex_enter(&stp->sd_lock);
8149 
8150         stp->sd_rputdatafunc = rdatafunc;
8151         stp->sd_wputdatafunc = wdatafunc;
8152 
8153         mutex_exit(&stp->sd_lock);
8154 }
8155 
8156 /* Used within framework when the queue is already locked */
8157 void
8158 qenable_locked(queue_t *q)
8159 {
8160         stdata_t *stp = STREAM(q);
8161 
8162         ASSERT(MUTEX_HELD(QLOCK(q)));
8163 
8164         if (!q->q_qinfo->qi_srvp)
8165                 return;
8166 
8167         /*
8168          * Do not place on run queue if already enabled or closing.
8169          */
8170         if (q->q_flag & (QWCLOSE|QENAB))
8171                 return;
8172 
8173         /*
8174          * mark queue enabled and place on run list if it is not already being
8175          * serviced. If it is serviced, the runservice() function will detect
8176          * that QENAB is set and call service procedure before clearing
8177          * QINSERVICE flag.
8178          */
8179         q->q_flag |= QENAB;
8180         if (q->q_flag & QINSERVICE)
8181                 return;
8182 
8183         /* Record the time of qenable */
8184         q->q_qtstamp = ddi_get_lbolt();
8185 
8186         /*
8187          * Put the queue in the stp list and schedule it for background
8188          * processing if it is not already scheduled or if stream head does not
8189          * intent to process it in the foreground later by setting
8190          * STRS_WILLSERVICE flag.
8191          */
8192         mutex_enter(&stp->sd_qlock);
8193         /*
8194          * If there are already something on the list, stp flags should show
8195          * intention to drain it.
8196          */
8197         IMPLY(STREAM_NEEDSERVICE(stp),
8198             (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8199 
8200         ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8201         stp->sd_nqueues++;
8202 
8203         /*
8204          * If no one will drain this stream we are the first producer and
8205          * need to schedule it for background thread.
8206          */
8207         if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8208                 /*
8209                  * No one will service this stream later, so we have to
8210                  * schedule it now.
8211                  */
8212                 STRSTAT(stenables);
8213                 stp->sd_svcflags |= STRS_SCHEDULED;
8214                 stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8215                     (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8216 
8217                 if (stp->sd_servid == NULL) {
8218                         /*
8219                          * Task queue failed so fail over to the backup
8220                          * servicing thread.
8221                          */
8222                         STRSTAT(taskqfails);
8223                         /*
8224                          * It is safe to clear STRS_SCHEDULED flag because it
8225                          * was set by this thread above.
8226                          */
8227                         stp->sd_svcflags &= ~STRS_SCHEDULED;
8228 
8229                         /*
8230                          * Failover scheduling is protected by service_queue
8231                          * lock.
8232                          */
8233                         mutex_enter(&service_queue);
8234                         ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8235                         ASSERT(q->q_link == NULL);
8236                         /*
8237                          * Append the queue to qhead/qtail list.
8238                          */
8239                         if (qhead == NULL)
8240                                 qhead = q;
8241                         else
8242                                 qtail->q_link = q;
8243                         qtail = q;
8244                         /*
8245                          * Clear stp queue list.
8246                          */
8247                         stp->sd_qhead = stp->sd_qtail = NULL;
8248                         stp->sd_nqueues = 0;
8249                         /*
8250                          * Wakeup background queue processing thread.
8251                          */
8252                         cv_signal(&services_to_run);
8253                         mutex_exit(&service_queue);
8254                 }
8255         }
8256         mutex_exit(&stp->sd_qlock);
8257 }
8258 
8259 static void
8260 queue_service(queue_t *q)
8261 {
8262         /*
8263          * The queue in the list should have
8264          * QENAB flag set and should not have
8265          * QINSERVICE flag set. QINSERVICE is
8266          * set when the queue is dequeued and
8267          * qenable_locked doesn't enqueue a
8268          * queue with QINSERVICE set.
8269          */
8270 
8271         ASSERT(!(q->q_flag & QINSERVICE));
8272         ASSERT((q->q_flag & QENAB));
8273         mutex_enter(QLOCK(q));
8274         q->q_flag &= ~QENAB;
8275         q->q_flag |= QINSERVICE;
8276         mutex_exit(QLOCK(q));
8277         runservice(q);
8278 }
8279 
8280 static void
8281 syncq_service(syncq_t *sq)
8282 {
8283         STRSTAT(syncqservice);
8284         mutex_enter(SQLOCK(sq));
8285         ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8286         ASSERT(sq->sq_servcount != 0);
8287         ASSERT(sq->sq_next == NULL);
8288 
8289         /* if we came here from the background thread, clear the flag */
8290         if (sq->sq_svcflags & SQ_BGTHREAD)
8291                 sq->sq_svcflags &= ~SQ_BGTHREAD;
8292 
8293         /* let drain_syncq know that it's being called in the background */
8294         sq->sq_svcflags |= SQ_SERVICE;
8295         drain_syncq(sq);
8296 }
8297 
8298 static void
8299 qwriter_outer_service(syncq_t *outer)
8300 {
8301         /*
8302          * Note that SQ_WRITER is used on the outer perimeter
8303          * to signal that a qwriter(OUTER) is either investigating
8304          * running or that it is actually running a function.
8305          */
8306         outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8307 
8308         /*
8309          * All inner syncq are empty and have SQ_WRITER set
8310          * to block entering the outer perimeter.
8311          *
8312          * We do not need to explicitly call write_now since
8313          * outer_exit does it for us.
8314          */
8315         outer_exit(outer);
8316 }
8317 
8318 static void
8319 mblk_free(mblk_t *mp)
8320 {
8321         dblk_t *dbp = mp->b_datap;
8322         frtn_t *frp = dbp->db_frtnp;
8323 
8324         mp->b_next = NULL;
8325         if (dbp->db_fthdr != NULL)
8326                 str_ftfree(dbp);
8327 
8328         ASSERT(dbp->db_fthdr == NULL);
8329         frp->free_func(frp->free_arg);
8330         ASSERT(dbp->db_mblk == mp);
8331 
8332         if (dbp->db_credp != NULL) {
8333                 crfree(dbp->db_credp);
8334                 dbp->db_credp = NULL;
8335         }
8336         dbp->db_cpid = -1;
8337         dbp->db_struioflag = 0;
8338         dbp->db_struioun.cksum.flags = 0;
8339 
8340         kmem_cache_free(dbp->db_cache, dbp);
8341 }
8342 
8343 /*
8344  * Background processing of the stream queue list.
8345  */
8346 static void
8347 stream_service(stdata_t *stp)
8348 {
8349         queue_t *q;
8350 
8351         mutex_enter(&stp->sd_qlock);
8352 
8353         STR_SERVICE(stp, q);
8354 
8355         stp->sd_svcflags &= ~STRS_SCHEDULED;
8356         stp->sd_servid = NULL;
8357         cv_signal(&stp->sd_qcv);
8358         mutex_exit(&stp->sd_qlock);
8359 }
8360 
8361 /*
8362  * Foreground processing of the stream queue list.
8363  */
8364 void
8365 stream_runservice(stdata_t *stp)
8366 {
8367         queue_t *q;
8368 
8369         mutex_enter(&stp->sd_qlock);
8370         STRSTAT(rservice);
8371         /*
8372          * We are going to drain this stream queue list, so qenable_locked will
8373          * not schedule it until we finish.
8374          */
8375         stp->sd_svcflags |= STRS_WILLSERVICE;
8376 
8377         STR_SERVICE(stp, q);
8378 
8379         stp->sd_svcflags &= ~STRS_WILLSERVICE;
8380         mutex_exit(&stp->sd_qlock);
8381         /*
8382          * Help backup background thread to drain the qhead/qtail list.
8383          */
8384         while (qhead != NULL) {
8385                 STRSTAT(qhelps);
8386                 mutex_enter(&service_queue);
8387                 DQ(q, qhead, qtail, q_link);
8388                 mutex_exit(&service_queue);
8389                 if (q != NULL)
8390                         queue_service(q);
8391         }
8392 }
8393 
8394 void
8395 stream_willservice(stdata_t *stp)
8396 {
8397         mutex_enter(&stp->sd_qlock);
8398         stp->sd_svcflags |= STRS_WILLSERVICE;
8399         mutex_exit(&stp->sd_qlock);
8400 }
8401 
8402 /*
8403  * Replace the cred currently in the mblk with a different one.
8404  * Also update db_cpid.
8405  */
8406 void
8407 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8408 {
8409         dblk_t *dbp = mp->b_datap;
8410         cred_t *ocr = dbp->db_credp;
8411 
8412         ASSERT(cr != NULL);
8413 
8414         if (cr != ocr) {
8415                 crhold(dbp->db_credp = cr);
8416                 if (ocr != NULL)
8417                         crfree(ocr);
8418         }
8419         /* Don't overwrite with NOPID */
8420         if (cpid != NOPID)
8421                 dbp->db_cpid = cpid;
8422 }
8423 
8424 /*
8425  * If the src message has a cred, then replace the cred currently in the mblk
8426  * with it.
8427  * Also update db_cpid.
8428  */
8429 void
8430 mblk_copycred(mblk_t *mp, const mblk_t *src)
8431 {
8432         dblk_t *dbp = mp->b_datap;
8433         cred_t *cr, *ocr;
8434         pid_t cpid;
8435 
8436         cr = msg_getcred(src, &cpid);
8437         if (cr == NULL)
8438                 return;
8439 
8440         ocr = dbp->db_credp;
8441         if (cr != ocr) {
8442                 crhold(dbp->db_credp = cr);
8443                 if (ocr != NULL)
8444                         crfree(ocr);
8445         }
8446         /* Don't overwrite with NOPID */
8447         if (cpid != NOPID)
8448                 dbp->db_cpid = cpid;
8449 }
8450 
8451 int
8452 hcksum_assoc(mblk_t *mp,  multidata_t *mmd, pdesc_t *pd,
8453     uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8454     uint32_t flags, int km_flags)
8455 {
8456         int rc = 0;
8457 
8458         ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8459         if (mp->b_datap->db_type == M_DATA) {
8460                 /* Associate values for M_DATA type */
8461                 DB_CKSUMSTART(mp) = (intptr_t)start;
8462                 DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8463                 DB_CKSUMEND(mp) = (intptr_t)end;
8464                 DB_CKSUMFLAGS(mp) = flags;
8465                 DB_CKSUM16(mp) = (uint16_t)value;
8466 
8467         } else {
8468                 pattrinfo_t pa_info;
8469 
8470                 ASSERT(mmd != NULL);
8471 
8472                 pa_info.type = PATTR_HCKSUM;
8473                 pa_info.len = sizeof (pattr_hcksum_t);
8474 
8475                 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8476                         pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8477 
8478                         hck->hcksum_start_offset = start;
8479                         hck->hcksum_stuff_offset = stuff;
8480                         hck->hcksum_end_offset = end;
8481                         hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8482                         hck->hcksum_flags = flags;
8483                 } else {
8484                         rc = -1;
8485                 }
8486         }
8487         return (rc);
8488 }
8489 
8490 void
8491 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8492     uint32_t *start, uint32_t *stuff, uint32_t *end,
8493     uint32_t *value, uint32_t *flags)
8494 {
8495         ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8496         if (mp->b_datap->db_type == M_DATA) {
8497                 if (flags != NULL) {
8498                         *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8499                         if ((*flags & (HCK_PARTIALCKSUM |
8500                             HCK_FULLCKSUM)) != 0) {
8501                                 if (value != NULL)
8502                                         *value = (uint32_t)DB_CKSUM16(mp);
8503                                 if ((*flags & HCK_PARTIALCKSUM) != 0) {
8504                                         if (start != NULL)
8505                                                 *start =
8506                                                     (uint32_t)DB_CKSUMSTART(mp);
8507                                         if (stuff != NULL)
8508                                                 *stuff =
8509                                                     (uint32_t)DB_CKSUMSTUFF(mp);
8510                                         if (end != NULL)
8511                                                 *end =
8512                                                     (uint32_t)DB_CKSUMEND(mp);
8513                                 }
8514                         }
8515                 }
8516         } else {
8517                 pattrinfo_t hck_attr = {PATTR_HCKSUM};
8518 
8519                 ASSERT(mmd != NULL);
8520 
8521                 /* get hardware checksum attribute */
8522                 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8523                         pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8524 
8525                         ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8526                         if (flags != NULL)
8527                                 *flags = hck->hcksum_flags;
8528                         if (start != NULL)
8529                                 *start = hck->hcksum_start_offset;
8530                         if (stuff != NULL)
8531                                 *stuff = hck->hcksum_stuff_offset;
8532                         if (end != NULL)
8533                                 *end = hck->hcksum_end_offset;
8534                         if (value != NULL)
8535                                 *value = (uint32_t)
8536                                     hck->hcksum_cksum_val.inet_cksum;
8537                 }
8538         }
8539 }
8540 
8541 void
8542 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8543 {
8544         ASSERT(DB_TYPE(mp) == M_DATA);
8545         ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8546 
8547         /* Set the flags */
8548         DB_LSOFLAGS(mp) |= flags;
8549         DB_LSOMSS(mp) = mss;
8550 }
8551 
8552 void
8553 lso_info_cleanup(mblk_t *mp)
8554 {
8555         ASSERT(DB_TYPE(mp) == M_DATA);
8556 
8557         /* Clear the flags */
8558         DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8559         DB_LSOMSS(mp) = 0;
8560 }
8561 
8562 /*
8563  * Checksum buffer *bp for len bytes with psum partial checksum,
8564  * or 0 if none, and return the 16 bit partial checksum.
8565  */
8566 unsigned
8567 bcksum(uchar_t *bp, int len, unsigned int psum)
8568 {
8569         int odd = len & 1;
8570         extern unsigned int ip_ocsum();
8571 
8572         if (((intptr_t)bp & 1) == 0 && !odd) {
8573                 /*
8574                  * Bp is 16 bit aligned and len is multiple of 16 bit word.
8575                  */
8576                 return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8577         }
8578         if (((intptr_t)bp & 1) != 0) {
8579                 /*
8580                  * Bp isn't 16 bit aligned.
8581                  */
8582                 unsigned int tsum;
8583 
8584 #ifdef _LITTLE_ENDIAN
8585                 psum += *bp;
8586 #else
8587                 psum += *bp << 8;
8588 #endif
8589                 len--;
8590                 bp++;
8591                 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8592                 psum += (tsum << 8) & 0xffff | (tsum >> 8);
8593                 if (len & 1) {
8594                         bp += len - 1;
8595 #ifdef _LITTLE_ENDIAN
8596                         psum += *bp << 8;
8597 #else
8598                         psum += *bp;
8599 #endif
8600                 }
8601         } else {
8602                 /*
8603                  * Bp is 16 bit aligned.
8604                  */
8605                 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8606                 if (odd) {
8607                         bp += len - 1;
8608 #ifdef _LITTLE_ENDIAN
8609                         psum += *bp;
8610 #else
8611                         psum += *bp << 8;
8612 #endif
8613                 }
8614         }
8615         /*
8616          * Normalize psum to 16 bits before returning the new partial
8617          * checksum. The max psum value before normalization is 0x3FDFE.
8618          */
8619         return ((psum >> 16) + (psum & 0xFFFF));
8620 }
8621 
8622 boolean_t
8623 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8624 {
8625         boolean_t rc;
8626 
8627         ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8628         if (DB_TYPE(mp) == M_DATA) {
8629                 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8630         } else {
8631                 pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8632 
8633                 ASSERT(mmd != NULL);
8634                 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8635         }
8636         return (rc);
8637 }
8638 
8639 void
8640 freemsgchain(mblk_t *mp)
8641 {
8642         mblk_t  *next;
8643 
8644         while (mp != NULL) {
8645                 next = mp->b_next;
8646                 mp->b_next = NULL;
8647 
8648                 freemsg(mp);
8649                 mp = next;
8650         }
8651 }
8652 
8653 mblk_t *
8654 copymsgchain(mblk_t *mp)
8655 {
8656         mblk_t  *nmp = NULL;
8657         mblk_t  **nmpp = &nmp;
8658 
8659         for (; mp != NULL; mp = mp->b_next) {
8660                 if ((*nmpp = copymsg(mp)) == NULL) {
8661                         freemsgchain(nmp);
8662                         return (NULL);
8663                 }
8664 
8665                 nmpp = &((*nmpp)->b_next);
8666         }
8667 
8668         return (nmp);
8669 }
8670 
8671 /* NOTE: Do not add code after this point. */
8672 #undef QLOCK
8673 
8674 /*
8675  * Replacement for QLOCK macro for those that can't use it.
8676  */
8677 kmutex_t *
8678 QLOCK(queue_t *q)
8679 {
8680         return (&(q)->q_lock);
8681 }
8682 
8683 /*
8684  * Dummy runqueues/queuerun functions functions for backwards compatibility.
8685  */
8686 #undef runqueues
8687 void
8688 runqueues(void)
8689 {
8690 }
8691 
8692 #undef queuerun
8693 void
8694 queuerun(void)
8695 {
8696 }
8697 
8698 /*
8699  * Initialize the STR stack instance, which tracks autopush and persistent
8700  * links.
8701  */
8702 /* ARGSUSED */
8703 static void *
8704 str_stack_init(netstackid_t stackid, netstack_t *ns)
8705 {
8706         str_stack_t     *ss;
8707         int i;
8708 
8709         ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8710         ss->ss_netstack = ns;
8711 
8712         /*
8713          * set up autopush
8714          */
8715         sad_initspace(ss);
8716 
8717         /*
8718          * set up mux_node structures.
8719          */
8720         ss->ss_devcnt = devcnt;      /* In case it should change before free */
8721         ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8722             ss->ss_devcnt), KM_SLEEP);
8723         for (i = 0; i < ss->ss_devcnt; i++)
8724                 ss->ss_mux_nodes[i].mn_imaj = i;
8725         return (ss);
8726 }
8727 
8728 /*
8729  * Note: run at zone shutdown and not destroy so that the PLINKs are
8730  * gone by the time other cleanup happens from the destroy callbacks.
8731  */
8732 static void
8733 str_stack_shutdown(netstackid_t stackid, void *arg)
8734 {
8735         str_stack_t *ss = (str_stack_t *)arg;
8736         int i;
8737         cred_t *cr;
8738 
8739         cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8740         ASSERT(cr != NULL);
8741 
8742         /* Undo all the I_PLINKs for this zone */
8743         for (i = 0; i < ss->ss_devcnt; i++) {
8744                 struct mux_edge         *ep;
8745                 ldi_handle_t            lh;
8746                 ldi_ident_t             li;
8747                 int                     ret;
8748                 int                     rval;
8749                 dev_t                   rdev;
8750 
8751                 ep = ss->ss_mux_nodes[i].mn_outp;
8752                 if (ep == NULL)
8753                         continue;
8754                 ret = ldi_ident_from_major((major_t)i, &li);
8755                 if (ret != 0) {
8756                         continue;
8757                 }
8758                 rdev = ep->me_dev;
8759                 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8760                     cr, &lh, li);
8761                 if (ret != 0) {
8762                         ldi_ident_release(li);
8763                         continue;
8764                 }
8765 
8766                 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8767                     cr, &rval);
8768                 if (ret) {
8769                         (void) ldi_close(lh, FREAD|FWRITE, cr);
8770                         ldi_ident_release(li);
8771                         continue;
8772                 }
8773                 (void) ldi_close(lh, FREAD|FWRITE, cr);
8774 
8775                 /* Close layered handles */
8776                 ldi_ident_release(li);
8777         }
8778         crfree(cr);
8779 
8780         sad_freespace(ss);
8781 
8782         kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8783         ss->ss_mux_nodes = NULL;
8784 }
8785 
8786 /*
8787  * Free the structure; str_stack_shutdown did the other cleanup work.
8788  */
8789 /* ARGSUSED */
8790 static void
8791 str_stack_fini(netstackid_t stackid, void *arg)
8792 {
8793         str_stack_t     *ss = (str_stack_t *)arg;
8794 
8795         kmem_free(ss, sizeof (*ss));
8796 }